diff --git a/.github/workflows/ci_e2e.yml b/.github/workflows/ci_e2e.yml index eaffc04e62..7f5fc8a989 100644 --- a/.github/workflows/ci_e2e.yml +++ b/.github/workflows/ci_e2e.yml @@ -58,7 +58,9 @@ jobs: wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb sudo dpkg -i google-chrome*.deb sudo apt-get install -f -y - wget -N https://chromedriver.storage.googleapis.com/83.0.4103.39/chromedriver_linux64.zip + google-chrome -version + googleVersion=`google-chrome -version | awk '{print $3}'` + wget -N https://chromedriver.storage.googleapis.com/${googleVersion}/chromedriver_linux64.zip unzip chromedriver_linux64.zip sudo mv -f chromedriver /usr/local/share/chromedriver sudo ln -s /usr/local/share/chromedriver /usr/local/bin/chromedriver diff --git a/.github/workflows/ci_ut.yml b/.github/workflows/ci_ut.yml index 70790a7650..739c9be7fa 100644 --- a/.github/workflows/ci_ut.yml +++ b/.github/workflows/ci_ut.yml @@ -91,3 +91,30 @@ jobs: mkdir -p ${LOG_DIR} docker-compose -f $(pwd)/docker/docker-swarm/docker-compose.yml logs dolphinscheduler-postgresql > ${LOG_DIR}/db.txt continue-on-error: true + + Checkstyle: + name: Check code style + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + # In the checkout@v2, it doesn't support git submodule. Execute the commands manually. + - name: checkout submodules + shell: bash + run: | + git submodule sync --recursive + git -c protocol.version=2 submodule update --init --force --recursive --depth=1 + - name: check code style + env: + WORKDIR: ./ + REVIEWDOG_GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }} + CHECKSTYLE_CONFIG: style/checkstyle.xml + REVIEWDOG_VERSION: v0.10.2 + run: | + wget -O - -q https://github.com/checkstyle/checkstyle/releases/download/checkstyle-8.22/checkstyle-8.22-all.jar > /opt/checkstyle.jar + wget -O - -q https://raw.githubusercontent.com/reviewdog/reviewdog/master/install.sh | sh -s -- -b /opt ${REVIEWDOG_VERSION} + java -jar /opt/checkstyle.jar "${WORKDIR}" -c "${CHECKSTYLE_CONFIG}" -f xml \ + | /opt/reviewdog -f=checkstyle \ + -reporter="${INPUT_REPORTER:-github-pr-check}" \ + -filter-mode="${INPUT_FILTER_MODE:-added}" \ + -fail-on-error="${INPUT_FAIL_ON_ERROR:-false}" \ No newline at end of file diff --git a/.gitignore b/.gitignore index 17b0dc6610..7a99e2e4b0 100644 --- a/.gitignore +++ b/.gitignore @@ -19,6 +19,7 @@ third-party-dependencies.txt *.iws *.tgz .*.swp +.factorypath .vim .tmp **/node_modules diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e02ed113c4..f26b06e850 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -56,7 +56,7 @@ If remote branch has a new branch `DEV-1.0`, you need to synchronize this branch ``` git checkout -b dev-1.0 upstream/dev-1.0 -git push --set-upstream origin dev1.0 +git push --set-upstream origin dev-1.0 ``` ## Create your feature branch diff --git a/docker/build/Dockerfile b/docker/build/Dockerfile index d0f16d5d0d..ceb94ea8c5 100644 --- a/docker/build/Dockerfile +++ b/docker/build/Dockerfile @@ -27,7 +27,7 @@ ENV DEBIAN_FRONTEND noninteractive #If install slowly, you can replcae alpine's mirror with aliyun's mirror, Example: #RUN sed -i "s/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g" /etc/apk/repositories RUN apk update && \ - apk add dos2unix shadow bash openrc python python3 sudo vim wget iputils net-tools openssh-server py2-pip tini && \ + apk --update add --no-cache dos2unix shadow bash openrc python2 python3 sudo vim wget iputils net-tools openssh-server py-pip tini && \ apk add --update procps && \ openrc boot && \ pip install kazoo diff --git a/docker/build/README.md b/docker/build/README.md index bc516bc214..951f2d6b51 100644 --- a/docker/build/README.md +++ b/docker/build/README.md @@ -238,6 +238,10 @@ This environment variable sets max cpu load avg for `worker-server`. The default This environment variable sets reserved memory for `worker-server`. The default value is `0.1`. +**`WORKER_WEIGHT`** + +This environment variable sets port for `worker-server`. The default value is `100`. + **`WORKER_LISTEN_PORT`** This environment variable sets port for `worker-server`. The default value is `1234`. diff --git a/docker/build/README_zh_CN.md b/docker/build/README_zh_CN.md index c2affc0691..c4339a945c 100644 --- a/docker/build/README_zh_CN.md +++ b/docker/build/README_zh_CN.md @@ -238,6 +238,10 @@ Dolphin Scheduler映像使用了几个容易遗漏的环境变量。虽然这些 配置`worker-server`的保留内存，默认值 `0.1`。 +**`WORKER_WEIGHT`** + +配置`worker-server`的权重，默认之`100`。 + **`WORKER_LISTEN_PORT`** 配置`worker-server`的端口，默认值 `1234`。 diff --git a/docker/build/conf/dolphinscheduler/worker.properties.tpl b/docker/build/conf/dolphinscheduler/worker.properties.tpl index d596be94bc..83097dd9a4 100644 --- a/docker/build/conf/dolphinscheduler/worker.properties.tpl +++ b/docker/build/conf/dolphinscheduler/worker.properties.tpl @@ -34,4 +34,7 @@ worker.reserved.memory=${WORKER_RESERVED_MEMORY} #worker.listen.port=${WORKER_LISTEN_PORT} # default worker group -#worker.group=${WORKER_GROUP} \ No newline at end of file +#worker.groups=${WORKER_GROUP} + +# default worker weight +#worker.weight=${WORKER_WEIGHT} \ No newline at end of file diff --git a/docker/build/conf/zookeeper/zoo.cfg b/docker/build/conf/zookeeper/zoo.cfg index 7980d37ae9..94f92d0620 100644 --- a/docker/build/conf/zookeeper/zoo.cfg +++ b/docker/build/conf/zookeeper/zoo.cfg @@ -43,3 +43,5 @@ clientPort=2181 # Purge task interval in hours # Set to "0" to disable auto purge feature #autopurge.purgeInterval=1 +#Four Letter Words commands:stat,ruok,conf,isro +4lw.commands.whitelist=* diff --git a/docker/build/startup-init-conf.sh b/docker/build/startup-init-conf.sh index 73fdad6798..d5cd86f1a4 100644 --- a/docker/build/startup-init-conf.sh +++ b/docker/build/startup-init-conf.sh @@ -74,6 +74,7 @@ export WORKER_MAX_CPULOAD_AVG=${WORKER_MAX_CPULOAD_AVG:-"100"} export WORKER_RESERVED_MEMORY=${WORKER_RESERVED_MEMORY:-"0.1"} export WORKER_LISTEN_PORT=${WORKER_LISTEN_PORT:-"1234"} export WORKER_GROUP=${WORKER_GROUP:-"default"} +export WORKER_WEIGHT=${WORKER_WEIGHT:-"100"} #============================================================================ # Alert Server diff --git a/docker/docker-swarm/docker-compose.yml b/docker/docker-swarm/docker-compose.yml index 51eb0aeaa5..349b3ad790 100644 --- a/docker/docker-swarm/docker-compose.yml +++ b/docker/docker-swarm/docker-compose.yml @@ -187,6 +187,7 @@ services: WORKER_MAX_CPULOAD_AVG: "100" WORKER_RESERVED_MEMORY: "0.1" WORKER_GROUP: "default" + WORKER_WEIGHT: "100" DOLPHINSCHEDULER_DATA_BASEDIR_PATH: "/tmp/dolphinscheduler" DATABASE_HOST: dolphinscheduler-postgresql DATABASE_PORT: 5432 diff --git a/docker/docker-swarm/docker-stack.yml b/docker/docker-swarm/docker-stack.yml index ca9f7c88c7..dff4a47b2c 100644 --- a/docker/docker-swarm/docker-stack.yml +++ b/docker/docker-swarm/docker-stack.yml @@ -187,6 +187,7 @@ services: WORKER_MAX_CPULOAD_AVG: "100" WORKER_RESERVED_MEMORY: "0.1" WORKER_GROUP: "default" + WORKER_WEIGHT: "100" DOLPHINSCHEDULER_DATA_BASEDIR_PATH: "/tmp/dolphinscheduler" DATABASE_HOST: dolphinscheduler-postgresql DATABASE_PORT: 5432 diff --git a/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-worker.yaml b/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-worker.yaml index 1e08b67b53..569341c225 100644 --- a/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-worker.yaml +++ b/docker/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-worker.yaml @@ -31,6 +31,7 @@ data: WORKER_RESERVED_MEMORY: {{ .Values.worker.configmap.WORKER_RESERVED_MEMORY | quote }} WORKER_LISTEN_PORT: {{ .Values.worker.configmap.WORKER_LISTEN_PORT | quote }} WORKER_GROUP: {{ .Values.worker.configmap.WORKER_GROUP | quote }} + WORKER_WEIGHT: {{ .Values.worker.configmap.WORKER_WEIGHT | quote }} DOLPHINSCHEDULER_DATA_BASEDIR_PATH: {{ include "dolphinscheduler.worker.base.dir" . | quote }} dolphinscheduler_env.sh: |- {{- range .Values.worker.configmap.DOLPHINSCHEDULER_ENV }} diff --git a/docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml b/docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml index 51a83bcfa7..92c2c72398 100644 --- a/docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml +++ b/docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml @@ -162,6 +162,12 @@ spec: {{- else }} value: {{ .Values.externalZookeeper.zookeeperQuorum }} {{- end }} + - name: ZOOKEEPER_ROOT + {{- if .Values.zookeeper.enabled }} + value: "/dolphinscheduler" + {{- else }} + value: {{ .Values.externalZookeeper.zookeeperRoot }} + {{- end }} - name: RESOURCE_STORAGE_TYPE valueFrom: configMapKeyRef: diff --git a/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml b/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml index 0949127dda..e9dc7919ca 100644 --- a/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml +++ b/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml @@ -228,6 +228,12 @@ spec: {{- else }} value: {{ .Values.externalZookeeper.zookeeperQuorum }} {{- end }} + - name: ZOOKEEPER_ROOT + {{- if .Values.zookeeper.enabled }} + value: "/dolphinscheduler" + {{- else }} + value: {{ .Values.externalZookeeper.zookeeperRoot }} + {{- end }} - name: RESOURCE_STORAGE_TYPE valueFrom: configMapKeyRef: diff --git a/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml b/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml index 097f8d8580..ae562cc62b 100644 --- a/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml +++ b/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml @@ -162,6 +162,11 @@ spec: configMapKeyRef: name: {{ include "dolphinscheduler.fullname" . }}-worker key: WORKER_GROUP + - name: WORKER_WEUGHT + valueFrom: + configMapKeyRef: + name: {{ include "dolphinscheduler.fullname" . }}-worker + key: WORKER_WEIGHT - name: DOLPHINSCHEDULER_DATA_BASEDIR_PATH valueFrom: configMapKeyRef: @@ -225,6 +230,12 @@ spec: {{- else }} value: {{ .Values.externalZookeeper.zookeeperQuorum }} {{- end }} + - name: ZOOKEEPER_ROOT + {{- if .Values.zookeeper.enabled }} + value: "/dolphinscheduler" + {{- else }} + value: {{ .Values.externalZookeeper.zookeeperRoot }} + {{- end }} - name: RESOURCE_STORAGE_TYPE valueFrom: configMapKeyRef: diff --git a/docker/kubernetes/dolphinscheduler/values.yaml b/docker/kubernetes/dolphinscheduler/values.yaml index 8acb1d326a..3261b08401 100644 --- a/docker/kubernetes/dolphinscheduler/values.yaml +++ b/docker/kubernetes/dolphinscheduler/values.yaml @@ -201,6 +201,7 @@ worker: WORKER_RESERVED_MEMORY: "0.1" WORKER_LISTEN_PORT: "1234" WORKER_GROUP: "default" + WORKER_WEIGHT: "100" DOLPHINSCHEDULER_DATA_BASEDIR_PATH: "/tmp/dolphinscheduler" DOLPHINSCHEDULER_ENV: - "export HADOOP_HOME=/opt/soft/hadoop" diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java index 36f903c25d..ef1022755f 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtils.java @@ -14,13 +14,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.alert.utils; import org.apache.dolphinscheduler.common.enums.ShowType; +import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.StringUtils; -import org.apache.dolphinscheduler.common.utils.*; - import org.apache.dolphinscheduler.plugin.model.AlertData; + import org.apache.http.HttpEntity; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpGet; @@ -29,11 +30,17 @@ import org.apache.http.entity.StringEntity; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClients; import org.apache.http.util.EntityUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.IOException; -import java.util.*; +import java.util.Collection; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Enterprise WeChat utils @@ -41,25 +48,21 @@ import java.util.*; public class EnterpriseWeChatUtils { public static final Logger logger = LoggerFactory.getLogger(EnterpriseWeChatUtils.class); - + public static final String ENTERPRISE_WE_CHAT_AGENT_ID = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_AGENT_ID); + public static final String ENTERPRISE_WE_CHAT_USERS = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_USERS); private static final String ENTERPRISE_WE_CHAT_CORP_ID = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_CORP_ID); - private static final String ENTERPRISE_WE_CHAT_SECRET = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_SECRET); - private static final String ENTERPRISE_WE_CHAT_TOKEN_URL = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_TOKEN_URL); private static final String ENTERPRISE_WE_CHAT_TOKEN_URL_REPLACE = ENTERPRISE_WE_CHAT_TOKEN_URL == null ? null : ENTERPRISE_WE_CHAT_TOKEN_URL - .replaceAll("\\{corpId\\}", ENTERPRISE_WE_CHAT_CORP_ID) - .replaceAll("\\{secret\\}", ENTERPRISE_WE_CHAT_SECRET); - + .replaceAll("\\{corpId}", ENTERPRISE_WE_CHAT_CORP_ID) + .replaceAll("\\{secret}", ENTERPRISE_WE_CHAT_SECRET); private static final String ENTERPRISE_WE_CHAT_PUSH_URL = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_PUSH_URL); - private static final String ENTERPRISE_WE_CHAT_TEAM_SEND_MSG = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_TEAM_SEND_MSG); - private static final String ENTERPRISE_WE_CHAT_USER_SEND_MSG = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_USER_SEND_MSG); - public static final String ENTERPRISE_WE_CHAT_AGENT_ID = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_AGENT_ID); - - public static final String ENTERPRISE_WE_CHAT_USERS = PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_USERS); + private static final String agentIdRegExp = "\\{agentId}"; + private static final String msgRegExp = "\\{msg}"; + private static final String userRegExp = "\\{toUser}"; /** * get Enterprise WeChat is enable @@ -116,13 +119,13 @@ public class EnterpriseWeChatUtils { * * @param toParty the toParty * @param agentId the agentId - * @param msg the msg + * @param msg the msg * @return Enterprise WeChat send message */ public static String makeTeamSendMsg(String toParty, String agentId, String msg) { - return ENTERPRISE_WE_CHAT_TEAM_SEND_MSG.replaceAll("\\{toParty\\}", toParty) - .replaceAll("\\{agentId\\}", agentId) - .replaceAll("\\{msg\\}", msg); + return ENTERPRISE_WE_CHAT_TEAM_SEND_MSG.replaceAll("\\{toParty}", toParty) + .replaceAll(agentIdRegExp, agentId) + .replaceAll(msgRegExp, msg); } /** @@ -130,56 +133,56 @@ public class EnterpriseWeChatUtils { * * @param toParty the toParty * @param agentId the agentId - * @param msg the msg + * @param msg the msg * @return Enterprise WeChat send message */ public static String makeTeamSendMsg(Collection toParty, String agentId, String msg) { String listParty = FuncUtils.mkString(toParty, "|"); - return ENTERPRISE_WE_CHAT_TEAM_SEND_MSG.replaceAll("\\{toParty\\}", listParty) - .replaceAll("\\{agentId\\}", agentId) - .replaceAll("\\{msg\\}", msg); + return ENTERPRISE_WE_CHAT_TEAM_SEND_MSG.replaceAll("\\{toParty}", listParty) + .replaceAll(agentIdRegExp, agentId) + .replaceAll(msgRegExp, msg); } /** * make team single user message * - * @param toUser the toUser + * @param toUser the toUser * @param agentId the agentId - * @param msg the msg + * @param msg the msg * @return Enterprise WeChat send message */ public static String makeUserSendMsg(String toUser, String agentId, String msg) { - return ENTERPRISE_WE_CHAT_USER_SEND_MSG.replaceAll("\\{toUser\\}", toUser) - .replaceAll("\\{agentId\\}", agentId) - .replaceAll("\\{msg\\}", msg); + return ENTERPRISE_WE_CHAT_USER_SEND_MSG.replaceAll("\\{toUser}", toUser) + .replaceAll(agentIdRegExp, agentId) + .replaceAll(msgRegExp, msg); } /** * make team multi user message * - * @param toUser the toUser + * @param toUser the toUser * @param agentId the agentId - * @param msg the msg + * @param msg the msg * @return Enterprise WeChat send message */ public static String makeUserSendMsg(Collection toUser, String agentId, String msg) { String listUser = FuncUtils.mkString(toUser, "|"); - return ENTERPRISE_WE_CHAT_USER_SEND_MSG.replaceAll("\\{toUser\\}", listUser) - .replaceAll("\\{agentId\\}", agentId) - .replaceAll("\\{msg\\}", msg); + return ENTERPRISE_WE_CHAT_USER_SEND_MSG.replaceAll(userRegExp, listUser) + .replaceAll(agentIdRegExp, agentId) + .replaceAll(msgRegExp, msg); } /** * send Enterprise WeChat * * @param charset the charset - * @param data the data - * @param token the token + * @param data the data + * @param token the token * @return Enterprise WeChat resp, demo: {"errcode":0,"errmsg":"ok","invaliduser":""} * @throws IOException the IOException */ public static String sendEnterpriseWeChat(String charset, String data, String token) throws IOException { - String enterpriseWeChatPushUrlReplace = ENTERPRISE_WE_CHAT_PUSH_URL.replaceAll("\\{token\\}", token); + String enterpriseWeChatPushUrlReplace = ENTERPRISE_WE_CHAT_PUSH_URL.replaceAll("\\{token}", token); CloseableHttpClient httpClient = HttpClients.createDefault(); try { @@ -205,7 +208,7 @@ public class EnterpriseWeChatUtils { /** * convert table to markdown style * - * @param title the title + * @param title the title * @param content the content * @return markdown table content */ @@ -215,13 +218,13 @@ public class EnterpriseWeChatUtils { if (null != mapItemsList) { for (LinkedHashMap mapItems : mapItemsList) { - Set> entries = mapItems.entrySet(); - Iterator> iterator = entries.iterator(); + Set> entries = mapItems.entrySet(); + Iterator> iterator = entries.iterator(); StringBuilder t = new StringBuilder(String.format("`%s`%s", title, Constants.MARKDOWN_ENTER)); while (iterator.hasNext()) { - Map.Entry entry = iterator.next(); + Map.Entry entry = iterator.next(); t.append(Constants.MARKDOWN_QUOTE); t.append(entry.getKey()).append(":").append(entry.getValue()); t.append(Constants.MARKDOWN_ENTER); @@ -235,29 +238,30 @@ public class EnterpriseWeChatUtils { /** * convert text to markdown style * - * @param title the title + * @param title the title * @param content the content * @return markdown text */ public static String markdownText(String title, String content) { if (StringUtils.isNotEmpty(content)) { - List list; - try { - list = JSONUtils.toList(content, String.class); - } catch (Exception e) { - logger.error("json format exception", e); - return null; - } + List mapItemsList = JSONUtils.toList(content, LinkedHashMap.class); + if (null != mapItemsList) { + StringBuilder contents = new StringBuilder(100); + contents.append(String.format("`%s`%n", title)); + for (LinkedHashMap mapItems : mapItemsList) { - StringBuilder contents = new StringBuilder(100); - contents.append(String.format("`%s`%n", title)); - for (String str : list) { - contents.append(Constants.MARKDOWN_QUOTE); - contents.append(str); - contents.append(Constants.MARKDOWN_ENTER); - } + Set> entries = mapItems.entrySet(); + Iterator> iterator = entries.iterator(); + while (iterator.hasNext()) { + Map.Entry entry = iterator.next(); + contents.append(Constants.MARKDOWN_QUOTE); + contents.append(entry.getKey()).append(":").append(entry.getValue()); + contents.append(Constants.MARKDOWN_ENTER); + } - return contents.toString(); + } + return contents.toString(); + } } return null; @@ -278,4 +282,5 @@ public class EnterpriseWeChatUtils { return result; } + } diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/MailUtils.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/MailUtils.java index 6f67462771..888c9dbb26 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/MailUtils.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/MailUtils.java @@ -59,7 +59,7 @@ public class MailUtils { public static final String STARTTLS_ENABLE = PropertyUtils.getString(Constants.MAIL_SMTP_STARTTLS_ENABLE); - public static final String SSL_ENABLE = PropertyUtils.getString(Constants.MAIL_SMTP_SSL_ENABLE); + public static final Boolean SSL_ENABLE = PropertyUtils.getBoolean(Constants.MAIL_SMTP_SSL_ENABLE); public static final String SSL_TRUST = PropertyUtils.getString(Constants.MAIL_SMTP_SSL_TRUST); @@ -213,6 +213,7 @@ public class MailUtils { /** * get session + * * @return the new Session */ private static Session getSession() { @@ -222,8 +223,10 @@ public class MailUtils { props.setProperty(Constants.MAIL_SMTP_AUTH, Constants.STRING_TRUE); props.setProperty(Constants.MAIL_TRANSPORT_PROTOCOL, MAIL_PROTOCOL); props.setProperty(Constants.MAIL_SMTP_STARTTLS_ENABLE, STARTTLS_ENABLE); - props.setProperty(Constants.MAIL_SMTP_SSL_ENABLE, SSL_ENABLE); - props.setProperty(Constants.MAIL_SMTP_SSL_TRUST, SSL_TRUST); + if (SSL_ENABLE) { + props.setProperty(Constants.MAIL_SMTP_SSL_ENABLE, "true"); + props.setProperty(Constants.MAIL_SMTP_SSL_TRUST, SSL_TRUST); + } Authenticator auth = new Authenticator() { @Override @@ -345,5 +348,5 @@ public class MailUtils { retMap.put(Constants.MESSAGE, "Send email to {" + String.join(",", receivers) + "} failed，" + e.toString()); } - + } diff --git a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtilsTest.java b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtilsTest.java index 7b6cdd013b..1a70c5becb 100644 --- a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtilsTest.java +++ b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/EnterpriseWeChatUtilsTest.java @@ -14,36 +14,38 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.alert.utils; import org.apache.dolphinscheduler.common.enums.AlertType; import org.apache.dolphinscheduler.common.enums.ShowType; +import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.dao.entity.Alert; import org.apache.dolphinscheduler.plugin.model.AlertData; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; + import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.Mockito; -import org.mockito.junit.MockitoJUnitRunner; import org.powermock.api.mockito.PowerMockito; import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; -import java.io.IOException; -import java.util.*; -import org.apache.dolphinscheduler.common.utils.*; - /** * Please manually modify the configuration file before testing. * file: alert.properties - * enterprise.wechat.corp.id - * enterprise.wechat.secret - * enterprise.wechat.token.url - * enterprise.wechat.push.url - * enterprise.wechat.send.msg - * enterprise.wechat.agent.id - * enterprise.wechat.users + * enterprise.wechat.corp.id + * enterprise.wechat.secret + * enterprise.wechat.token.url + * enterprise.wechat.push.url + * enterprise.wechat.send.msg + * enterprise.wechat.agent.id + * enterprise.wechat.users */ @PrepareForTest(PropertyUtils.class) @RunWith(PowerMockRunner.class) @@ -52,14 +54,18 @@ public class EnterpriseWeChatUtilsTest { private static final String toParty = "wwc99134b6fc1edb6"; private static final String enterpriseWechatSecret = "Uuv2KFrkdf7SeKOsTDCpsTkpawXBMNRhFy6VKX5FV"; private static final String enterpriseWechatAgentId = "1000004"; - private static final String enterpriseWechatUsers="LiGang,journey"; + private static final String enterpriseWechatUsers = "LiGang,journey"; private static final String msg = "hello world"; - private static final String enterpriseWechatTeamSendMsg = "{\\\"toparty\\\":\\\"{toParty}\\\",\\\"agentid\\\":\\\"{agentId}\\\",\\\"msgtype\\\":\\\"text\\\",\\\"text\\\":{\\\"content\\\":\\\"{msg}\\\"},\\\"safe\\\":\\\"0\\\"}"; - private static final String enterpriseWechatUserSendMsg = "{\\\"touser\\\":\\\"{toUser}\\\",\\\"agentid\\\":\\\"{agentId}\\\",\\\"msgtype\\\":\\\"markdown\\\",\\\"markdown\\\":{\\\"content\\\":\\\"{msg}\\\"}}"; + private static final String enterpriseWechatTeamSendMsg = "{\\\"toparty\\\":\\\"{toParty}\\\",\\\"agentid\\\":\\\"{agentId}\\\"" + + + ",\\\"msgtype\\\":\\\"text\\\",\\\"text\\\":{\\\"content\\\":\\\"{msg}\\\"},\\\"safe\\\":\\\"0\\\"}"; + private static final String enterpriseWechatUserSendMsg = "{\\\"touser\\\":\\\"{toUser}\\\",\\\"agentid\\\":\\\"{agentId}\\\"" + + + ",\\\"msgtype\\\":\\\"markdown\\\",\\\"markdown\\\":{\\\"content\\\":\\\"{msg}\\\"}}"; @Before - public void init(){ + public void init() { PowerMockito.mockStatic(PropertyUtils.class); Mockito.when(PropertyUtils.getBoolean(Constants.ENTERPRISE_WECHAT_ENABLE)).thenReturn(true); Mockito.when(PropertyUtils.getString(Constants.ENTERPRISE_WECHAT_USER_SEND_MSG)).thenReturn(enterpriseWechatUserSendMsg); @@ -67,14 +73,13 @@ public class EnterpriseWeChatUtilsTest { } @Test - public void testIsEnable(){ + public void testIsEnable() { Boolean weChartEnable = EnterpriseWeChatUtils.isEnable(); Assert.assertTrue(weChartEnable); } - @Test - public void testMakeTeamSendMsg1(){ + public void testMakeTeamSendMsg1() { String sendMsg = EnterpriseWeChatUtils.makeTeamSendMsg(toParty, enterpriseWechatSecret, msg); Assert.assertTrue(sendMsg.contains(toParty)); Assert.assertTrue(sendMsg.contains(enterpriseWechatSecret)); @@ -82,9 +87,8 @@ public class EnterpriseWeChatUtilsTest { } - @Test - public void testMakeTeamSendMsg2(){ + public void testMakeTeamSendMsg2() { List parties = new ArrayList<>(); parties.add(toParty); parties.add("test1"); @@ -96,7 +100,7 @@ public class EnterpriseWeChatUtilsTest { } @Test - public void tesMakeUserSendMsg1(){ + public void tesMakeUserSendMsg1() { String sendMsg = EnterpriseWeChatUtils.makeUserSendMsg(enterpriseWechatUsers, enterpriseWechatAgentId, msg); Assert.assertTrue(sendMsg.contains(enterpriseWechatUsers)); @@ -105,7 +109,7 @@ public class EnterpriseWeChatUtilsTest { } @Test - public void tesMakeUserSendMsg2(){ + public void tesMakeUserSendMsg2() { List users = new ArrayList<>(); users.add("user1"); users.add("user2"); @@ -118,7 +122,7 @@ public class EnterpriseWeChatUtilsTest { } @Test - public void testMarkdownByAlertForText(){ + public void testMarkdownByAlertForText() { Alert alertForText = createAlertForText(); AlertData alertData = new AlertData(); alertData.setTitle(alertForText.getTitle()) @@ -129,7 +133,7 @@ public class EnterpriseWeChatUtilsTest { } @Test - public void testMarkdownByAlertForTable(){ + public void testMarkdownByAlertForTable() { Alert alertForText = createAlertForTable(); AlertData alertData = new AlertData(); alertData.setTitle(alertForText.getTitle()) @@ -139,17 +143,26 @@ public class EnterpriseWeChatUtilsTest { Assert.assertNotNull(result); } - private Alert createAlertForText(){ - String content ="[\"id:69\"," + - "\"name:UserBehavior-0--1193959466\"," + - "\"Job name: Start workflow\"," + - "\"State: SUCCESS\"," + - "\"Recovery:NO\"," + - "\"Run time: 1\"," + - "\"Start time: 2018-08-06 10:31:34.0\"," + - "\"End time: 2018-08-06 10:31:49.0\"," + - "\"Host: 192.168.xx.xx\"," + - "\"Notify group :4\"]"; + private Alert createAlertForText() { + String content = "[{\"id\":\"69\"," + + + "\"name\":\"UserBehavior-0--1193959466\"," + + + "\"Job name\":\"Start workflow\"," + + + "\"State\":\"SUCCESS\"," + + + "\"Recovery\":\"NO\"," + + + "\"Run time\":\"1\"," + + + "\"Start time\": \"2018-08-06 10:31:34.0\"," + + + "\"End time\": \"2018-08-06 10:31:49.0\"," + + + "\"Host\": \"192.168.xx.xx\"," + + + "\"Notify group\" :\"4\"}]"; Alert alert = new Alert(); alert.setTitle("Mysql Exception"); @@ -161,18 +174,18 @@ public class EnterpriseWeChatUtilsTest { return alert; } - private String list2String(){ + private String list2String() { LinkedHashMap map1 = new LinkedHashMap<>(); - map1.put("mysql service name","mysql200"); - map1.put("mysql address","192.168.xx.xx"); - map1.put("port","3306"); - map1.put("no index of number","80"); - map1.put("database client connections","190"); + map1.put("mysql service name", "mysql200"); + map1.put("mysql address", "192.168.xx.xx"); + map1.put("port", "3306"); + map1.put("no index of number", "80"); + map1.put("database client connections", "190"); LinkedHashMap map2 = new LinkedHashMap<>(); - map2.put("mysql service name","mysql210"); - map2.put("mysql address","192.168.xx.xx"); + map2.put("mysql service name", "mysql210"); + map2.put("mysql address", "192.168.xx.xx"); map2.put("port", "3306"); map2.put("no index of number", "10"); map2.put("database client connections", "90"); @@ -184,11 +197,11 @@ public class EnterpriseWeChatUtilsTest { return mapjson; } - private Alert createAlertForTable(){ + private Alert createAlertForTable() { Alert alert = new Alert(); alert.setTitle("Mysql Exception"); alert.setShowType(ShowType.TABLE); - String content= list2String(); + String content = list2String(); alert.setContent(content); alert.setAlertType(AlertType.EMAIL); alert.setAlertGroupId(1); @@ -196,77 +209,75 @@ public class EnterpriseWeChatUtilsTest { } - - -// @Test -// public void testSendSingleTeamWeChat() { -// try { -// String token = EnterpriseWeChatUtils.getToken(); -// String msg = EnterpriseWeChatUtils.makeTeamSendMsg(partyId, agentId, "hello world"); -// String resp = EnterpriseWeChatUtils.sendEnterpriseWeChat("utf-8", msg, token); -// -// String errmsg = JSONUtils.parseObject(resp).getString("errmsg"); -// Assert.assertEquals("ok",errmsg); -// } catch (IOException e) { -// e.printStackTrace(); -// } -// } -// -// @Test -// public void testSendMultiTeamWeChat() { -// -// try { -// String token = EnterpriseWeChatUtils.getToken(); -// String msg = EnterpriseWeChatUtils.makeTeamSendMsg(listPartyId, agentId, "hello world"); -// String resp = EnterpriseWeChatUtils.sendEnterpriseWeChat("utf-8", msg, token); -// -// String errmsg = JSONUtils.parseObject(resp).getString("errmsg"); -// Assert.assertEquals("ok",errmsg); -// } catch (IOException e) { -// e.printStackTrace(); -// } -// } -// -// @Test -// public void testSendSingleUserWeChat() { -// try { -// String token = EnterpriseWeChatUtils.getToken(); -// String msg = EnterpriseWeChatUtils.makeUserSendMsg(listUserId.stream().findFirst().get(), agentId, "your meeting room has been booked and will be synced to the 'mailbox' later \n" + -// ">**matter details** \n" + -// ">matter：meeting
" + -// ">organizer：@miglioguan \n" + -// ">participant：@miglioguan、@kunliu、@jamdeezhou、@kanexiong、@kisonwang \n" + -// "> \n" + -// ">meeting room：Guangzhou TIT 1st Floor 301 \n" + -// ">date：May 18, 2018 \n" + -// ">time：9:00-11:00 am \n" + -// "> \n" + -// ">please attend the meeting on time\n" + -// "> \n" + -// ">to modify the meeting information, please click: [Modify Meeting Information](https://work.weixin.qq.com)\""); -// -// String resp = EnterpriseWeChatUtils.sendEnterpriseWeChat("utf-8", msg, token); -// -// String errmsg = JSONUtils.parseObject(resp).getString("errmsg"); -// Assert.assertEquals("ok",errmsg); -// } catch (IOException e) { -// e.printStackTrace(); -// } -// } -// -// @Test -// public void testSendMultiUserWeChat() { -// try { -// String token = EnterpriseWeChatUtils.getToken(); -// -// String msg = EnterpriseWeChatUtils.makeUserSendMsg(listUserId, agentId, "hello world"); -// String resp = EnterpriseWeChatUtils.sendEnterpriseWeChat("utf-8", msg, token); -// -// String errmsg = JSONUtils.parseObject(resp).getString("errmsg"); -// Assert.assertEquals("ok",errmsg); -// } catch (IOException e) { -// e.printStackTrace(); -// } -// } + // @Test + // public void testSendSingleTeamWeChat() { + // try { + // String token = EnterpriseWeChatUtils.getToken(); + // String msg = EnterpriseWeChatUtils.makeTeamSendMsg(partyId, agentId, "hello world"); + // String resp = EnterpriseWeChatUtils.sendEnterpriseWeChat("utf-8", msg, token); + // + // String errmsg = JSONUtils.parseObject(resp).getString("errmsg"); + // Assert.assertEquals("ok",errmsg); + // } catch (IOException e) { + // e.printStackTrace(); + // } + // } + // + // @Test + // public void testSendMultiTeamWeChat() { + // + // try { + // String token = EnterpriseWeChatUtils.getToken(); + // String msg = EnterpriseWeChatUtils.makeTeamSendMsg(listPartyId, agentId, "hello world"); + // String resp = EnterpriseWeChatUtils.sendEnterpriseWeChat("utf-8", msg, token); + // + // String errmsg = JSONUtils.parseObject(resp).getString("errmsg"); + // Assert.assertEquals("ok",errmsg); + // } catch (IOException e) { + // e.printStackTrace(); + // } + // } + // + // @Test + // public void testSendSingleUserWeChat() { + // try { + // String token = EnterpriseWeChatUtils.getToken(); + // String msg = EnterpriseWeChatUtils.makeUserSendMsg(listUserId.stream().findFirst().get(), agentId, "your meeting room has been booked and will be synced to the 'mailbox' later \n" + + // ">**matter details** \n" + + // ">matter：meeting
" + + // ">organizer：@miglioguan \n" + + // ">participant：@miglioguan、@kunliu、@jamdeezhou、@kanexiong、@kisonwang \n" + + // "> \n" + + // ">meeting room：Guangzhou TIT 1st Floor 301 \n" + + // ">date：May 18, 2018 \n" + + // ">time：9:00-11:00 am \n" + + // "> \n" + + // ">please attend the meeting on time\n" + + // "> \n" + + // ">to modify the meeting information, please click: [Modify Meeting Information](https://work.weixin.qq.com)\""); + // + // String resp = EnterpriseWeChatUtils.sendEnterpriseWeChat("utf-8", msg, token); + // + // String errmsg = JSONUtils.parseObject(resp).getString("errmsg"); + // Assert.assertEquals("ok",errmsg); + // } catch (IOException e) { + // e.printStackTrace(); + // } + // } + // + // @Test + // public void testSendMultiUserWeChat() { + // try { + // String token = EnterpriseWeChatUtils.getToken(); + // + // String msg = EnterpriseWeChatUtils.makeUserSendMsg(listUserId, agentId, "hello world"); + // String resp = EnterpriseWeChatUtils.sendEnterpriseWeChat("utf-8", msg, token); + // + // String errmsg = JSONUtils.parseObject(resp).getString("errmsg"); + // Assert.assertEquals("ok",errmsg); + // } catch (IOException e) { + // e.printStackTrace(); + // } + // } } diff --git a/dolphinscheduler-api/pom.xml b/dolphinscheduler-api/pom.xml index 035551e669..76dd8980b7 100644 --- a/dolphinscheduler-api/pom.xml +++ b/dolphinscheduler-api/pom.xml @@ -152,6 +152,10 @@ javax.servlet servlet-api + + org.apache.curator + curator-client + @@ -244,4 +248,4 @@ - \ No newline at end of file + diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/configuration/ServiceModelToSwagger2MapperImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/configuration/ServiceModelToSwagger2MapperImpl.java index dac88925a2..d10cd78652 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/configuration/ServiceModelToSwagger2MapperImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/configuration/ServiceModelToSwagger2MapperImpl.java @@ -220,11 +220,7 @@ public class ServiceModelToSwagger2MapperImpl extends ServiceModelToSwagger2Mapp if (resourceListing == null) { return null; } - ApiInfo info = resourceListing.getInfo(); - if (info == null) { - return null; - } - return info; + return resourceListing.getInfo(); } protected List tagSetToTagList(Set set) { diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/AccessTokenController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/AccessTokenController.java index 8731b264e9..2457177cdf 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/AccessTokenController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/AccessTokenController.java @@ -17,6 +17,12 @@ package org.apache.dolphinscheduler.api.controller; +import static org.apache.dolphinscheduler.api.enums.Status.CREATE_ACCESS_TOKEN_ERROR; +import static org.apache.dolphinscheduler.api.enums.Status.DELETE_ACCESS_TOKEN_ERROR; +import static org.apache.dolphinscheduler.api.enums.Status.GENERATE_TOKEN_ERROR; +import static org.apache.dolphinscheduler.api.enums.Status.QUERY_ACCESSTOKEN_LIST_PAGING_ERROR; +import static org.apache.dolphinscheduler.api.enums.Status.UPDATE_ACCESS_TOKEN_ERROR; + import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.exceptions.ApiException; import org.apache.dolphinscheduler.api.service.AccessTokenService; @@ -24,21 +30,27 @@ import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.utils.ParameterUtils; import org.apache.dolphinscheduler.dao.entity.User; -import io.swagger.annotations.Api; -import io.swagger.annotations.ApiImplicitParam; -import io.swagger.annotations.ApiImplicitParams; -import io.swagger.annotations.ApiOperation; + +import java.util.Map; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.HttpStatus; -import org.springframework.web.bind.annotation.*; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestAttribute; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.ResponseStatus; +import org.springframework.web.bind.annotation.RestController; + +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiImplicitParam; +import io.swagger.annotations.ApiImplicitParams; +import io.swagger.annotations.ApiOperation; import springfox.documentation.annotations.ApiIgnore; -import java.util.Map; - -import static org.apache.dolphinscheduler.api.enums.Status.*; - /** * access token controller */ diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataSourceController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataSourceController.java index a34d61a26d..4bdaa365ee 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataSourceController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataSourceController.java @@ -100,7 +100,7 @@ public class DataSourceController extends BaseController { @RequestParam(value = "other") String other) { logger.info("login user {} create datasource name: {}, note: {}, type: {}, host: {}, port: {}, database : {}, principal: {}, userName : {}, connectType: {}, other: {}", loginUser.getUserName(), name, note, type, host, port, database, principal, userName, connectType, other); - String parameter = dataSourceService.buildParameter(name, note, type, host, port, database, principal, userName, password, connectType, other); + String parameter = dataSourceService.buildParameter(type, host, port, database, principal, userName, password, connectType, other); Map result = dataSourceService.createDataSource(loginUser, name, note, type, parameter); return returnDataList(result); } @@ -155,7 +155,7 @@ public class DataSourceController extends BaseController { @RequestParam(value = "other") String other) { logger.info("login user {} updateProcessInstance datasource name: {}, note: {}, type: {}, connectType: {}, other: {}", loginUser.getUserName(), name, note, type, connectType, other); - String parameter = dataSourceService.buildParameter(name, note, type, host, port, database, principal, userName, password, connectType, other); + String parameter = dataSourceService.buildParameter(type, host, port, database, principal, userName, password, connectType, other); Map dataSource = dataSourceService.updateDataSource(id, loginUser, name, note, type, parameter); return returnDataList(dataSource); } @@ -280,7 +280,7 @@ public class DataSourceController extends BaseController { @RequestParam(value = "other") String other) { logger.info("login user {}, connect datasource: {}, note: {}, type: {}, connectType: {}, other: {}", loginUser.getUserName(), name, note, type, connectType, other); - String parameter = dataSourceService.buildParameter(name, note, type, host, port, database, principal, userName, password, connectType, other); + String parameter = dataSourceService.buildParameter(type, host, port, database, principal, userName, password, connectType, other); Boolean isConnection = dataSourceService.checkConnection(type, parameter); Result result = new Result(); @@ -310,7 +310,7 @@ public class DataSourceController extends BaseController { @RequestParam("id") int id) { logger.info("connection test, login user:{}, id:{}", loginUser.getUserName(), id); - Boolean isConnection = dataSourceService.connectionTest(loginUser, id); + Boolean isConnection = dataSourceService.connectionTest(id); Result result = new Result(); if (isConnection) { @@ -361,7 +361,7 @@ public class DataSourceController extends BaseController { logger.info("login user {}, verfiy datasource name: {}", loginUser.getUserName(), name); - return dataSourceService.verifyDataSourceName(loginUser, name); + return dataSourceService.verifyDataSourceName(name); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/LoggerController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/LoggerController.java index a5b8176a48..7d612b8b1d 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/LoggerController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/LoggerController.java @@ -17,25 +17,34 @@ package org.apache.dolphinscheduler.api.controller; +import static org.apache.dolphinscheduler.api.enums.Status.DOWNLOAD_TASK_INSTANCE_LOG_FILE_ERROR; +import static org.apache.dolphinscheduler.api.enums.Status.QUERY_TASK_INSTANCE_LOG_ERROR; + import org.apache.dolphinscheduler.api.exceptions.ApiException; import org.apache.dolphinscheduler.api.service.LoggerService; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.dao.entity.User; -import io.swagger.annotations.Api; -import io.swagger.annotations.ApiImplicitParam; -import io.swagger.annotations.ApiImplicitParams; -import io.swagger.annotations.ApiOperation; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.HttpHeaders; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; -import org.springframework.web.bind.annotation.*; -import springfox.documentation.annotations.ApiIgnore; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestAttribute; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.ResponseBody; +import org.springframework.web.bind.annotation.ResponseStatus; +import org.springframework.web.bind.annotation.RestController; -import static org.apache.dolphinscheduler.api.enums.Status.*; +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiImplicitParam; +import io.swagger.annotations.ApiImplicitParams; +import io.swagger.annotations.ApiOperation; +import springfox.documentation.annotations.ApiIgnore; /** @@ -70,7 +79,7 @@ public class LoggerController extends BaseController { @GetMapping(value = "/detail") @ResponseStatus(HttpStatus.OK) @ApiException(QUERY_TASK_INSTANCE_LOG_ERROR) - public Result queryLog(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser, + public Result queryLog(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @RequestParam(value = "taskInstanceId") int taskInstanceId, @RequestParam(value = "skipLineNum") int skipNum, @RequestParam(value = "limit") int limit) { diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessDefinitionController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessDefinitionController.java index 6b539d01b1..48cb53c5b2 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessDefinitionController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessDefinitionController.java @@ -14,32 +14,65 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.api.controller; -import com.fasterxml.jackson.core.JsonProcessingException; +import static org.apache.dolphinscheduler.api.enums.Status.BATCH_COPY_PROCESS_DEFINITION_ERROR; +import static org.apache.dolphinscheduler.api.enums.Status.BATCH_DELETE_PROCESS_DEFINE_BY_IDS_ERROR; +import static org.apache.dolphinscheduler.api.enums.Status.BATCH_MOVE_PROCESS_DEFINITION_ERROR; +import static org.apache.dolphinscheduler.api.enums.Status.CREATE_PROCESS_DEFINITION; +import static org.apache.dolphinscheduler.api.enums.Status.DELETE_PROCESS_DEFINE_BY_ID_ERROR; +import static org.apache.dolphinscheduler.api.enums.Status.DELETE_PROCESS_DEFINITION_VERSION_ERROR; +import static org.apache.dolphinscheduler.api.enums.Status.ENCAPSULATION_TREEVIEW_STRUCTURE_ERROR; +import static org.apache.dolphinscheduler.api.enums.Status.GET_TASKS_LIST_BY_PROCESS_DEFINITION_ID_ERROR; +import static org.apache.dolphinscheduler.api.enums.Status.QUERY_DATAIL_OF_PROCESS_DEFINITION_ERROR; +import static org.apache.dolphinscheduler.api.enums.Status.QUERY_PROCESS_DEFINITION_LIST; +import static org.apache.dolphinscheduler.api.enums.Status.QUERY_PROCESS_DEFINITION_LIST_PAGING_ERROR; +import static org.apache.dolphinscheduler.api.enums.Status.QUERY_PROCESS_DEFINITION_VERSIONS_ERROR; +import static org.apache.dolphinscheduler.api.enums.Status.RELEASE_PROCESS_DEFINITION_ERROR; +import static org.apache.dolphinscheduler.api.enums.Status.SWITCH_PROCESS_DEFINITION_VERSION_ERROR; +import static org.apache.dolphinscheduler.api.enums.Status.UPDATE_PROCESS_DEFINITION_ERROR; +import static org.apache.dolphinscheduler.api.enums.Status.VERIFY_PROCESS_DEFINITION_NAME_UNIQUE_ERROR; + import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.exceptions.ApiException; import org.apache.dolphinscheduler.api.service.ProcessDefinitionService; +import org.apache.dolphinscheduler.api.service.ProcessDefinitionVersionService; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.utils.ParameterUtils; import org.apache.dolphinscheduler.common.utils.StringUtils; import org.apache.dolphinscheduler.dao.entity.User; -import io.swagger.annotations.*; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.http.HttpStatus; -import org.springframework.web.bind.annotation.*; -import springfox.documentation.annotations.ApiIgnore; -import javax.servlet.http.HttpServletResponse; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; -import static org.apache.dolphinscheduler.api.enums.Status.*; +import javax.servlet.http.HttpServletResponse; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.HttpStatus; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestAttribute; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.ResponseBody; +import org.springframework.web.bind.annotation.ResponseStatus; +import org.springframework.web.bind.annotation.RestController; + +import com.fasterxml.jackson.core.JsonProcessingException; + +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiImplicitParam; +import io.swagger.annotations.ApiImplicitParams; +import io.swagger.annotations.ApiOperation; +import io.swagger.annotations.ApiParam; +import springfox.documentation.annotations.ApiIgnore; /** @@ -55,16 +88,19 @@ public class ProcessDefinitionController extends BaseController { @Autowired private ProcessDefinitionService processDefinitionService; + @Autowired + private ProcessDefinitionVersionService processDefinitionVersionService; + /** * create process definition * - * @param loginUser login user + * @param loginUser login user * @param projectName project name - * @param name process definition name - * @param json process definition json + * @param name process definition name + * @param json process definition json * @param description description - * @param locations locations for nodes - * @param connects connects for nodes + * @param locations locations for nodes + * @param connects connects for nodes * @return create result code */ @ApiOperation(value = "save", notes = "CREATE_PROCESS_DEFINITION_NOTES") @@ -86,8 +122,8 @@ public class ProcessDefinitionController extends BaseController { @RequestParam(value = "connects", required = true) String connects, @RequestParam(value = "description", required = false) String description) throws JsonProcessingException { - logger.info("login user {}, create process definition, project name: {}, process definition name: {}, " + - "process_definition_json: {}, desc: {} locations:{}, connects:{}", + logger.info("login user {}, create process definition, project name: {}, process definition name: {}, " + + "process_definition_json: {}, desc: {} locations:{}, connects:{}", loginUser.getUserName(), projectName, name, json, description, locations, connects); Map result = processDefinitionService.createProcessDefinition(loginUser, projectName, name, json, description, locations, connects); @@ -95,35 +131,73 @@ public class ProcessDefinitionController extends BaseController { } /** - * copy process definition + * copy process definition * - * @param loginUser login user + * @param loginUser login user * @param projectName project name - * @param processId process definition id + * @param processDefinitionIds process definition ids + * @param targetProjectId target project id * @return copy result code */ - @ApiOperation(value = "copyProcessDefinition", notes= "COPY_PROCESS_DEFINITION_NOTES") + @ApiOperation(value = "copyProcessDefinition", notes = "COPY_PROCESS_DEFINITION_NOTES") @ApiImplicitParams({ - @ApiImplicitParam(name = "processId", value = "PROCESS_DEFINITION_ID", required = true, dataType = "Int", example = "100") + @ApiImplicitParam(name = "processDefinitionIds", value = "PROCESS_DEFINITION_IDS", required = true, dataType = "String", example = "3,4"), + @ApiImplicitParam(name = "targetProjectId", value = "TARGET_PROJECT_ID", required = true, type = "Integer") }) @PostMapping(value = "/copy") @ResponseStatus(HttpStatus.OK) - @ApiException(COPY_PROCESS_DEFINITION_ERROR) + @ApiException(BATCH_COPY_PROCESS_DEFINITION_ERROR) public Result copyProcessDefinition(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @ApiParam(name = "projectName", value = "PROJECT_NAME", required = true) @PathVariable String projectName, - @RequestParam(value = "processId", required = true) int processId) throws JsonProcessingException { - logger.info("copy process definition, login user:{}, project name:{}, process definition id:{}", - loginUser.getUserName(), projectName, processId); - Map result = processDefinitionService.copyProcessDefinition(loginUser, projectName, processId); - return returnDataList(result); + @RequestParam(value = "processDefinitionIds", required = true) String processDefinitionIds, + @RequestParam(value = "targetProjectId", required = true) int targetProjectId) { + logger.info("batch copy process definition, login user:{}, project name:{}, process definition ids:{}，target project id:{}", + StringUtils.replaceNRTtoUnderline(loginUser.getUserName()), + StringUtils.replaceNRTtoUnderline(projectName), + StringUtils.replaceNRTtoUnderline(processDefinitionIds), + StringUtils.replaceNRTtoUnderline(String.valueOf(targetProjectId))); + + return returnDataList( + processDefinitionService.batchCopyProcessDefinition(loginUser, projectName, processDefinitionIds, targetProjectId)); + } + + /** + * move process definition + * + * @param loginUser login user + * @param projectName project name + * @param processDefinitionIds process definition ids + * @param targetProjectId target project id + * @return move result code + */ + @ApiOperation(value = "moveProcessDefinition", notes = "MOVE_PROCESS_DEFINITION_NOTES") + @ApiImplicitParams({ + @ApiImplicitParam(name = "processDefinitionIds", value = "PROCESS_DEFINITION_IDS", required = true, dataType = "String", example = "3,4"), + @ApiImplicitParam(name = "targetProjectId", value = "TARGET_PROJECT_ID", required = true, type = "Integer") + }) + @PostMapping(value = "/move") + @ResponseStatus(HttpStatus.OK) + @ApiException(BATCH_MOVE_PROCESS_DEFINITION_ERROR) + public Result moveProcessDefinition(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser, + @ApiParam(name = "projectName", value = "PROJECT_NAME", required = true) @PathVariable String projectName, + @RequestParam(value = "processDefinitionIds", required = true) String processDefinitionIds, + @RequestParam(value = "targetProjectId", required = true) int targetProjectId) { + logger.info("batch move process definition, login user:{}, project name:{}, process definition ids:{}，target project id:{}", + StringUtils.replaceNRTtoUnderline(loginUser.getUserName()), + StringUtils.replaceNRTtoUnderline(projectName), + StringUtils.replaceNRTtoUnderline(processDefinitionIds), + StringUtils.replaceNRTtoUnderline(String.valueOf(targetProjectId))); + + return returnDataList( + processDefinitionService.batchMoveProcessDefinition(loginUser, projectName, processDefinitionIds, targetProjectId)); } /** * verify process definition name unique * - * @param loginUser login user + * @param loginUser login user * @param projectName project name - * @param name name + * @param name name * @return true if process definition name not exists, otherwise false */ @ApiOperation(value = "verify-name", notes = "VERIFY_PROCESS_DEFINITION_NAME_NOTES") @@ -134,8 +208,8 @@ public class ProcessDefinitionController extends BaseController { @ResponseStatus(HttpStatus.OK) @ApiException(VERIFY_PROCESS_DEFINITION_NAME_UNIQUE_ERROR) public Result verifyProcessDefinitionName(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser, - @ApiParam(name = "projectName", value = "PROJECT_NAME", required = true) @PathVariable String projectName, - @RequestParam(value = "name", required = true) String name) { + @ApiParam(name = "projectName", value = "PROJECT_NAME", required = true) @PathVariable String projectName, + @RequestParam(value = "name", required = true) String name) { logger.info("verify process definition name unique, user:{}, project name:{}, process definition name:{}", loginUser.getUserName(), projectName, name); Map result = processDefinitionService.verifyProcessDefinitionName(loginUser, projectName, name); @@ -145,18 +219,18 @@ public class ProcessDefinitionController extends BaseController { /** * update process definition * - * @param loginUser login user - * @param projectName project name - * @param name process definition name - * @param id process definition id + * @param loginUser login user + * @param projectName project name + * @param name process definition name + * @param id process definition id * @param processDefinitionJson process definition json - * @param description description - * @param locations locations for nodes - * @param connects connects for nodes + * @param description description + * @param locations locations for nodes + * @param connects connects for nodes * @return update result code */ - @ApiOperation(value = "updateProcessDefinition", notes= "UPDATE_PROCESS_DEFINITION_NOTES") + @ApiOperation(value = "updateProcessDefinition", notes = "UPDATE_PROCESS_DEFINITION_NOTES") @ApiImplicitParams({ @ApiImplicitParam(name = "name", value = "PROCESS_DEFINITION_NAME", required = true, type = "String"), @ApiImplicitParam(name = "id", value = "PROCESS_DEFINITION_ID", required = true, dataType = "Int", example = "100"), @@ -169,33 +243,115 @@ public class ProcessDefinitionController extends BaseController { @ResponseStatus(HttpStatus.OK) @ApiException(UPDATE_PROCESS_DEFINITION_ERROR) public Result updateProcessDefinition(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser, - @ApiParam(name = "projectName", value = "PROJECT_NAME", required = true) @PathVariable String projectName, - @RequestParam(value = "name", required = true) String name, - @RequestParam(value = "id", required = true) int id, - @RequestParam(value = "processDefinitionJson", required = true) String processDefinitionJson, - @RequestParam(value = "locations", required = false) String locations, - @RequestParam(value = "connects", required = false) String connects, - @RequestParam(value = "description", required = false) String description) { + @ApiParam(name = "projectName", value = "PROJECT_NAME", required = true) @PathVariable String projectName, + @RequestParam(value = "name", required = true) String name, + @RequestParam(value = "id", required = true) int id, + @RequestParam(value = "processDefinitionJson", required = true) String processDefinitionJson, + @RequestParam(value = "locations", required = false) String locations, + @RequestParam(value = "connects", required = false) String connects, + @RequestParam(value = "description", required = false) String description) { - logger.info("login user {}, update process define, project name: {}, process define name: {}, " + - "process_definition_json: {}, desc: {}, locations:{}, connects:{}", + logger.info("login user {}, update process define, project name: {}, process define name: {}, " + + "process_definition_json: {}, desc: {}, locations:{}, connects:{}", loginUser.getUserName(), projectName, name, processDefinitionJson, description, locations, connects); Map result = processDefinitionService.updateProcessDefinition(loginUser, projectName, id, name, processDefinitionJson, description, locations, connects); return returnDataList(result); } + /** + * query process definition version paging list info + * + * @param loginUser login user info + * @param projectName the process definition project name + * @param pageNo the process definition version list current page number + * @param pageSize the process definition version list page size + * @param processDefinitionId the process definition id + * @return the process definition version list + */ + @ApiOperation(value = "queryProcessDefinitionVersions", notes = "QUERY_PROCESS_DEFINITION_VERSIONS_NOTES") + @ApiImplicitParams({ + @ApiImplicitParam(name = "pageNo", value = "PAGE_NO", required = true, dataType = "Int", example = "100"), + @ApiImplicitParam(name = "pageSize", value = "PAGE_SIZE", required = true, dataType = "Int", example = "100"), + @ApiImplicitParam(name = "processDefinitionId", value = "PROCESS_DEFINITION_ID", required = true, dataType = "Int", example = "100") + }) + @GetMapping(value = "/versions") + @ResponseStatus(HttpStatus.OK) + @ApiException(QUERY_PROCESS_DEFINITION_VERSIONS_ERROR) + public Result queryProcessDefinitionVersions(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser, + @ApiParam(name = "projectName", value = "PROJECT_NAME", required = true) @PathVariable String projectName, + @RequestParam(value = "pageNo") int pageNo, + @RequestParam(value = "pageSize") int pageSize, + @RequestParam(value = "processDefinitionId") int processDefinitionId) { + + Map result = processDefinitionVersionService.queryProcessDefinitionVersions(loginUser + , projectName, pageNo, pageSize, processDefinitionId); + return returnDataList(result); + } + + /** + * switch certain process definition version + * + * @param loginUser login user info + * @param projectName the process definition project name + * @param processDefinitionId the process definition id + * @param version the version user want to switch + * @return switch version result code + */ + @ApiOperation(value = "switchProcessDefinitionVersion", notes = "SWITCH_PROCESS_DEFINITION_VERSION_NOTES") + @ApiImplicitParams({ + @ApiImplicitParam(name = "processDefinitionId", value = "PROCESS_DEFINITION_ID", required = true, dataType = "Int", example = "100"), + @ApiImplicitParam(name = "version", value = "VERSION", required = true, dataType = "Long", example = "100") + }) + @GetMapping(value = "/version/switch") + @ResponseStatus(HttpStatus.OK) + @ApiException(SWITCH_PROCESS_DEFINITION_VERSION_ERROR) + public Result switchProcessDefinitionVersion(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser, + @ApiParam(name = "projectName", value = "PROJECT_NAME", required = true) @PathVariable String projectName, + @RequestParam(value = "processDefinitionId") int processDefinitionId, + @RequestParam(value = "version") long version) { + + Map result = processDefinitionService.switchProcessDefinitionVersion(loginUser, projectName + , processDefinitionId, version); + return returnDataList(result); + } + + /** + * delete the certain process definition version by version and process definition id + * + * @param loginUser login user info + * @param projectName the process definition project name + * @param processDefinitionId process definition id + * @param version the process definition version user want to delete + * @return delete version result code + */ + @ApiOperation(value = "deleteProcessDefinitionVersion", notes = "DELETE_PROCESS_DEFINITION_VERSION_NOTES") + @ApiImplicitParams({ + @ApiImplicitParam(name = "processDefinitionId", value = "PROCESS_DEFINITION_ID", required = true, dataType = "Int", example = "100"), + @ApiImplicitParam(name = "version", value = "VERSION", required = true, dataType = "Long", example = "100") + }) + @GetMapping(value = "/version/delete") + @ResponseStatus(HttpStatus.OK) + @ApiException(DELETE_PROCESS_DEFINITION_VERSION_ERROR) + public Result deleteProcessDefinitionVersion(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser, + @ApiParam(name = "projectName", value = "PROJECT_NAME", required = true) @PathVariable String projectName, + @RequestParam(value = "processDefinitionId") int processDefinitionId, + @RequestParam(value = "version") long version) { + + Map result = processDefinitionVersionService.deleteByProcessDefinitionIdAndVersion(loginUser, projectName, processDefinitionId, version); + return returnDataList(result); + } + /** * release process definition * - * @param loginUser login user - * @param projectName project name - * @param processId process definition id + * @param loginUser login user + * @param projectName project name + * @param processId process definition id * @param releaseState release state * @return release result code */ - - @ApiOperation(value = "releaseProcessDefinition", notes= "RELEASE_PROCESS_DEFINITION_NOTES") + @ApiOperation(value = "releaseProcessDefinition", notes = "RELEASE_PROCESS_DEFINITION_NOTES") @ApiImplicitParams({ @ApiImplicitParam(name = "name", value = "PROCESS_DEFINITION_NAME", required = true, type = "String"), @ApiImplicitParam(name = "processId", value = "PROCESS_DEFINITION_ID", required = true, dataType = "Int", example = "100"), @@ -205,9 +361,9 @@ public class ProcessDefinitionController extends BaseController { @ResponseStatus(HttpStatus.OK) @ApiException(RELEASE_PROCESS_DEFINITION_ERROR) public Result releaseProcessDefinition(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser, - @ApiParam(name = "projectName", value = "PROJECT_NAME", required = true) @PathVariable String projectName, - @RequestParam(value = "processId", required = true) int processId, - @RequestParam(value = "releaseState", required = true) int releaseState) { + @ApiParam(name = "projectName", value = "PROJECT_NAME", required = true) @PathVariable String projectName, + @RequestParam(value = "processId", required = true) int processId, + @RequestParam(value = "releaseState", required = true) int releaseState) { logger.info("login user {}, release process definition, project name: {}, release state: {}", loginUser.getUserName(), projectName, releaseState); @@ -218,12 +374,12 @@ public class ProcessDefinitionController extends BaseController { /** * query datail of process definition * - * @param loginUser login user + * @param loginUser login user * @param projectName project name - * @param processId process definition id + * @param processId process definition id * @return process definition detail */ - @ApiOperation(value = "queryProcessDefinitionById", notes= "QUERY_PROCESS_DEFINITION_BY_ID_NOTES") + @ApiOperation(value = "queryProcessDefinitionById", notes = "QUERY_PROCESS_DEFINITION_BY_ID_NOTES") @ApiImplicitParams({ @ApiImplicitParam(name = "processId", value = "PROCESS_DEFINITION_ID", required = true, dataType = "Int", example = "100") }) @@ -231,8 +387,8 @@ public class ProcessDefinitionController extends BaseController { @ResponseStatus(HttpStatus.OK) @ApiException(QUERY_DATAIL_OF_PROCESS_DEFINITION_ERROR) public Result queryProcessDefinitionById(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser, - @ApiParam(name = "projectName", value = "PROJECT_NAME", required = true) @PathVariable String projectName, - @RequestParam("processId") Integer processId + @ApiParam(name = "projectName", value = "PROJECT_NAME", required = true) @PathVariable String projectName, + @RequestParam("processId") Integer processId ) { logger.info("query detail of process definition, login user:{}, project name:{}, process definition id:{}", loginUser.getUserName(), projectName, processId); @@ -243,7 +399,7 @@ public class ProcessDefinitionController extends BaseController { /** * query Process definition list * - * @param loginUser login user + * @param loginUser login user * @param projectName project name * @return process definition list */ @@ -252,7 +408,7 @@ public class ProcessDefinitionController extends BaseController { @ResponseStatus(HttpStatus.OK) @ApiException(QUERY_PROCESS_DEFINITION_LIST) public Result queryProcessDefinitionList(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser, - @ApiParam(name = "projectName", value = "PROJECT_NAME", required = true) @PathVariable String projectName + @ApiParam(name = "projectName", value = "PROJECT_NAME", required = true) @PathVariable String projectName ) { logger.info("query process definition list, login user:{}, project name:{}", loginUser.getUserName(), projectName); @@ -263,15 +419,15 @@ public class ProcessDefinitionController extends BaseController { /** * query process definition list paging * - * @param loginUser login user + * @param loginUser login user * @param projectName project name - * @param searchVal search value - * @param pageNo page number - * @param pageSize page size - * @param userId user id + * @param searchVal search value + * @param pageNo page number + * @param pageSize page size + * @param userId user id * @return process definition page */ - @ApiOperation(value = "queryProcessDefinitionListPaging", notes= "QUERY_PROCESS_DEFINITION_LIST_PAGING_NOTES") + @ApiOperation(value = "queryProcessDefinitionListPaging", notes = "QUERY_PROCESS_DEFINITION_LIST_PAGING_NOTES") @ApiImplicitParams({ @ApiImplicitParam(name = "pageNo", value = "PAGE_NO", required = true, dataType = "Int", example = "100"), @ApiImplicitParam(name = "searchVal", value = "SEARCH_VAL", required = false, type = "String"), @@ -300,10 +456,10 @@ public class ProcessDefinitionController extends BaseController { /** * encapsulation treeview structure * - * @param loginUser login user + * @param loginUser login user * @param projectName project name - * @param id process definition id - * @param limit limit + * @param id process definition id + * @param limit limit * @return tree view json data */ @ApiOperation(value = "viewTree", notes = "VIEW_TREE_NOTES") @@ -325,8 +481,8 @@ public class ProcessDefinitionController extends BaseController { /** * get tasks list by process definition id * - * @param loginUser login user - * @param projectName project name + * @param loginUser login user + * @param projectName project name * @param processDefinitionId process definition id * @return task list */ @@ -350,8 +506,8 @@ public class ProcessDefinitionController extends BaseController { /** * get tasks list by process definition id * - * @param loginUser login user - * @param projectName project name + * @param loginUser login user + * @param projectName project name * @param processDefinitionIdList process definition id list * @return node list data */ @@ -365,7 +521,7 @@ public class ProcessDefinitionController extends BaseController { public Result getNodeListByDefinitionIdList( @ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @ApiParam(name = "projectName", value = "PROJECT_NAME", required = true) @PathVariable String projectName, - @RequestParam("processDefinitionIdList") String processDefinitionIdList) throws Exception { + @RequestParam("processDefinitionIdList") String processDefinitionIdList) { logger.info("query task node name list by definitionId list, login user:{}, project name:{}, id list: {}", loginUser.getUserName(), projectName, processDefinitionIdList); @@ -376,8 +532,8 @@ public class ProcessDefinitionController extends BaseController { /** * delete process definition by id * - * @param loginUser login user - * @param projectName project name + * @param loginUser login user + * @param projectName project name * @param processDefinitionId process definition id * @return delete result code */ @@ -401,8 +557,8 @@ public class ProcessDefinitionController extends BaseController { /** * batch delete process definition by ids * - * @param loginUser login user - * @param projectName project name + * @param loginUser login user + * @param projectName project name * @param processDefinitionIds process definition id list * @return delete result code */ @@ -420,7 +576,7 @@ public class ProcessDefinitionController extends BaseController { logger.info("delete process definition by ids, login user:{}, project name:{}, process definition ids:{}", loginUser.getUserName(), projectName, processDefinitionIds); - Map result = new HashMap<>(5); + Map result = new HashMap<>(); List deleteFailedIdList = new ArrayList<>(); if (StringUtils.isNotEmpty(processDefinitionIds)) { String[] processDefinitionIdArray = processDefinitionIds.split(","); @@ -451,13 +607,13 @@ public class ProcessDefinitionController extends BaseController { /** * batch export process definition by ids * - * @param loginUser login user - * @param projectName project name + * @param loginUser login user + * @param projectName project name * @param processDefinitionIds process definition ids - * @param response response + * @param response response */ - @ApiOperation(value = "batchExportProcessDefinitionByIds", notes= "BATCH_EXPORT_PROCESS_DEFINITION_BY_IDS_NOTES") + @ApiOperation(value = "batchExportProcessDefinitionByIds", notes = "BATCH_EXPORT_PROCESS_DEFINITION_BY_IDS_NOTES") @ApiImplicitParams({ @ApiImplicitParam(name = "processDefinitionIds", value = "PROCESS_DEFINITION_ID", required = true, dataType = "String") }) @@ -488,7 +644,7 @@ public class ProcessDefinitionController extends BaseController { @ResponseStatus(HttpStatus.OK) @ApiException(QUERY_PROCESS_DEFINITION_LIST) public Result queryProcessDefinitionAllByProjectId(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser, - @RequestParam("projectId") Integer projectId) { + @RequestParam("projectId") Integer projectId) { logger.info("query process definition list, login user:{}, project id:{}", loginUser.getUserName(), projectId); Map result = processDefinitionService.queryProcessDefinitionAllByProjectId(projectId); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessInstanceController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessInstanceController.java index 7e9473d81c..1f1ec1ed7b 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessInstanceController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessInstanceController.java @@ -370,7 +370,7 @@ public class ProcessInstanceController extends BaseController { logger.info("delete process instance by ids, login user:{}, project name:{}, process instance ids :{}", loginUser.getUserName(), projectName, processInstanceIds); // task queue - Map result = new HashMap<>(5); + Map result = new HashMap<>(); List deleteFailedIdList = new ArrayList<>(); if (StringUtils.isNotEmpty(processInstanceIds)) { String[] processInstanceIdArray = processInstanceIds.split(","); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProjectController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProjectController.java index cc9e0f657f..dac97bca9d 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProjectController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProjectController.java @@ -23,6 +23,7 @@ import org.apache.dolphinscheduler.api.service.ProjectService; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.utils.ParameterUtils; +import org.apache.dolphinscheduler.common.utils.StringUtils; import org.apache.dolphinscheduler.dao.entity.User; import io.swagger.annotations.Api; import io.swagger.annotations.ApiImplicitParam; @@ -226,6 +227,25 @@ public class ProjectController extends BaseController { return returnDataList(result); } + /** + * query user created project + * + * @param loginUser login user + * @return projects which the user create + */ + @ApiOperation(value = "queryProjectCreatedByUser", notes = "QUERY_USER_CREATED_PROJECT_NOTES") + + @GetMapping(value = "/login-user-created-project") + @ResponseStatus(HttpStatus.OK) + @ApiException(QUERY_USER_CREATED_PROJECT_ERROR) + public Result queryProjectCreatedByUser(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser) { + logger.info("login user {}, query authorized project by user id: {}.", + StringUtils.replaceNRTtoUnderline(loginUser.getUserName()), + StringUtils.replaceNRTtoUnderline(String.valueOf(loginUser.getId()))); + Map result = projectService.queryProjectCreatedByUser(loginUser); + return returnDataList(result); + } + /** * import process definition * diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TenantController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TenantController.java index a603ac050c..2676a774e7 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TenantController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TenantController.java @@ -14,8 +14,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.api.controller; +import static org.apache.dolphinscheduler.api.enums.Status.CREATE_TENANT_ERROR; +import static org.apache.dolphinscheduler.api.enums.Status.DELETE_TENANT_BY_ID_ERROR; +import static org.apache.dolphinscheduler.api.enums.Status.QUERY_TENANT_LIST_ERROR; +import static org.apache.dolphinscheduler.api.enums.Status.QUERY_TENANT_LIST_PAGING_ERROR; +import static org.apache.dolphinscheduler.api.enums.Status.UPDATE_TENANT_ERROR; +import static org.apache.dolphinscheduler.api.enums.Status.VERIFY_TENANT_CODE_ERROR; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.exceptions.ApiException; @@ -24,21 +31,27 @@ import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.utils.ParameterUtils; import org.apache.dolphinscheduler.dao.entity.User; -import io.swagger.annotations.Api; -import io.swagger.annotations.ApiImplicitParam; -import io.swagger.annotations.ApiImplicitParams; -import io.swagger.annotations.ApiOperation; + +import java.util.Map; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.HttpStatus; -import org.springframework.web.bind.annotation.*; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestAttribute; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.ResponseStatus; +import org.springframework.web.bind.annotation.RestController; + +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiImplicitParam; +import io.swagger.annotations.ApiImplicitParams; +import io.swagger.annotations.ApiOperation; import springfox.documentation.annotations.ApiIgnore; -import java.util.Map; - -import static org.apache.dolphinscheduler.api.enums.Status.*; - /** * tenant controller @@ -57,10 +70,10 @@ public class TenantController extends BaseController { /** * create tenant * - * @param loginUser login user - * @param tenantCode tenant code - * @param tenantName tenant name - * @param queueId queue id + * @param loginUser login user + * @param tenantCode tenant code + * @param tenantName tenant name + * @param queueId queue id * @param description description * @return create result code */ @@ -92,8 +105,8 @@ public class TenantController extends BaseController { * * @param loginUser login user * @param searchVal search value - * @param pageNo page number - * @param pageSize page size + * @param pageNo page number + * @param pageSize page size * @return tenant list page */ @ApiOperation(value = "queryTenantlistPaging", notes = "QUERY_TENANT_LIST_PAGING_NOTES") @@ -141,11 +154,11 @@ public class TenantController extends BaseController { /** * udpate tenant * - * @param loginUser login user - * @param id tennat id - * @param tenantCode tennat code - * @param tenantName tennat name - * @param queueId queue id + * @param loginUser login user + * @param id tennat id + * @param tenantCode tennat code + * @param tenantName tennat name + * @param queueId queue id * @param description description * @return update result code */ @@ -177,7 +190,7 @@ public class TenantController extends BaseController { * delete tenant by id * * @param loginUser login user - * @param id tenant id + * @param id tenant id * @return delete result code */ @ApiOperation(value = "deleteTenantById", notes = "DELETE_TENANT_NOTES") @@ -195,11 +208,10 @@ public class TenantController extends BaseController { return returnDataList(result); } - /** * verify tenant code * - * @param loginUser login user + * @param loginUser login user * @param tenantCode tenant code * @return true if tenant code can user, otherwise return false */ @@ -211,12 +223,10 @@ public class TenantController extends BaseController { @ResponseStatus(HttpStatus.OK) @ApiException(VERIFY_TENANT_CODE_ERROR) public Result verifyTenantCode(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser, - @RequestParam(value = "tenantCode") String tenantCode - ) { + @RequestParam(value = "tenantCode") String tenantCode) { logger.info("login user {}, verfiy tenant code: {}", loginUser.getUserName(), tenantCode); return tenantService.verifyTenantCode(tenantCode); } - } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/UsersController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/UsersController.java index 39b9b06337..8d6f9fc820 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/UsersController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/UsersController.java @@ -35,10 +35,12 @@ import org.springframework.http.HttpStatus; import org.springframework.web.bind.annotation.*; import springfox.documentation.annotations.ApiIgnore; -import java.util.Map; - import static org.apache.dolphinscheduler.api.enums.Status.*; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + /** * user controller @@ -432,14 +434,54 @@ public class UsersController extends BaseController { @RequestParam(value = "userPassword") String userPassword, @RequestParam(value = "repeatPassword") String repeatPassword, @RequestParam(value = "email") String email) throws Exception { - userName = userName.replaceAll("[\n|\r|\t]", ""); - userPassword = userPassword.replaceAll("[\n|\r|\t]", ""); - repeatPassword = repeatPassword.replaceAll("[\n|\r|\t]", ""); - email = email.replaceAll("[\n|\r|\t]", ""); + userName = ParameterUtils.handleEscapes(userName); + userPassword = ParameterUtils.handleEscapes(userPassword); + repeatPassword = ParameterUtils.handleEscapes(repeatPassword); + email = ParameterUtils.handleEscapes(email); logger.info("user self-register, userName: {}, userPassword {}, repeatPassword {}, eamil {}", - userName, userPassword, repeatPassword, email); + userName, Constants.PASSWORD_DEFAULT, Constants.PASSWORD_DEFAULT, email); Map result = usersService.registerUser(userName, userPassword, repeatPassword, email); return returnDataList(result); } + /** + * user activate + * + * @param userName user name + */ + @ApiOperation(value="activateUser",notes = "ACTIVATE_USER_NOTES") + @ApiImplicitParams({ + @ApiImplicitParam(name = "userName", value = "USER_NAME", type = "String"), + }) + @PostMapping("/activate") + @ResponseStatus(HttpStatus.OK) + @ApiException(UPDATE_USER_ERROR) + public Result activateUser(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser, + @RequestParam(value = "userName") String userName) { + userName = ParameterUtils.handleEscapes(userName); + logger.info("login user {}, activate user, userName: {}", + loginUser.getUserName(), userName); + Map result = usersService.activateUser(loginUser, userName); + return returnDataList(result); + } + + /** + * user batch activate + * + * @param userNames user names + */ + @ApiOperation(value = "batchActivateUser",notes = "BATCH_ACTIVATE_USER_NOTES") + @ApiImplicitParams({ + @ApiImplicitParam(name = "userNames", value = "USER_NAMES", type = "String"), + }) + @PostMapping("/batch/activate") + @ResponseStatus(HttpStatus.OK) + @ApiException(UPDATE_USER_ERROR) + public Result batchActivateUser(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser, + @RequestBody List userNames) { + List formatUserNames = userNames.stream().map(ParameterUtils::handleEscapes).collect(Collectors.toList()); + logger.info(" activate userNames: {}", formatUserNames); + Map result = usersService.batchActivateUser(loginUser, formatUserNames); + return returnDataList(result); + } } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/TaskCountDto.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/TaskCountDto.java index fa7588f2ed..35aaaf34dd 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/TaskCountDto.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/TaskCountDto.java @@ -42,9 +42,10 @@ public class TaskCountDto { countTaskDtos(taskInstanceStateCounts); } - private void countTaskDtos(List taskInstanceStateCounts){ + private void countTaskDtos(List taskInstanceStateCounts) { int submittedSuccess = 0; - int runningExeution = 0; + int runningExecution = 0; + int delayExecution = 0; int readyPause = 0; int pause = 0; int readyStop = 0; @@ -55,15 +56,18 @@ public class TaskCountDto { int kill = 0; int waittingThread = 0; - for(ExecuteStatusCount taskInstanceStateCount : taskInstanceStateCounts){ + for (ExecuteStatusCount taskInstanceStateCount : taskInstanceStateCounts) { ExecutionStatus status = taskInstanceStateCount.getExecutionStatus(); totalCount += taskInstanceStateCount.getCount(); - switch (status){ + switch (status) { case SUBMITTED_SUCCESS: submittedSuccess += taskInstanceStateCount.getCount(); break; case RUNNING_EXECUTION: - runningExeution += taskInstanceStateCount.getCount(); + runningExecution += taskInstanceStateCount.getCount(); + break; + case DELAY_EXECUTION: + delayExecution += taskInstanceStateCount.getCount(); break; case READY_PAUSE: readyPause += taskInstanceStateCount.getCount(); @@ -93,13 +97,14 @@ public class TaskCountDto { waittingThread += taskInstanceStateCount.getCount(); break; - default: - break; + default: + break; } } this.taskCountDtos = new ArrayList<>(); this.taskCountDtos.add(new TaskStateCount(ExecutionStatus.SUBMITTED_SUCCESS, submittedSuccess)); - this.taskCountDtos.add(new TaskStateCount(ExecutionStatus.RUNNING_EXECUTION, runningExeution)); + this.taskCountDtos.add(new TaskStateCount(ExecutionStatus.RUNNING_EXECUTION, runningExecution)); + this.taskCountDtos.add(new TaskStateCount(ExecutionStatus.DELAY_EXECUTION, delayExecution)); this.taskCountDtos.add(new TaskStateCount(ExecutionStatus.READY_PAUSE, readyPause)); this.taskCountDtos.add(new TaskStateCount(ExecutionStatus.PAUSE, pause)); this.taskCountDtos.add(new TaskStateCount(ExecutionStatus.READY_STOP, readyStop)); @@ -111,8 +116,7 @@ public class TaskCountDto { this.taskCountDtos.add(new TaskStateCount(ExecutionStatus.WAITTING_THREAD, waittingThread)); } - - public List getTaskCountDtos(){ + public List getTaskCountDtos() { return taskCountDtos; } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/filter/ResourceFilter.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/filter/ResourceFilter.java index c918a160af..bf5a49597c 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/filter/ResourceFilter.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/filter/ResourceFilter.java @@ -57,11 +57,10 @@ public class ResourceFilter implements IFilter { * @return file filtered by suffix */ public Set fileFilter(){ - Set resources = resourceList.stream().filter(t -> { + return resourceList.stream().filter(t -> { String alias = t.getAlias(); return alias.endsWith(suffix); }).collect(Collectors.toSet()); - return resources; } /** diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java index 8e90b4cb08..43c03f09d8 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java @@ -14,14 +14,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.dolphinscheduler.api.enums; -import org.springframework.context.i18n.LocaleContextHolder; +package org.apache.dolphinscheduler.api.enums; import java.util.Locale; +import org.springframework.context.i18n.LocaleContextHolder; + /** - * status enum + * status enum */ public enum Status { @@ -32,15 +33,15 @@ public enum Status { REQUEST_PARAMS_NOT_VALID_ERROR(10001, "request parameter {0} is not valid", "请求参数[{0}]无效"), TASK_TIMEOUT_PARAMS_ERROR(10002, "task timeout parameter is not valid", "任务超时参数无效"), USER_NAME_EXIST(10003, "user name already exists", "用户名已存在"), - USER_NAME_NULL(10004,"user name is null", "用户名不能为空"), + USER_NAME_NULL(10004, "user name is null", "用户名不能为空"), HDFS_OPERATION_ERROR(10006, "hdfs operation error", "hdfs操作错误"), TASK_INSTANCE_NOT_FOUND(10008, "task instance not found", "任务实例不存在"), TENANT_NAME_EXIST(10009, "tenant code {0} already exists", "租户编码[{0}]已存在"), USER_NOT_EXIST(10010, "user {0} not exists", "用户[{0}]不存在"), ALERT_GROUP_NOT_EXIST(10011, "alarm group not found", "告警组不存在"), ALERT_GROUP_EXIST(10012, "alarm group already exists", "告警组名称已存在"), - USER_NAME_PASSWD_ERROR(10013,"user name or password error", "用户名或密码错误"), - LOGIN_SESSION_FAILED(10014,"create session failed!", "创建session失败"), + USER_NAME_PASSWD_ERROR(10013, "user name or password error", "用户名或密码错误"), + LOGIN_SESSION_FAILED(10014, "create session failed!", "创建session失败"), DATASOURCE_EXIST(10015, "data source name already exists", "数据源名称已存在"), DATASOURCE_CONNECT_FAILED(10016, "data source connection failed", "建立数据源连接失败"), TENANT_NOT_EXIST(10017, "tenant not exists", "租户不存在"), @@ -53,105 +54,105 @@ public enum Status { SCHEDULE_CRON_CHECK_FAILED(10024, "scheduler crontab expression validation failure: {0}", "调度配置定时表达式验证失败: {0}"), MASTER_NOT_EXISTS(10025, "master does not exist", "无可用master节点"), SCHEDULE_STATUS_UNKNOWN(10026, "unknown status: {0}", "未知状态: {0}"), - CREATE_ALERT_GROUP_ERROR(10027,"create alert group error", "创建告警组错误"), - QUERY_ALL_ALERTGROUP_ERROR(10028,"query all alertgroup error", "查询告警组错误"), - LIST_PAGING_ALERT_GROUP_ERROR(10029,"list paging alert group error", "分页查询告警组错误"), - UPDATE_ALERT_GROUP_ERROR(10030,"update alert group error", "更新告警组错误"), - DELETE_ALERT_GROUP_ERROR(10031,"delete alert group error", "删除告警组错误"), - ALERT_GROUP_GRANT_USER_ERROR(10032,"alert group grant user error", "告警组授权用户错误"), - CREATE_DATASOURCE_ERROR(10033,"create datasource error", "创建数据源错误"), - UPDATE_DATASOURCE_ERROR(10034,"update datasource error", "更新数据源错误"), - QUERY_DATASOURCE_ERROR(10035,"query datasource error", "查询数据源错误"), - CONNECT_DATASOURCE_FAILURE(10036,"connect datasource failure", "建立数据源连接失败"), - CONNECTION_TEST_FAILURE(10037,"connection test failure", "测试数据源连接失败"), - DELETE_DATA_SOURCE_FAILURE(10038,"delete data source failure", "删除数据源失败"), - VERIFY_DATASOURCE_NAME_FAILURE(10039,"verify datasource name failure", "验证数据源名称失败"), - UNAUTHORIZED_DATASOURCE(10040,"unauthorized datasource", "未经授权的数据源"), - AUTHORIZED_DATA_SOURCE(10041,"authorized data source", "授权数据源失败"), - LOGIN_SUCCESS(10042,"login success", "登录成功"), - USER_LOGIN_FAILURE(10043,"user login failure", "用户登录失败"), - LIST_WORKERS_ERROR(10044,"list workers error", "查询worker列表错误"), - LIST_MASTERS_ERROR(10045,"list masters error", "查询master列表错误"), - UPDATE_PROJECT_ERROR(10046,"update project error", "更新项目信息错误"), - QUERY_PROJECT_DETAILS_BY_ID_ERROR(10047,"query project details by id error", "查询项目详细信息错误"), - CREATE_PROJECT_ERROR(10048,"create project error", "创建项目错误"), - LOGIN_USER_QUERY_PROJECT_LIST_PAGING_ERROR(10049,"login user query project list paging error", "分页查询项目列表错误"), - DELETE_PROJECT_ERROR(10050,"delete project error", "删除项目错误"), - QUERY_UNAUTHORIZED_PROJECT_ERROR(10051,"query unauthorized project error", "查询未授权项目错误"), - QUERY_AUTHORIZED_PROJECT(10052,"query authorized project", "查询授权项目错误"), - QUERY_QUEUE_LIST_ERROR(10053,"query queue list error", "查询队列列表错误"), - CREATE_RESOURCE_ERROR(10054,"create resource error", "创建资源错误"), - UPDATE_RESOURCE_ERROR(10055,"update resource error", "更新资源错误"), - QUERY_RESOURCES_LIST_ERROR(10056,"query resources list error", "查询资源列表错误"), - QUERY_RESOURCES_LIST_PAGING(10057,"query resources list paging", "分页查询资源列表错误"), - DELETE_RESOURCE_ERROR(10058,"delete resource error", "删除资源错误"), - VERIFY_RESOURCE_BY_NAME_AND_TYPE_ERROR(10059,"verify resource by name and type error", "资源名称或类型验证错误"), - VIEW_RESOURCE_FILE_ON_LINE_ERROR(10060,"view resource file online error", "查看资源文件错误"), - CREATE_RESOURCE_FILE_ON_LINE_ERROR(10061,"create resource file online error", "创建资源文件错误"), - RESOURCE_FILE_IS_EMPTY(10062,"resource file is empty", "资源文件内容不能为空"), - EDIT_RESOURCE_FILE_ON_LINE_ERROR(10063,"edit resource file online error", "更新资源文件错误"), - DOWNLOAD_RESOURCE_FILE_ERROR(10064,"download resource file error", "下载资源文件错误"), - CREATE_UDF_FUNCTION_ERROR(10065 ,"create udf function error", "创建UDF函数错误"), - VIEW_UDF_FUNCTION_ERROR( 10066,"view udf function error", "查询UDF函数错误"), - UPDATE_UDF_FUNCTION_ERROR(10067,"update udf function error", "更新UDF函数错误"), - QUERY_UDF_FUNCTION_LIST_PAGING_ERROR( 10068,"query udf function list paging error", "分页查询UDF函数列表错误"), - QUERY_DATASOURCE_BY_TYPE_ERROR( 10069,"query datasource by type error", "查询数据源信息错误"), - VERIFY_UDF_FUNCTION_NAME_ERROR( 10070,"verify udf function name error", "UDF函数名称验证错误"), - DELETE_UDF_FUNCTION_ERROR( 10071,"delete udf function error", "删除UDF函数错误"), - AUTHORIZED_FILE_RESOURCE_ERROR( 10072,"authorized file resource error", "授权资源文件错误"), - AUTHORIZE_RESOURCE_TREE( 10073,"authorize resource tree display error","授权资源目录树错误"), - UNAUTHORIZED_UDF_FUNCTION_ERROR( 10074,"unauthorized udf function error", "查询未授权UDF函数错误"), - AUTHORIZED_UDF_FUNCTION_ERROR(10075,"authorized udf function error", "授权UDF函数错误"), - CREATE_SCHEDULE_ERROR(10076,"create schedule error", "创建调度配置错误"), - UPDATE_SCHEDULE_ERROR(10077,"update schedule error", "更新调度配置错误"), - PUBLISH_SCHEDULE_ONLINE_ERROR(10078,"publish schedule online error", "上线调度配置错误"), - OFFLINE_SCHEDULE_ERROR(10079,"offline schedule error", "下线调度配置错误"), - QUERY_SCHEDULE_LIST_PAGING_ERROR(10080,"query schedule list paging error", "分页查询调度配置列表错误"), - QUERY_SCHEDULE_LIST_ERROR(10081,"query schedule list error", "查询调度配置列表错误"), - QUERY_TASK_LIST_PAGING_ERROR(10082,"query task list paging error", "分页查询任务列表错误"), - QUERY_TASK_RECORD_LIST_PAGING_ERROR(10083,"query task record list paging error", "分页查询任务记录错误"), - CREATE_TENANT_ERROR(10084,"create tenant error", "创建租户错误"), - QUERY_TENANT_LIST_PAGING_ERROR(10085,"query tenant list paging error", "分页查询租户列表错误"), - QUERY_TENANT_LIST_ERROR(10086,"query tenant list error", "查询租户列表错误"), - UPDATE_TENANT_ERROR(10087,"update tenant error", "更新租户错误"), - DELETE_TENANT_BY_ID_ERROR(10088,"delete tenant by id error", "删除租户错误"), - VERIFY_TENANT_CODE_ERROR(10089,"verify tenant code error", "租户编码验证错误"), - CREATE_USER_ERROR(10090,"create user error", "创建用户错误"), - QUERY_USER_LIST_PAGING_ERROR(10091,"query user list paging error", "分页查询用户列表错误"), - UPDATE_USER_ERROR(10092,"update user error", "更新用户错误"), - DELETE_USER_BY_ID_ERROR(10093,"delete user by id error", "删除用户错误"), - GRANT_PROJECT_ERROR(10094,"grant project error", "授权项目错误"), - GRANT_RESOURCE_ERROR(10095,"grant resource error", "授权资源错误"), - GRANT_UDF_FUNCTION_ERROR(10096,"grant udf function error", "授权UDF函数错误"), - GRANT_DATASOURCE_ERROR(10097,"grant datasource error", "授权数据源错误"), - GET_USER_INFO_ERROR(10098,"get user info error", "获取用户信息错误"), - USER_LIST_ERROR(10099,"user list error", "查询用户列表错误"), - VERIFY_USERNAME_ERROR(10100,"verify username error", "用户名验证错误"), - UNAUTHORIZED_USER_ERROR(10101,"unauthorized user error", "查询未授权用户错误"), - AUTHORIZED_USER_ERROR(10102,"authorized user error", "查询授权用户错误"), - QUERY_TASK_INSTANCE_LOG_ERROR(10103,"view task instance log error", "查询任务实例日志错误"), - DOWNLOAD_TASK_INSTANCE_LOG_FILE_ERROR(10104,"download task instance log file error", "下载任务日志文件错误"), - CREATE_PROCESS_DEFINITION(10105,"create process definition", "创建工作流错误"), - VERIFY_PROCESS_DEFINITION_NAME_UNIQUE_ERROR(10106,"verify process definition name unique error", "工作流名称已存在"), - UPDATE_PROCESS_DEFINITION_ERROR(10107,"update process definition error", "更新工作流定义错误"), - RELEASE_PROCESS_DEFINITION_ERROR(10108,"release process definition error", "上线工作流错误"), - QUERY_DATAIL_OF_PROCESS_DEFINITION_ERROR(10109,"query datail of process definition error", "查询工作流详细信息错误"), - QUERY_PROCESS_DEFINITION_LIST(10110,"query process definition list", "查询工作流列表错误"), - ENCAPSULATION_TREEVIEW_STRUCTURE_ERROR(10111,"encapsulation treeview structure error", "查询工作流树形图数据错误"), - GET_TASKS_LIST_BY_PROCESS_DEFINITION_ID_ERROR(10112,"get tasks list by process definition id error", "查询工作流定义节点信息错误"), - QUERY_PROCESS_INSTANCE_LIST_PAGING_ERROR(10113,"query process instance list paging error", "分页查询工作流实例列表错误"), - QUERY_TASK_LIST_BY_PROCESS_INSTANCE_ID_ERROR(10114,"query task list by process instance id error", "查询任务实例列表错误"), - UPDATE_PROCESS_INSTANCE_ERROR(10115,"update process instance error", "更新工作流实例错误"), - QUERY_PROCESS_INSTANCE_BY_ID_ERROR(10116,"query process instance by id error", "查询工作流实例错误"), - DELETE_PROCESS_INSTANCE_BY_ID_ERROR(10117,"delete process instance by id error", "删除工作流实例错误"), - QUERY_SUB_PROCESS_INSTANCE_DETAIL_INFO_BY_TASK_ID_ERROR(10118,"query sub process instance detail info by task id error", "查询子流程任务实例错误"), - QUERY_PARENT_PROCESS_INSTANCE_DETAIL_INFO_BY_SUB_PROCESS_INSTANCE_ID_ERROR(10119,"query parent process instance detail info by sub process instance id error", "查询子流程该工作流实例错误"), - QUERY_PROCESS_INSTANCE_ALL_VARIABLES_ERROR(10120,"query process instance all variables error", "查询工作流自定义变量信息错误"), - ENCAPSULATION_PROCESS_INSTANCE_GANTT_STRUCTURE_ERROR(10121,"encapsulation process instance gantt structure error", "查询工作流实例甘特图数据错误"), - QUERY_PROCESS_DEFINITION_LIST_PAGING_ERROR(10122,"query process definition list paging error", "分页查询工作流定义列表错误"), - SIGN_OUT_ERROR(10123,"sign out error", "退出错误"), - TENANT_CODE_HAS_ALREADY_EXISTS(10124,"tenant code has already exists", "租户编码已存在"), - IP_IS_EMPTY(10125,"ip is empty", "IP地址不能为空"), + CREATE_ALERT_GROUP_ERROR(10027, "create alert group error", "创建告警组错误"), + QUERY_ALL_ALERTGROUP_ERROR(10028, "query all alertgroup error", "查询告警组错误"), + LIST_PAGING_ALERT_GROUP_ERROR(10029, "list paging alert group error", "分页查询告警组错误"), + UPDATE_ALERT_GROUP_ERROR(10030, "update alert group error", "更新告警组错误"), + DELETE_ALERT_GROUP_ERROR(10031, "delete alert group error", "删除告警组错误"), + ALERT_GROUP_GRANT_USER_ERROR(10032, "alert group grant user error", "告警组授权用户错误"), + CREATE_DATASOURCE_ERROR(10033, "create datasource error", "创建数据源错误"), + UPDATE_DATASOURCE_ERROR(10034, "update datasource error", "更新数据源错误"), + QUERY_DATASOURCE_ERROR(10035, "query datasource error", "查询数据源错误"), + CONNECT_DATASOURCE_FAILURE(10036, "connect datasource failure", "建立数据源连接失败"), + CONNECTION_TEST_FAILURE(10037, "connection test failure", "测试数据源连接失败"), + DELETE_DATA_SOURCE_FAILURE(10038, "delete data source failure", "删除数据源失败"), + VERIFY_DATASOURCE_NAME_FAILURE(10039, "verify datasource name failure", "验证数据源名称失败"), + UNAUTHORIZED_DATASOURCE(10040, "unauthorized datasource", "未经授权的数据源"), + AUTHORIZED_DATA_SOURCE(10041, "authorized data source", "授权数据源失败"), + LOGIN_SUCCESS(10042, "login success", "登录成功"), + USER_LOGIN_FAILURE(10043, "user login failure", "用户登录失败"), + LIST_WORKERS_ERROR(10044, "list workers error", "查询worker列表错误"), + LIST_MASTERS_ERROR(10045, "list masters error", "查询master列表错误"), + UPDATE_PROJECT_ERROR(10046, "update project error", "更新项目信息错误"), + QUERY_PROJECT_DETAILS_BY_ID_ERROR(10047, "query project details by id error", "查询项目详细信息错误"), + CREATE_PROJECT_ERROR(10048, "create project error", "创建项目错误"), + LOGIN_USER_QUERY_PROJECT_LIST_PAGING_ERROR(10049, "login user query project list paging error", "分页查询项目列表错误"), + DELETE_PROJECT_ERROR(10050, "delete project error", "删除项目错误"), + QUERY_UNAUTHORIZED_PROJECT_ERROR(10051, "query unauthorized project error", "查询未授权项目错误"), + QUERY_AUTHORIZED_PROJECT(10052, "query authorized project", "查询授权项目错误"), + QUERY_QUEUE_LIST_ERROR(10053, "query queue list error", "查询队列列表错误"), + CREATE_RESOURCE_ERROR(10054, "create resource error", "创建资源错误"), + UPDATE_RESOURCE_ERROR(10055, "update resource error", "更新资源错误"), + QUERY_RESOURCES_LIST_ERROR(10056, "query resources list error", "查询资源列表错误"), + QUERY_RESOURCES_LIST_PAGING(10057, "query resources list paging", "分页查询资源列表错误"), + DELETE_RESOURCE_ERROR(10058, "delete resource error", "删除资源错误"), + VERIFY_RESOURCE_BY_NAME_AND_TYPE_ERROR(10059, "verify resource by name and type error", "资源名称或类型验证错误"), + VIEW_RESOURCE_FILE_ON_LINE_ERROR(10060, "view resource file online error", "查看资源文件错误"), + CREATE_RESOURCE_FILE_ON_LINE_ERROR(10061, "create resource file online error", "创建资源文件错误"), + RESOURCE_FILE_IS_EMPTY(10062, "resource file is empty", "资源文件内容不能为空"), + EDIT_RESOURCE_FILE_ON_LINE_ERROR(10063, "edit resource file online error", "更新资源文件错误"), + DOWNLOAD_RESOURCE_FILE_ERROR(10064, "download resource file error", "下载资源文件错误"), + CREATE_UDF_FUNCTION_ERROR(10065, "create udf function error", "创建UDF函数错误"), + VIEW_UDF_FUNCTION_ERROR(10066, "view udf function error", "查询UDF函数错误"), + UPDATE_UDF_FUNCTION_ERROR(10067, "update udf function error", "更新UDF函数错误"), + QUERY_UDF_FUNCTION_LIST_PAGING_ERROR(10068, "query udf function list paging error", "分页查询UDF函数列表错误"), + QUERY_DATASOURCE_BY_TYPE_ERROR(10069, "query datasource by type error", "查询数据源信息错误"), + VERIFY_UDF_FUNCTION_NAME_ERROR(10070, "verify udf function name error", "UDF函数名称验证错误"), + DELETE_UDF_FUNCTION_ERROR(10071, "delete udf function error", "删除UDF函数错误"), + AUTHORIZED_FILE_RESOURCE_ERROR(10072, "authorized file resource error", "授权资源文件错误"), + AUTHORIZE_RESOURCE_TREE(10073, "authorize resource tree display error", "授权资源目录树错误"), + UNAUTHORIZED_UDF_FUNCTION_ERROR(10074, "unauthorized udf function error", "查询未授权UDF函数错误"), + AUTHORIZED_UDF_FUNCTION_ERROR(10075, "authorized udf function error", "授权UDF函数错误"), + CREATE_SCHEDULE_ERROR(10076, "create schedule error", "创建调度配置错误"), + UPDATE_SCHEDULE_ERROR(10077, "update schedule error", "更新调度配置错误"), + PUBLISH_SCHEDULE_ONLINE_ERROR(10078, "publish schedule online error", "上线调度配置错误"), + OFFLINE_SCHEDULE_ERROR(10079, "offline schedule error", "下线调度配置错误"), + QUERY_SCHEDULE_LIST_PAGING_ERROR(10080, "query schedule list paging error", "分页查询调度配置列表错误"), + QUERY_SCHEDULE_LIST_ERROR(10081, "query schedule list error", "查询调度配置列表错误"), + QUERY_TASK_LIST_PAGING_ERROR(10082, "query task list paging error", "分页查询任务列表错误"), + QUERY_TASK_RECORD_LIST_PAGING_ERROR(10083, "query task record list paging error", "分页查询任务记录错误"), + CREATE_TENANT_ERROR(10084, "create tenant error", "创建租户错误"), + QUERY_TENANT_LIST_PAGING_ERROR(10085, "query tenant list paging error", "分页查询租户列表错误"), + QUERY_TENANT_LIST_ERROR(10086, "query tenant list error", "查询租户列表错误"), + UPDATE_TENANT_ERROR(10087, "update tenant error", "更新租户错误"), + DELETE_TENANT_BY_ID_ERROR(10088, "delete tenant by id error", "删除租户错误"), + VERIFY_TENANT_CODE_ERROR(10089, "verify tenant code error", "租户编码验证错误"), + CREATE_USER_ERROR(10090, "create user error", "创建用户错误"), + QUERY_USER_LIST_PAGING_ERROR(10091, "query user list paging error", "分页查询用户列表错误"), + UPDATE_USER_ERROR(10092, "update user error", "更新用户错误"), + DELETE_USER_BY_ID_ERROR(10093, "delete user by id error", "删除用户错误"), + GRANT_PROJECT_ERROR(10094, "grant project error", "授权项目错误"), + GRANT_RESOURCE_ERROR(10095, "grant resource error", "授权资源错误"), + GRANT_UDF_FUNCTION_ERROR(10096, "grant udf function error", "授权UDF函数错误"), + GRANT_DATASOURCE_ERROR(10097, "grant datasource error", "授权数据源错误"), + GET_USER_INFO_ERROR(10098, "get user info error", "获取用户信息错误"), + USER_LIST_ERROR(10099, "user list error", "查询用户列表错误"), + VERIFY_USERNAME_ERROR(10100, "verify username error", "用户名验证错误"), + UNAUTHORIZED_USER_ERROR(10101, "unauthorized user error", "查询未授权用户错误"), + AUTHORIZED_USER_ERROR(10102, "authorized user error", "查询授权用户错误"), + QUERY_TASK_INSTANCE_LOG_ERROR(10103, "view task instance log error", "查询任务实例日志错误"), + DOWNLOAD_TASK_INSTANCE_LOG_FILE_ERROR(10104, "download task instance log file error", "下载任务日志文件错误"), + CREATE_PROCESS_DEFINITION(10105, "create process definition", "创建工作流错误"), + VERIFY_PROCESS_DEFINITION_NAME_UNIQUE_ERROR(10106, "verify process definition name unique error", "工作流名称已存在"), + UPDATE_PROCESS_DEFINITION_ERROR(10107, "update process definition error", "更新工作流定义错误"), + RELEASE_PROCESS_DEFINITION_ERROR(10108, "release process definition error", "上线工作流错误"), + QUERY_DATAIL_OF_PROCESS_DEFINITION_ERROR(10109, "query datail of process definition error", "查询工作流详细信息错误"), + QUERY_PROCESS_DEFINITION_LIST(10110, "query process definition list", "查询工作流列表错误"), + ENCAPSULATION_TREEVIEW_STRUCTURE_ERROR(10111, "encapsulation treeview structure error", "查询工作流树形图数据错误"), + GET_TASKS_LIST_BY_PROCESS_DEFINITION_ID_ERROR(10112, "get tasks list by process definition id error", "查询工作流定义节点信息错误"), + QUERY_PROCESS_INSTANCE_LIST_PAGING_ERROR(10113, "query process instance list paging error", "分页查询工作流实例列表错误"), + QUERY_TASK_LIST_BY_PROCESS_INSTANCE_ID_ERROR(10114, "query task list by process instance id error", "查询任务实例列表错误"), + UPDATE_PROCESS_INSTANCE_ERROR(10115, "update process instance error", "更新工作流实例错误"), + QUERY_PROCESS_INSTANCE_BY_ID_ERROR(10116, "query process instance by id error", "查询工作流实例错误"), + DELETE_PROCESS_INSTANCE_BY_ID_ERROR(10117, "delete process instance by id error", "删除工作流实例错误"), + QUERY_SUB_PROCESS_INSTANCE_DETAIL_INFO_BY_TASK_ID_ERROR(10118, "query sub process instance detail info by task id error", "查询子流程任务实例错误"), + QUERY_PARENT_PROCESS_INSTANCE_DETAIL_INFO_BY_SUB_PROCESS_INSTANCE_ID_ERROR(10119, "query parent process instance detail info by sub process instance id error", "查询子流程该工作流实例错误"), + QUERY_PROCESS_INSTANCE_ALL_VARIABLES_ERROR(10120, "query process instance all variables error", "查询工作流自定义变量信息错误"), + ENCAPSULATION_PROCESS_INSTANCE_GANTT_STRUCTURE_ERROR(10121, "encapsulation process instance gantt structure error", "查询工作流实例甘特图数据错误"), + QUERY_PROCESS_DEFINITION_LIST_PAGING_ERROR(10122, "query process definition list paging error", "分页查询工作流定义列表错误"), + SIGN_OUT_ERROR(10123, "sign out error", "退出错误"), + TENANT_CODE_HAS_ALREADY_EXISTS(10124, "tenant code has already exists", "租户编码已存在"), + IP_IS_EMPTY(10125, "ip is empty", "IP地址不能为空"), SCHEDULE_CRON_REALEASE_NEED_NOT_CHANGE(10126, "schedule release is already {0}", "调度配置上线错误[{0}]"), CREATE_QUEUE_ERROR(10127, "create queue error", "创建队列错误"), QUEUE_NOT_EXIST(10128, "queue {0} not exists", "队列ID[{0}]不存在"), @@ -159,24 +160,41 @@ public enum Status { QUEUE_NAME_EXIST(10130, "queue name {0} already exists", "队列名称[{0}]已存在"), UPDATE_QUEUE_ERROR(10131, "update queue error", "更新队列信息错误"), NEED_NOT_UPDATE_QUEUE(10132, "no content changes, no updates are required", "数据未变更，不需要更新队列信息"), - VERIFY_QUEUE_ERROR(10133,"verify queue error", "验证队列信息错误"), - NAME_NULL(10134,"name must be not null", "名称不能为空"), + VERIFY_QUEUE_ERROR(10133, "verify queue error", "验证队列信息错误"), + NAME_NULL(10134, "name must be not null", "名称不能为空"), NAME_EXIST(10135, "name {0} already exists", "名称[{0}]已存在"), SAVE_ERROR(10136, "save error", "保存错误"), DELETE_PROJECT_ERROR_DEFINES_NOT_NULL(10137, "please delete the process definitions in project first!", "请先删除全部工作流定义"), - BATCH_DELETE_PROCESS_INSTANCE_BY_IDS_ERROR(10117,"batch delete process instance by ids {0} error", "批量删除工作流实例错误"), - PREVIEW_SCHEDULE_ERROR(10139,"preview schedule error", "预览调度配置错误"), - PARSE_TO_CRON_EXPRESSION_ERROR(10140,"parse cron to cron expression error", "解析调度表达式错误"), - SCHEDULE_START_TIME_END_TIME_SAME(10141,"The start time must not be the same as the end", "开始时间不能和结束时间一样"), - DELETE_TENANT_BY_ID_FAIL(100142,"delete tenant by id fail, for there are {0} process instances in executing using it", "删除租户失败，有[{0}]个运行中的工作流实例正在使用"), - DELETE_TENANT_BY_ID_FAIL_DEFINES(100143,"delete tenant by id fail, for there are {0} process definitions using it", "删除租户失败，有[{0}]个工作流定义正在使用"), - DELETE_TENANT_BY_ID_FAIL_USERS(100144,"delete tenant by id fail, for there are {0} users using it", "删除租户失败，有[{0}]个用户正在使用"), - DELETE_WORKER_GROUP_BY_ID_FAIL(100145,"delete worker group by id fail, for there are {0} process instances in executing using it", "删除Worker分组失败，有[{0}]个运行中的工作流实例正在使用"), - QUERY_WORKER_GROUP_FAIL(100146,"query worker group fail ", "查询worker分组失败"), - DELETE_WORKER_GROUP_FAIL(100147,"delete worker group fail ", "删除worker分组失败"), - QUERY_WORKFLOW_LINEAGE_ERROR(10143,"query workflow lineage error", "查询血缘失败"), - COPY_PROCESS_DEFINITION_ERROR(10148,"copy process definition error", "复制工作流错误"), - USER_DISABLED(10149,"The current user is disabled", "当前用户已停用"), + BATCH_DELETE_PROCESS_INSTANCE_BY_IDS_ERROR(10117, "batch delete process instance by ids {0} error", "批量删除工作流实例错误"), + PREVIEW_SCHEDULE_ERROR(10139, "preview schedule error", "预览调度配置错误"), + PARSE_TO_CRON_EXPRESSION_ERROR(10140, "parse cron to cron expression error", "解析调度表达式错误"), + SCHEDULE_START_TIME_END_TIME_SAME(10141, "The start time must not be the same as the end", "开始时间不能和结束时间一样"), + DELETE_TENANT_BY_ID_FAIL(10142, "delete tenant by id fail, for there are {0} process instances in executing using it", "删除租户失败，有[{0}]个运行中的工作流实例正在使用"), + DELETE_TENANT_BY_ID_FAIL_DEFINES(10143, "delete tenant by id fail, for there are {0} process definitions using it", "删除租户失败，有[{0}]个工作流定义正在使用"), + DELETE_TENANT_BY_ID_FAIL_USERS(10144, "delete tenant by id fail, for there are {0} users using it", "删除租户失败，有[{0}]个用户正在使用"), + DELETE_WORKER_GROUP_BY_ID_FAIL(10145, "delete worker group by id fail, for there are {0} process instances in executing using it", "删除Worker分组失败，有[{0}]个运行中的工作流实例正在使用"), + QUERY_WORKER_GROUP_FAIL(10146, "query worker group fail ", "查询worker分组失败"), + DELETE_WORKER_GROUP_FAIL(10147, "delete worker group fail ", "删除worker分组失败"), + USER_DISABLED(10148, "The current user is disabled", "当前用户已停用"), + COPY_PROCESS_DEFINITION_ERROR(10149, "copy process definition from {0} to {1} error : {2}", "从{0}复制工作流到{1}错误 : {2}"), + MOVE_PROCESS_DEFINITION_ERROR(10150, "move process definition from {0} to {1} error : {2}", "从{0}移动工作流到{1}错误 : {2}"), + SWITCH_PROCESS_DEFINITION_VERSION_ERROR(10151, "Switch process definition version error", "切换工作流版本出错"), + SWITCH_PROCESS_DEFINITION_VERSION_NOT_EXIST_PROCESS_DEFINITION_ERROR(10152 + , "Switch process definition version error: not exists process definition, [process definition id {0}]", "切换工作流版本出错：工作流不存在，[工作流id {0}]"), + SWITCH_PROCESS_DEFINITION_VERSION_NOT_EXIST_PROCESS_DEFINITION_VERSION_ERROR(10153 + , "Switch process definition version error: not exists process definition version, [process definition id {0}] [version number {1}]", "切换工作流版本出错：工作流版本信息不存在，[工作流id {0}] [版本号 {1}]"), + QUERY_PROCESS_DEFINITION_VERSIONS_ERROR(10154, "query process definition versions error", "查询工作流历史版本信息出错"), + QUERY_PROCESS_DEFINITION_VERSIONS_PAGE_NO_OR_PAGE_SIZE_LESS_THAN_1_ERROR(10155 + , "query process definition versions error: [page number:{0}] < 1 or [page size:{1}] < 1", "查询工作流历史版本出错：[pageNo:{0}] < 1 或 [pageSize:{1}] < 1"), + DELETE_PROCESS_DEFINITION_VERSION_ERROR(10156, "delete process definition version error", "删除工作流历史版本出错"), + + QUERY_USER_CREATED_PROJECT_ERROR(10157, "query user created project error error", "查询用户创建的项目错误"), + PROCESS_DEFINITION_IDS_IS_EMPTY(10158, "process definition ids is empty", "工作流IDS不能为空"), + BATCH_COPY_PROCESS_DEFINITION_ERROR(10159, "batch copy process definition error", "复制工作流错误"), + BATCH_MOVE_PROCESS_DEFINITION_ERROR(10160, "batch move process definition error", "移动工作流错误"), + QUERY_WORKFLOW_LINEAGE_ERROR(10161, "query workflow lineage error", "查询血缘失败"), + DELETE_PROCESS_DEFINITION_BY_ID_FAIL(10162,"delete process definition by id fail, for there are {0} process instances in executing using it", "删除工作流定义失败，有[{0}]个运行中的工作流实例正在使用"), + UDF_FUNCTION_NOT_EXIST(20001, "UDF function not found", "UDF函数不存在"), UDF_FUNCTION_EXISTS(20002, "UDF function already exists", "UDF函数已存在"), RESOURCE_NOT_EXIST(20004, "resource not exist", "资源不存在"), @@ -188,10 +206,10 @@ public enum Status { HDFS_COPY_FAIL(20010, "hdfs copy {0} -> {1} fail", "hdfs复制失败：[{0}] -> [{1}]"), RESOURCE_FILE_EXIST(20011, "resource file {0} already exists in hdfs,please delete it or change name!", "资源文件[{0}]在hdfs中已存在，请删除或修改资源名"), RESOURCE_FILE_NOT_EXIST(20012, "resource file {0} not exists in hdfs!", "资源文件[{0}]在hdfs中不存在"), - UDF_RESOURCE_IS_BOUND(20013, "udf resource file is bound by UDF functions:{0}","udf函数绑定了资源文件[{0}]"), - RESOURCE_IS_USED(20014, "resource file is used by process definition","资源文件被上线的流程定义使用了"), - PARENT_RESOURCE_NOT_EXIST(20015, "parent resource not exist","父资源文件不存在"), - RESOURCE_NOT_EXIST_OR_NO_PERMISSION(20016, "resource not exist or no permission,please view the task node and remove error resource","请检查任务节点并移除无权限或者已删除的资源"), + UDF_RESOURCE_IS_BOUND(20013, "udf resource file is bound by UDF functions:{0}", "udf函数绑定了资源文件[{0}]"), + RESOURCE_IS_USED(20014, "resource file is used by process definition", "资源文件被上线的流程定义使用了"), + PARENT_RESOURCE_NOT_EXIST(20015, "parent resource not exist", "父资源文件不存在"), + RESOURCE_NOT_EXIST_OR_NO_PERMISSION(20016, "resource not exist or no permission,please view the task node and remove error resource", "请检查任务节点并移除无权限或者已删除的资源"), RESOURCE_IS_AUTHORIZED(20017, "resource is authorized to user {0},suffix not allowed to be modified", "资源文件已授权其他用户[{0}],后缀不允许修改"), USER_NO_OPERATION_PERM(30001, "user has no operation privilege", "当前用户没有操作权限"), @@ -208,52 +226,51 @@ public enum Status { PROCESS_DEFINE_NOT_ALLOWED_EDIT(50008, "process definition {0} does not allow edit", "工作流定义[{0}]不允许修改"), PROCESS_INSTANCE_EXECUTING_COMMAND(50009, "process instance {0} is executing the command, please wait ...", "工作流实例[{0}]正在执行命令，请稍等..."), PROCESS_INSTANCE_NOT_SUB_PROCESS_INSTANCE(50010, "process instance {0} is not sub process instance", "工作流实例[{0}]不是子工作流实例"), - TASK_INSTANCE_STATE_COUNT_ERROR(50011,"task instance state count error", "查询各状态任务实例数错误"), - COUNT_PROCESS_INSTANCE_STATE_ERROR(50012,"count process instance state error", "查询各状态流程实例数错误"), - COUNT_PROCESS_DEFINITION_USER_ERROR(50013,"count process definition user error", "查询各用户流程定义数错误"), - START_PROCESS_INSTANCE_ERROR(50014,"start process instance error", "运行工作流实例错误"), - EXECUTE_PROCESS_INSTANCE_ERROR(50015,"execute process instance error", "操作工作流实例错误"), - CHECK_PROCESS_DEFINITION_ERROR(50016,"check process definition error", "检查工作流实例错误"), - QUERY_RECIPIENTS_AND_COPYERS_BY_PROCESS_DEFINITION_ERROR(50017,"query recipients and copyers by process definition error", "查询收件人和抄送人错误"), - DATA_IS_NOT_VALID(50017,"data {0} not valid", "数据[{0}]无效"), - DATA_IS_NULL(50018,"data {0} is null", "数据[{0}]不能为空"), - PROCESS_NODE_HAS_CYCLE(50019,"process node has cycle", "流程节点间存在循环依赖"), - PROCESS_NODE_S_PARAMETER_INVALID(50020,"process node {0} parameter invalid", "流程节点[{0}]参数无效"), + TASK_INSTANCE_STATE_COUNT_ERROR(50011, "task instance state count error", "查询各状态任务实例数错误"), + COUNT_PROCESS_INSTANCE_STATE_ERROR(50012, "count process instance state error", "查询各状态流程实例数错误"), + COUNT_PROCESS_DEFINITION_USER_ERROR(50013, "count process definition user error", "查询各用户流程定义数错误"), + START_PROCESS_INSTANCE_ERROR(50014, "start process instance error", "运行工作流实例错误"), + EXECUTE_PROCESS_INSTANCE_ERROR(50015, "execute process instance error", "操作工作流实例错误"), + CHECK_PROCESS_DEFINITION_ERROR(50016, "check process definition error", "检查工作流实例错误"), + QUERY_RECIPIENTS_AND_COPYERS_BY_PROCESS_DEFINITION_ERROR(50017, "query recipients and copyers by process definition error", "查询收件人和抄送人错误"), + DATA_IS_NOT_VALID(50017, "data {0} not valid", "数据[{0}]无效"), + DATA_IS_NULL(50018, "data {0} is null", "数据[{0}]不能为空"), + PROCESS_NODE_HAS_CYCLE(50019, "process node has cycle", "流程节点间存在循环依赖"), + PROCESS_NODE_S_PARAMETER_INVALID(50020, "process node {0} parameter invalid", "流程节点[{0}]参数无效"), PROCESS_DEFINE_STATE_ONLINE(50021, "process definition {0} is already on line", "工作流定义[{0}]已上线"), - DELETE_PROCESS_DEFINE_BY_ID_ERROR(50022,"delete process definition by id error", "删除工作流定义错误"), - SCHEDULE_CRON_STATE_ONLINE(50023,"the status of schedule {0} is already on line", "调度配置[{0}]已上线"), - DELETE_SCHEDULE_CRON_BY_ID_ERROR(50024,"delete schedule by id error", "删除调度配置错误"), - BATCH_DELETE_PROCESS_DEFINE_ERROR(50025,"batch delete process definition error", "批量删除工作流定义错误"), - BATCH_DELETE_PROCESS_DEFINE_BY_IDS_ERROR(50026,"batch delete process definition by ids {0} error", "批量删除工作流定义[{0}]错误"), - TENANT_NOT_SUITABLE(50027,"there is not any tenant suitable, please choose a tenant available.", "没有合适的租户，请选择可用的租户"), - EXPORT_PROCESS_DEFINE_BY_ID_ERROR(50028,"export process definition by id error", "导出工作流定义错误"), - BATCH_EXPORT_PROCESS_DEFINE_BY_IDS_ERROR(50028,"batch export process definition by ids error", "批量导出工作流定义错误"), - IMPORT_PROCESS_DEFINE_ERROR(50029,"import process definition error", "导入工作流定义错误"), + DELETE_PROCESS_DEFINE_BY_ID_ERROR(50022, "delete process definition by id error", "删除工作流定义错误"), + SCHEDULE_CRON_STATE_ONLINE(50023, "the status of schedule {0} is already on line", "调度配置[{0}]已上线"), + DELETE_SCHEDULE_CRON_BY_ID_ERROR(50024, "delete schedule by id error", "删除调度配置错误"), + BATCH_DELETE_PROCESS_DEFINE_ERROR(50025, "batch delete process definition error", "批量删除工作流定义错误"), + BATCH_DELETE_PROCESS_DEFINE_BY_IDS_ERROR(50026, "batch delete process definition by ids {0} error", "批量删除工作流定义[{0}]错误"), + TENANT_NOT_SUITABLE(50027, "there is not any tenant suitable, please choose a tenant available.", "没有合适的租户，请选择可用的租户"), + EXPORT_PROCESS_DEFINE_BY_ID_ERROR(50028, "export process definition by id error", "导出工作流定义错误"), + BATCH_EXPORT_PROCESS_DEFINE_BY_IDS_ERROR(50028, "batch export process definition by ids error", "批量导出工作流定义错误"), + IMPORT_PROCESS_DEFINE_ERROR(50029, "import process definition error", "导入工作流定义错误"), - HDFS_NOT_STARTUP(60001,"hdfs not startup", "hdfs未启用"), + HDFS_NOT_STARTUP(60001, "hdfs not startup", "hdfs未启用"), /** * for monitor */ - QUERY_DATABASE_STATE_ERROR(70001,"query database state error", "查询数据库状态错误"), - QUERY_ZOOKEEPER_STATE_ERROR(70002,"query zookeeper state error", "查询zookeeper状态错误"), + QUERY_DATABASE_STATE_ERROR(70001, "query database state error", "查询数据库状态错误"), + QUERY_ZOOKEEPER_STATE_ERROR(70002, "query zookeeper state error", "查询zookeeper状态错误"), - - CREATE_ACCESS_TOKEN_ERROR(70010,"create access token error", "创建访问token错误"), - GENERATE_TOKEN_ERROR(70011,"generate token error", "生成token错误"), - QUERY_ACCESSTOKEN_LIST_PAGING_ERROR(70012,"query access token list paging error", "分页查询访问token列表错误"), - UPDATE_ACCESS_TOKEN_ERROR(70013,"update access token error", "更新访问token错误"), - DELETE_ACCESS_TOKEN_ERROR(70014,"delete access token error", "删除访问token错误"), + CREATE_ACCESS_TOKEN_ERROR(70010, "create access token error", "创建访问token错误"), + GENERATE_TOKEN_ERROR(70011, "generate token error", "生成token错误"), + QUERY_ACCESSTOKEN_LIST_PAGING_ERROR(70012, "query access token list paging error", "分页查询访问token列表错误"), + UPDATE_ACCESS_TOKEN_ERROR(70013, "update access token error", "更新访问token错误"), + DELETE_ACCESS_TOKEN_ERROR(70014, "delete access token error", "删除访问token错误"), ACCESS_TOKEN_NOT_EXIST(70015, "access token not exist", "访问token不存在"), - COMMAND_STATE_COUNT_ERROR(80001,"task instance state count error", "查询各状态任务实例数错误"), - NEGTIVE_SIZE_NUMBER_ERROR(80002,"query size number error","查询size错误"), - START_TIME_BIGGER_THAN_END_TIME_ERROR(80003,"start time bigger than end time error","开始时间在结束时间之后错误"), - QUEUE_COUNT_ERROR(90001,"queue count error", "查询队列数据错误"), + COMMAND_STATE_COUNT_ERROR(80001, "task instance state count error", "查询各状态任务实例数错误"), + NEGTIVE_SIZE_NUMBER_ERROR(80002, "query size number error", "查询size错误"), + START_TIME_BIGGER_THAN_END_TIME_ERROR(80003, "start time bigger than end time error", "开始时间在结束时间之后错误"), + QUEUE_COUNT_ERROR(90001, "queue count error", "查询队列数据错误"), - KERBEROS_STARTUP_STATE(100001,"get kerberos startup state error", "获取kerberos启动状态错误"), + KERBEROS_STARTUP_STATE(100001, "get kerberos startup state error", "获取kerberos启动状态错误"), ; private final int code; @@ -277,4 +294,4 @@ public enum Status { return this.enMsg; } } -} +} \ No newline at end of file diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/exceptions/ApiExceptionHandler.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/exceptions/ApiExceptionHandler.java index 90d1afea49..cd6ac2b622 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/exceptions/ApiExceptionHandler.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/exceptions/ApiExceptionHandler.java @@ -18,17 +18,18 @@ package org.apache.dolphinscheduler.api.exceptions; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.utils.Result; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.web.bind.annotation.ControllerAdvice; import org.springframework.web.bind.annotation.ExceptionHandler; import org.springframework.web.bind.annotation.ResponseBody; +import org.springframework.web.bind.annotation.RestControllerAdvice; import org.springframework.web.method.HandlerMethod; /** * Exception Handler */ -@ControllerAdvice +@RestControllerAdvice @ResponseBody public class ApiExceptionHandler { diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/interceptor/LoginHandlerInterceptor.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/interceptor/LoginHandlerInterceptor.java index cb7a8e653f..83eb4fefce 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/interceptor/LoginHandlerInterceptor.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/interceptor/LoginHandlerInterceptor.java @@ -16,32 +16,28 @@ */ package org.apache.dolphinscheduler.api.interceptor; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.apache.commons.httpclient.HttpStatus; +import org.apache.commons.lang.StringUtils; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.security.Authenticator; -import org.apache.dolphinscheduler.api.service.SessionService; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.Flag; import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.mapper.UserMapper; -import org.apache.commons.httpclient.HttpStatus; -import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.servlet.HandlerInterceptor; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - /** * login interceptor, must login first */ public class LoginHandlerInterceptor implements HandlerInterceptor { private static final Logger logger = LoggerFactory.getLogger(LoginHandlerInterceptor.class); - @Autowired - private SessionService sessionService; - @Autowired private UserMapper userMapper; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AccessTokenService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AccessTokenService.java index 5d176961bb..98eef47090 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AccessTokenService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AccessTokenService.java @@ -16,35 +16,14 @@ */ package org.apache.dolphinscheduler.api.service; -import org.apache.dolphinscheduler.api.enums.Status; -import org.apache.dolphinscheduler.api.utils.PageInfo; -import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.common.enums.UserType; -import org.apache.dolphinscheduler.dao.entity.AccessToken; import org.apache.dolphinscheduler.dao.entity.User; -import com.baomidou.mybatisplus.core.metadata.IPage; -import com.baomidou.mybatisplus.extension.plugins.pagination.Page; -import org.apache.dolphinscheduler.common.utils.DateUtils; -import org.apache.dolphinscheduler.common.utils.EncryptionUtils; -import org.apache.dolphinscheduler.dao.mapper.AccessTokenMapper; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; -import java.util.*; +import java.util.Map; /** - * user service + * access token service */ -@Service -public class AccessTokenService extends BaseService { - - private static final Logger logger = LoggerFactory.getLogger(AccessTokenService.class); - - @Autowired - private AccessTokenMapper accessTokenMapper; - +public interface AccessTokenService { /** * query access token list @@ -55,123 +34,44 @@ public class AccessTokenService extends BaseService { * @param pageSize page size * @return token list for page number and page size */ - public Map queryAccessTokenList(User loginUser, String searchVal, Integer pageNo, Integer pageSize) { - Map result = new HashMap<>(5); - - PageInfo pageInfo = new PageInfo<>(pageNo, pageSize); - Page page = new Page(pageNo, pageSize); - int userId = loginUser.getId(); - if (loginUser.getUserType() == UserType.ADMIN_USER){ - userId = 0; - } - IPage accessTokenList = accessTokenMapper.selectAccessTokenPage(page, searchVal, userId); - pageInfo.setTotalCount((int)accessTokenList.getTotal()); - pageInfo.setLists(accessTokenList.getRecords()); - result.put(Constants.DATA_LIST, pageInfo); - putMsg(result, Status.SUCCESS); - - return result; - } + Map queryAccessTokenList(User loginUser, String searchVal, Integer pageNo, Integer pageSize); /** * create token + * * @param userId token for user * @param expireTime token expire time * @param token token string * @return create result code */ - public Map createToken(int userId, String expireTime, String token) { - Map result = new HashMap<>(5); - - if (userId <= 0) { - throw new IllegalArgumentException("User id should not less than or equals to 0."); - } - AccessToken accessToken = new AccessToken(); - accessToken.setUserId(userId); - accessToken.setExpireTime(DateUtils.stringToDate(expireTime)); - accessToken.setToken(token); - accessToken.setCreateTime(new Date()); - accessToken.setUpdateTime(new Date()); - - // insert - int insert = accessTokenMapper.insert(accessToken); - - if (insert > 0) { - putMsg(result, Status.SUCCESS); - } else { - putMsg(result, Status.CREATE_ACCESS_TOKEN_ERROR); - } - - return result; - } + Map createToken(int userId, String expireTime, String token); /** * generate token + * * @param userId token for user * @param expireTime token expire time * @return token string */ - public Map generateToken(int userId, String expireTime) { - Map result = new HashMap<>(5); - String token = EncryptionUtils.getMd5(userId + expireTime + String.valueOf(System.currentTimeMillis())); - result.put(Constants.DATA_LIST, token); - putMsg(result, Status.SUCCESS); - return result; - } + Map generateToken(int userId, String expireTime); /** - * delete access token + * delete access token + * * @param loginUser login user * @param id token id * @return delete result code */ - public Map delAccessTokenById(User loginUser, int id) { - Map result = new HashMap<>(5); - - AccessToken accessToken = accessTokenMapper.selectById(id); - - if (accessToken == null) { - logger.error("access token not exist, access token id {}", id); - putMsg(result, Status.ACCESS_TOKEN_NOT_EXIST); - return result; - } - - if (loginUser.getId() != accessToken.getUserId() && - loginUser.getUserType() != UserType.ADMIN_USER) { - putMsg(result, Status.USER_NO_OPERATION_PERM); - return result; - } - - accessTokenMapper.deleteById(id); - putMsg(result, Status.SUCCESS); - return result; - } + Map delAccessTokenById(User loginUser, int id); /** * update token by id + * * @param id token id * @param userId token for user * @param expireTime token expire time * @param token token string * @return update result code */ - public Map updateToken(int id,int userId, String expireTime, String token) { - Map result = new HashMap<>(5); - - AccessToken accessToken = accessTokenMapper.selectById(id); - if (accessToken == null) { - logger.error("access token not exist, access token id {}", id); - putMsg(result, Status.ACCESS_TOKEN_NOT_EXIST); - return result; - } - accessToken.setUserId(userId); - accessToken.setExpireTime(DateUtils.stringToDate(expireTime)); - accessToken.setToken(token); - accessToken.setUpdateTime(new Date()); - - accessTokenMapper.updateById(accessToken); - - putMsg(result, Status.SUCCESS); - return result; - } + Map updateToken(int id, int userId, String expireTime, String token); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AlertGroupService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AlertGroupService.java index f3dcbfa237..72cbd50833 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AlertGroupService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AlertGroupService.java @@ -55,7 +55,7 @@ public class AlertGroupService extends BaseService{ */ public HashMap queryAlertgroup() { - HashMap result = new HashMap<>(5); + HashMap result = new HashMap<>(); List alertGroups = alertGroupMapper.queryAllGroupList(); result.put(Constants.DATA_LIST, alertGroups); putMsg(result, Status.SUCCESS); @@ -74,7 +74,7 @@ public class AlertGroupService extends BaseService{ */ public Map listPaging(User loginUser, String searchVal, Integer pageNo, Integer pageSize) { - Map result = new HashMap<>(5); + Map result = new HashMap<>(); if (checkAdmin(loginUser, result)) { return result; } @@ -101,7 +101,7 @@ public class AlertGroupService extends BaseService{ * @return create result code */ public Map createAlertgroup(User loginUser, String groupName, AlertType groupType, String desc) { - Map result = new HashMap<>(5); + Map result = new HashMap<>(); //only admin can operate if (checkAdmin(loginUser, result)){ return result; @@ -138,7 +138,7 @@ public class AlertGroupService extends BaseService{ * @return update result code */ public Map updateAlertgroup(User loginUser, int id, String groupName, AlertType groupType, String desc) { - Map result = new HashMap<>(5); + Map result = new HashMap<>(); if (checkAdmin(loginUser, result)){ return result; @@ -179,7 +179,7 @@ public class AlertGroupService extends BaseService{ */ @Transactional(rollbackFor = RuntimeException.class) public Map delAlertgroupById(User loginUser, int id) { - Map result = new HashMap<>(5); + Map result = new HashMap<>(); result.put(Constants.STATUS, false); //only admin can operate @@ -209,7 +209,7 @@ public class AlertGroupService extends BaseService{ * @return grant result code */ public Map grantUser(User loginUser, int alertgroupId, String userIds) { - Map result = new HashMap<>(5); + Map result = new HashMap<>(); result.put(Constants.STATUS, false); //only admin can operate diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/BaseDAGService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/BaseDAGService.java deleted file mode 100644 index edc115b3d4..0000000000 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/BaseDAGService.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.dolphinscheduler.api.service; - -import org.apache.dolphinscheduler.common.graph.DAG; -import org.apache.dolphinscheduler.common.model.TaskNode; -import org.apache.dolphinscheduler.common.model.TaskNodeRelation; -import org.apache.dolphinscheduler.common.process.ProcessDag; -import org.apache.dolphinscheduler.common.utils.*; -import org.apache.dolphinscheduler.dao.entity.ProcessData; -import org.apache.dolphinscheduler.dao.entity.ProcessInstance; -import org.apache.dolphinscheduler.dao.utils.DagHelper; - -import java.util.List; - -/** - * base DAG service - */ -public class BaseDAGService extends BaseService{ - - - /** - * process instance to DAG - * - * @param processInstance input process instance - * @return process instance dag. - */ - public static DAG processInstance2DAG(ProcessInstance processInstance) { - - String processDefinitionJson = processInstance.getProcessInstanceJson(); - - ProcessData processData = JSONUtils.parseObject(processDefinitionJson, ProcessData.class); - - List taskNodeList = processData.getTasks(); - - ProcessDag processDag = DagHelper.getProcessDag(taskNodeList); - - return DagHelper.buildDagGraph(processDag); - } -} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/BaseService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/BaseService.java index 646a67ab04..4b094ea494 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/BaseService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/BaseService.java @@ -16,6 +16,12 @@ */ package org.apache.dolphinscheduler.api.service; +import java.text.MessageFormat; +import java.util.Map; + +import javax.servlet.http.Cookie; +import javax.servlet.http.HttpServletRequest; + import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; @@ -24,11 +30,6 @@ import org.apache.dolphinscheduler.common.utils.HadoopUtils; import org.apache.dolphinscheduler.common.utils.StringUtils; import org.apache.dolphinscheduler.dao.entity.User; -import javax.servlet.http.Cookie; -import javax.servlet.http.HttpServletRequest; -import java.text.MessageFormat; -import java.util.Map; - /** * base service */ @@ -96,6 +97,7 @@ public class BaseService { /** * get cookie info by name + * * @param request request * @param name 'sessionId' * @return get cookie info @@ -115,10 +117,11 @@ public class BaseService { /** * create tenant dir if not exists + * * @param tenantCode tenant code * @throws Exception if hdfs operation exception */ - protected void createTenantDirIfNotExists(String tenantCode)throws Exception{ + protected void createTenantDirIfNotExists(String tenantCode) throws Exception { String resourcePath = HadoopUtils.getHdfsResDir(tenantCode); String udfsPath = HadoopUtils.getHdfsUdfDir(tenantCode); @@ -129,7 +132,7 @@ public class BaseService { HadoopUtils.getInstance().mkdir(udfsPath); } - protected boolean hasPerm(User operateUser, int createUserId){ + protected boolean hasPerm(User operateUser, int createUserId) { return operateUser.getId() == createUserId || isAdmin(operateUser); } } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataAnalysisService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataAnalysisService.java index 39bec56357..70fb272bea 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataAnalysisService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataAnalysisService.java @@ -17,57 +17,14 @@ package org.apache.dolphinscheduler.api.service; -import org.apache.dolphinscheduler.api.dto.CommandStateCount; -import org.apache.dolphinscheduler.api.dto.DefineUserDto; -import org.apache.dolphinscheduler.api.dto.TaskCountDto; -import org.apache.dolphinscheduler.api.enums.Status; -import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.common.enums.CommandType; -import org.apache.dolphinscheduler.common.enums.UserType; -import org.apache.dolphinscheduler.common.utils.DateUtils; -import org.apache.dolphinscheduler.common.utils.StringUtils; -import org.apache.dolphinscheduler.dao.entity.*; -import org.apache.dolphinscheduler.dao.mapper.*; -import org.apache.dolphinscheduler.service.process.ProcessService; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; +import org.apache.dolphinscheduler.dao.entity.User; -import java.text.MessageFormat; -import java.util.*; +import java.util.Map; /** * data analysis service */ -@Service -public class DataAnalysisService extends BaseService{ - - private static final Logger logger = LoggerFactory.getLogger(DataAnalysisService.class); - - @Autowired - ProjectMapper projectMapper; - - @Autowired - ProjectService projectService; - - @Autowired - ProcessInstanceMapper processInstanceMapper; - - @Autowired - ProcessDefinitionMapper processDefinitionMapper; - - @Autowired - CommandMapper commandMapper; - - @Autowired - ErrorCommandMapper errorCommandMapper; - - @Autowired - TaskInstanceMapper taskInstanceMapper; - - @Autowired - ProcessService processService; +public interface DataAnalysisService { /** * statistical task instance status data @@ -78,46 +35,7 @@ public class DataAnalysisService extends BaseService{ * @param endDate end date * @return task state count data */ - public Map countTaskStateByProject(User loginUser, int projectId, String startDate, String endDate) { - - Map result = new HashMap<>(5); - boolean checkProject = checkProject(loginUser, projectId, result); - if(!checkProject){ - return result; - } - - /** - * find all the task lists in the project under the user - * statistics based on task status execution, failure, completion, wait, total - */ - Date start = null; - Date end = null; - - try { - start = DateUtils.getScheduleDate(startDate); - end = DateUtils.getScheduleDate(endDate); - } catch (Exception e) { - logger.error(e.getMessage(),e); - putErrorRequestParamsMsg(result); - return result; - } - - Integer[] projectIds = getProjectIdsArrays(loginUser, projectId); - List taskInstanceStateCounts = - taskInstanceMapper.countTaskInstanceStateByUser(start, end, projectIds); - - if (taskInstanceStateCounts != null) { - TaskCountDto taskCountResult = new TaskCountDto(taskInstanceStateCounts); - result.put(Constants.DATA_LIST, taskCountResult); - putMsg(result, Status.SUCCESS); - } - return result; - } - - private void putErrorRequestParamsMsg(Map result) { - result.put(Constants.STATUS, Status.REQUEST_PARAMS_NOT_VALID_ERROR); - result.put(Constants.MSG, MessageFormat.format(Status.REQUEST_PARAMS_NOT_VALID_ERROR.getMsg(), "startDate,endDate")); - } + Map countTaskStateByProject(User loginUser, int projectId, String startDate, String endDate); /** * statistical process instance status data @@ -128,37 +46,7 @@ public class DataAnalysisService extends BaseService{ * @param endDate end date * @return process instance state count data */ - public Map countProcessInstanceStateByProject(User loginUser, int projectId, String startDate, String endDate) { - - Map result = new HashMap<>(5); - boolean checkProject = checkProject(loginUser, projectId, result); - if(!checkProject){ - return result; - } - - Date start = null; - Date end = null; - try { - start = DateUtils.getScheduleDate(startDate); - end = DateUtils.getScheduleDate(endDate); - } catch (Exception e) { - logger.error(e.getMessage(),e); - putErrorRequestParamsMsg(result); - return result; - } - Integer[] projectIdArray = getProjectIdsArrays(loginUser, projectId); - List processInstanceStateCounts = - processInstanceMapper.countInstanceStateByUser(start, end, - projectIdArray); - - if (processInstanceStateCounts != null) { - TaskCountDto taskCountResult = new TaskCountDto(processInstanceStateCounts); - result.put(Constants.DATA_LIST, taskCountResult); - putMsg(result, Status.SUCCESS); - } - return result; - } - + Map countProcessInstanceStateByProject(User loginUser, int projectId, String startDate, String endDate); /** * statistics the process definition quantities of certain person @@ -167,20 +55,7 @@ public class DataAnalysisService extends BaseService{ * @param projectId project id * @return definition count data */ - public Map countDefinitionByUser(User loginUser, int projectId) { - Map result = new HashMap<>(); - - - Integer[] projectIdArray = getProjectIdsArrays(loginUser, projectId); - List defineGroupByUsers = processDefinitionMapper.countDefinitionGroupByUser( - loginUser.getId(), projectIdArray,isAdmin(loginUser)); - - DefineUserDto dto = new DefineUserDto(defineGroupByUsers); - result.put(Constants.DATA_LIST, dto); - putMsg(result, Status.SUCCESS); - return result; - } - + Map countDefinitionByUser(User loginUser, int projectId); /** * statistical command status data @@ -191,189 +66,15 @@ public class DataAnalysisService extends BaseService{ * @param endDate end date * @return command state count data */ - public Map countCommandState(User loginUser, int projectId, String startDate, String endDate) { - - Map result = new HashMap<>(5); - boolean checkProject = checkProject(loginUser, projectId, result); - if(!checkProject){ - return result; - } - - /** - * find all the task lists in the project under the user - * statistics based on task status execution, failure, completion, wait, total - */ - Date start = null; - Date end = null; - - if (startDate != null && endDate != null){ - try { - start = DateUtils.getScheduleDate(startDate); - end = DateUtils.getScheduleDate(endDate); - } catch (Exception e) { - logger.error(e.getMessage(),e); - putErrorRequestParamsMsg(result); - return result; - } - } - - - Integer[] projectIdArray = getProjectIdsArrays(loginUser, projectId); - // count command state - List commandStateCounts = - commandMapper.countCommandState( - loginUser.getId(), - start, - end, - projectIdArray); - - // count error command state - List errorCommandStateCounts = - errorCommandMapper.countCommandState( - start, end, projectIdArray); - - // - Map> dataMap = new HashMap<>(); - - Map commonCommand = new HashMap<>(); - commonCommand.put("commandState",0); - commonCommand.put("errorCommandState",0); - - - // init data map - /** - * START_PROCESS, START_CURRENT_TASK_PROCESS, RECOVER_TOLERANCE_FAULT_PROCESS, RECOVER_SUSPENDED_PROCESS, - START_FAILURE_TASK_PROCESS,COMPLEMENT_DATA,SCHEDULER, REPEAT_RUNNING,PAUSE,STOP,RECOVER_WAITTING_THREAD; - */ - dataMap.put(CommandType.START_PROCESS,commonCommand); - dataMap.put(CommandType.START_CURRENT_TASK_PROCESS,commonCommand); - dataMap.put(CommandType.RECOVER_TOLERANCE_FAULT_PROCESS,commonCommand); - dataMap.put(CommandType.RECOVER_SUSPENDED_PROCESS,commonCommand); - dataMap.put(CommandType.START_FAILURE_TASK_PROCESS,commonCommand); - dataMap.put(CommandType.COMPLEMENT_DATA,commonCommand); - dataMap.put(CommandType.SCHEDULER,commonCommand); - dataMap.put(CommandType.REPEAT_RUNNING,commonCommand); - dataMap.put(CommandType.PAUSE,commonCommand); - dataMap.put(CommandType.STOP,commonCommand); - dataMap.put(CommandType.RECOVER_WAITTING_THREAD,commonCommand); - - // put command state - for (CommandCount executeStatusCount : commandStateCounts){ - Map commandStateCountsMap = new HashMap<>(dataMap.get(executeStatusCount.getCommandType())); - commandStateCountsMap.put("commandState", executeStatusCount.getCount()); - dataMap.put(executeStatusCount.getCommandType(),commandStateCountsMap); - } - - // put error command state - for (CommandCount errorExecutionStatus : errorCommandStateCounts){ - Map errorCommandStateCountsMap = new HashMap<>(dataMap.get(errorExecutionStatus.getCommandType())); - errorCommandStateCountsMap.put("errorCommandState",errorExecutionStatus.getCount()); - dataMap.put(errorExecutionStatus.getCommandType(),errorCommandStateCountsMap); - } - - List list = new ArrayList<>(); - Iterator>> iterator = dataMap.entrySet().iterator(); - while (iterator.hasNext()){ - Map.Entry> next = iterator.next(); - CommandStateCount commandStateCount = new CommandStateCount(next.getValue().get("errorCommandState"), - next.getValue().get("commandState"),next.getKey()); - list.add(commandStateCount); - } - - result.put(Constants.DATA_LIST, list); - putMsg(result, Status.SUCCESS); - return result; - } - - private Integer[] getProjectIdsArrays(User loginUser, int projectId) { - List projectIds = new ArrayList<>(); - if(projectId !=0){ - projectIds.add(projectId); - }else if(loginUser.getUserType() == UserType.GENERAL_USER){ - projectIds = processService.getProjectIdListHavePerm(loginUser.getId()); - if(projectIds.size() ==0 ){ - projectIds.add(0); - } - } - return projectIds.toArray(new Integer[projectIds.size()]); - } + Map countCommandState(User loginUser, int projectId, String startDate, String endDate); /** * count queue state + * * @param loginUser login user * @param projectId project id * @return queue state count data */ - public Map countQueueState(User loginUser, int projectId) { - Map result = new HashMap<>(5); + Map countQueueState(User loginUser, int projectId); - boolean checkProject = checkProject(loginUser, projectId, result); - if(!checkProject){ - return result; - } - - List tasksQueueList = new ArrayList<>(); - List tasksKillList = new ArrayList<>(); - - Map dataMap = new HashMap<>(); - if (loginUser.getUserType() == UserType.ADMIN_USER){ - dataMap.put("taskQueue",tasksQueueList.size()); - dataMap.put("taskKill",tasksKillList.size()); - - result.put(Constants.DATA_LIST, dataMap); - putMsg(result, Status.SUCCESS); - return result; - } - - int[] tasksQueueIds = new int[tasksQueueList.size()]; - int[] tasksKillIds = new int[tasksKillList.size()]; - - int i =0; - for (String taskQueueStr : tasksQueueList){ - if (StringUtils.isNotEmpty(taskQueueStr)){ - String[] splits = taskQueueStr.split("_"); - if (splits.length >= 4){ - tasksQueueIds[i++] = Integer.parseInt(splits[3]); - } - } - } - - i = 0; - for (String taskKillStr : tasksKillList){ - if (StringUtils.isNotEmpty(taskKillStr)){ - String[] splits = taskKillStr.split("-"); - if (splits.length == 2){ - tasksKillIds[i++] = Integer.parseInt(splits[1]); - } - } - } - Integer taskQueueCount = 0; - Integer taskKillCount = 0; - - Integer[] projectIds = getProjectIdsArrays(loginUser, projectId); - if (tasksQueueIds.length != 0){ - taskQueueCount = taskInstanceMapper.countTask( - projectIds, - tasksQueueIds); - } - - if (tasksKillIds.length != 0){ - taskKillCount = taskInstanceMapper.countTask(projectIds, tasksKillIds); - } - - dataMap.put("taskQueue",taskQueueCount); - dataMap.put("taskKill",taskKillCount); - - result.put(Constants.DATA_LIST, dataMap); - putMsg(result, Status.SUCCESS); - return result; - } - - private boolean checkProject(User loginUser, int projectId, Map result){ - if(projectId != 0){ - Project project = projectMapper.selectById(projectId); - return projectService.hasProjectAndPerm(loginUser, project, result); - } - return true; - } } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java index 41374f4478..74c2f6908f 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java @@ -53,7 +53,7 @@ import static org.apache.dolphinscheduler.common.utils.PropertyUtils.getString; * datasource service */ @Service -public class DataSourceService extends BaseService{ +public class DataSourceService extends BaseService { private static final Logger logger = LoggerFactory.getLogger(DataSourceService.class); @@ -65,7 +65,6 @@ public class DataSourceService extends BaseService{ public static final String PRINCIPAL = "principal"; public static final String DATABASE = "database"; public static final String USER_NAME = "userName"; - public static final String PASSWORD = Constants.PASSWORD; public static final String OTHER = "other"; @@ -80,15 +79,15 @@ public class DataSourceService extends BaseService{ * create data source * * @param loginUser login user - * @param name data source name - * @param desc data source description - * @param type data source type + * @param name data source name + * @param desc data source description + * @param type data source type * @param parameter datasource parameters * @return create result code */ public Map createDataSource(User loginUser, String name, String desc, DbType type, String parameter) { - Map result = new HashMap<>(5); + Map result = new HashMap<>(); // check name can use or not if (checkName(name)) { putMsg(result, Status.DATASOURCE_EXIST); @@ -131,11 +130,11 @@ public class DataSourceService extends BaseService{ * updateProcessInstance datasource * * @param loginUser login user - * @param name data source name - * @param desc data source description - * @param type data source type + * @param name data source name + * @param desc data source description + * @param type data source type * @param parameter datasource parameters - * @param id data source id + * @param id data source id * @return update result code */ public Map updateDataSource(int id, User loginUser, String name, String desc, DbType type, String parameter) { @@ -148,13 +147,13 @@ public class DataSourceService extends BaseService{ return result; } - if(!hasPerm(loginUser, dataSource.getUserId())){ + if (!hasPerm(loginUser, dataSource.getUserId())) { putMsg(result, Status.USER_NO_OPERATION_PERM); return result; } //check name can use or not - if(!name.trim().equals(dataSource.getName()) && checkName(name)){ + if (!name.trim().equals(dataSource.getName()) && checkName(name)) { putMsg(result, Status.DATASOURCE_EXIST); return result; } @@ -190,15 +189,13 @@ public class DataSourceService extends BaseService{ private boolean checkName(String name) { List queryDataSource = dataSourceMapper.queryDataSourceByName(name.trim()); - if (queryDataSource != null && queryDataSource.size() > 0) { - return true; - } - return false; + return queryDataSource != null && queryDataSource.size() > 0; } /** * updateProcessInstance datasource + * * @param id datasource id * @return data source detail */ @@ -220,11 +217,11 @@ public class DataSourceService extends BaseService{ String parameter = dataSource.getConnectionParams(); BaseDataSource datasourceForm = DataSourceFactory.getDatasource(dataSource.getType(), parameter); - DbConnectType connectType = null; + DbConnectType connectType = null; String hostSeperator = Constants.DOUBLE_SLASH; - if(DbType.ORACLE.equals(dataSource.getType())){ + if (DbType.ORACLE.equals(dataSource.getType())) { connectType = ((OracleDataSource) datasourceForm).getConnectType(); - if(DbConnectType.ORACLE_SID.equals(connectType)){ + if (DbConnectType.ORACLE_SID.equals(connectType)) { hostSeperator = Constants.AT_SIGN; } } @@ -233,7 +230,7 @@ public class DataSourceService extends BaseService{ String other = datasourceForm.getOther(); String address = datasourceForm.getAddress(); - String[] hostsPorts = getHostsAndPort(address,hostSeperator); + String[] hostsPorts = getHostsAndPort(address, hostSeperator); // ip host String host = hostsPorts[0]; // prot @@ -249,6 +246,7 @@ public class DataSourceService extends BaseService{ case POSTGRESQL: case CLICKHOUSE: case ORACLE: + case PRESTO: separator = "&"; break; default: @@ -284,14 +282,13 @@ public class DataSourceService extends BaseService{ return result; } - /** * query datasource list by keyword * * @param loginUser login user * @param searchVal search value - * @param pageNo page number - * @param pageSize page size + * @param pageNo page number + * @param pageSize page size * @return data source list page */ public Map queryDataSourceListPaging(User loginUser, String searchVal, Integer pageNo, Integer pageSize) { @@ -301,14 +298,14 @@ public class DataSourceService extends BaseService{ if (isAdmin(loginUser)) { dataSourceList = dataSourceMapper.selectPaging(dataSourcePage, 0, searchVal); - }else{ + } else { dataSourceList = dataSourceMapper.selectPaging(dataSourcePage, loginUser.getId(), searchVal); } - List dataSources = dataSourceList.getRecords(); + List dataSources = dataSourceList != null ? dataSourceList.getRecords() : new ArrayList<>(); handlePasswd(dataSources); PageInfo pageInfo = new PageInfo(pageNo, pageSize); - pageInfo.setTotalCount((int)(dataSourceList.getTotal())); + pageInfo.setTotalCount((int) (dataSourceList != null ? dataSourceList.getTotal() : 0L)); pageInfo.setLists(dataSources); result.put(Constants.DATA_LIST, pageInfo); putMsg(result, Status.SUCCESS); @@ -318,14 +315,15 @@ public class DataSourceService extends BaseService{ /** * handle datasource connection password for safety + * * @param dataSourceList */ private void handlePasswd(List dataSourceList) { for (DataSource dataSource : dataSourceList) { - String connectionParams = dataSource.getConnectionParams(); - ObjectNode object = JSONUtils.parseObject(connectionParams); + String connectionParams = dataSource.getConnectionParams(); + ObjectNode object = JSONUtils.parseObject(connectionParams); object.put(Constants.PASSWORD, Constants.XXXXXX); dataSource.setConnectionParams(object.toString()); @@ -336,17 +334,17 @@ public class DataSourceService extends BaseService{ * query data resource list * * @param loginUser login user - * @param type data source type + * @param type data source type * @return data source list page */ public Map queryDataSourceList(User loginUser, Integer type) { - Map result = new HashMap<>(5); + Map result = new HashMap<>(); List datasourceList; if (isAdmin(loginUser)) { datasourceList = dataSourceMapper.listAllDataSourceByType(type); - }else{ + } else { datasourceList = dataSourceMapper.queryDataSourceByType(loginUser.getId(), type); } @@ -359,11 +357,10 @@ public class DataSourceService extends BaseService{ /** * verify datasource exists * - * @param loginUser login user - * @param name datasource name + * @param name datasource name * @return true if data datasource not exists, otherwise return false */ - public Result verifyDataSourceName(User loginUser, String name) { + public Result verifyDataSourceName(String name) { Result result = new Result(); List dataSourceList = dataSourceMapper.queryDataSourceByName(name); if (dataSourceList != null && dataSourceList.size() > 0) { @@ -379,7 +376,7 @@ public class DataSourceService extends BaseService{ /** * get connection * - * @param dbType datasource type + * @param dbType datasource type * @param parameter parameter * @return connection for datasource */ @@ -398,18 +395,18 @@ public class DataSourceService extends BaseService{ break; case HIVE: case SPARK: - if (CommonUtils.getKerberosStartupState()) { - System.setProperty(org.apache.dolphinscheduler.common.Constants.JAVA_SECURITY_KRB5_CONF, - getString(org.apache.dolphinscheduler.common.Constants.JAVA_SECURITY_KRB5_CONF_PATH)); - Configuration configuration = new Configuration(); - configuration.set(org.apache.dolphinscheduler.common.Constants.HADOOP_SECURITY_AUTHENTICATION, "kerberos"); - UserGroupInformation.setConfiguration(configuration); - UserGroupInformation.loginUserFromKeytab(getString(org.apache.dolphinscheduler.common.Constants.LOGIN_USER_KEY_TAB_USERNAME), - getString(org.apache.dolphinscheduler.common.Constants.LOGIN_USER_KEY_TAB_PATH)); + if (CommonUtils.getKerberosStartupState()) { + System.setProperty(org.apache.dolphinscheduler.common.Constants.JAVA_SECURITY_KRB5_CONF, + getString(org.apache.dolphinscheduler.common.Constants.JAVA_SECURITY_KRB5_CONF_PATH)); + Configuration configuration = new Configuration(); + configuration.set(org.apache.dolphinscheduler.common.Constants.HADOOP_SECURITY_AUTHENTICATION, "kerberos"); + UserGroupInformation.setConfiguration(configuration); + UserGroupInformation.loginUserFromKeytab(getString(org.apache.dolphinscheduler.common.Constants.LOGIN_USER_KEY_TAB_USERNAME), + getString(org.apache.dolphinscheduler.common.Constants.LOGIN_USER_KEY_TAB_PATH)); } - if (dbType == DbType.HIVE){ + if (dbType == DbType.HIVE) { datasource = JSONUtils.parseObject(parameter, HiveDataSource.class); - }else if (dbType == DbType.SPARK){ + } else if (dbType == DbType.SPARK) { datasource = JSONUtils.parseObject(parameter, SparkDataSource.class); } Class.forName(Constants.ORG_APACHE_HIVE_JDBC_HIVE_DRIVER); @@ -430,24 +427,27 @@ public class DataSourceService extends BaseService{ datasource = JSONUtils.parseObject(parameter, DB2ServerDataSource.class); Class.forName(Constants.COM_DB2_JDBC_DRIVER); break; + case PRESTO: + datasource = JSONUtils.parseObject(parameter, PrestoDataSource.class); + Class.forName(Constants.COM_PRESTO_JDBC_DRIVER); + break; default: break; } - if(datasource != null){ + if (datasource != null) { connection = DriverManager.getConnection(datasource.getJdbcUrl(), datasource.getUser(), datasource.getPassword()); } } catch (Exception e) { - logger.error(e.getMessage(),e); + logger.error(e.getMessage(), e); } return connection; } - /** * check connection * - * @param type data source type + * @param type data source type * @param parameter data source parameters * @return true if connect successfully, otherwise false */ @@ -465,35 +465,35 @@ public class DataSourceService extends BaseService{ return isConnection; } - /** * test connection * - * @param loginUser login user * @param id datasource id * @return connect result code */ - public boolean connectionTest(User loginUser, int id) { + public boolean connectionTest(int id) { DataSource dataSource = dataSourceMapper.selectById(id); - return checkConnection(dataSource.getType(), dataSource.getConnectionParams()); + if (dataSource != null) { + return checkConnection(dataSource.getType(), dataSource.getConnectionParams()); + } else { + return false; + } } /** * build paramters * - * @param name data source name - * @param desc data source description - * @param type data source type - * @param host data source host - * @param port data source port - * @param database data source database name - * @param userName user name - * @param password password - * @param other other parameters + * @param type data source type + * @param host data source host + * @param port data source port + * @param database data source database name + * @param userName user name + * @param password password + * @param other other parameters * @param principal principal * @return datasource parameter */ - public String buildParameter(String name, String desc, DbType type, String host, + public String buildParameter(DbType type, String host, String port, String database, String principal, String userName, String password, DbConnectType connectType, String other) { @@ -505,7 +505,7 @@ public class DataSourceService extends BaseService{ } if (CommonUtils.getKerberosStartupState() && - (type == DbType.HIVE || type == DbType.SPARK)){ + (type == DbType.HIVE || type == DbType.SPARK)) { jdbcUrl += ";principal=" + principal; } @@ -513,7 +513,8 @@ public class DataSourceService extends BaseService{ if (Constants.MYSQL.equals(type.name()) || Constants.POSTGRESQL.equals(type.name()) || Constants.CLICKHOUSE.equals(type.name()) - || Constants.ORACLE.equals(type.name())) { + || Constants.ORACLE.equals(type.name()) + || Constants.PRESTO.equals(type.name())) { separator = "&"; } else if (Constants.HIVE.equals(type.name()) || Constants.SPARK.equals(type.name()) @@ -529,14 +530,14 @@ public class DataSourceService extends BaseService{ parameterMap.put(Constants.USER, userName); parameterMap.put(Constants.PASSWORD, CommonUtils.encodePassword(password)); if (CommonUtils.getKerberosStartupState() && - (type == DbType.HIVE || type == DbType.SPARK)){ - parameterMap.put(Constants.PRINCIPAL,principal); + (type == DbType.HIVE || type == DbType.SPARK)) { + parameterMap.put(Constants.PRINCIPAL, principal); } if (other != null && !"".equals(other)) { Map map = JSONUtils.toMap(other); if (map.size() > 0) { StringBuilder otherSb = new StringBuilder(); - for (Map.Entry entry: map.entrySet()) { + for (Map.Entry entry : map.entrySet()) { otherSb.append(String.format("%s=%s%s", entry.getKey(), entry.getValue(), separator)); } if (!Constants.DB2.equals(type.name())) { @@ -547,7 +548,7 @@ public class DataSourceService extends BaseService{ } - if(logger.isDebugEnabled()){ + if (logger.isDebugEnabled()) { logger.info("parameters map:{}", JSONUtils.toJsonString(parameterMap)); } return JSONUtils.toJsonString(parameterMap); @@ -585,9 +586,12 @@ public class DataSourceService extends BaseService{ } else if (Constants.SQLSERVER.equals(type.name())) { sb.append(Constants.JDBC_SQLSERVER); sb.append(host).append(":").append(port); - }else if (Constants.DB2.equals(type.name())) { + } else if (Constants.DB2.equals(type.name())) { sb.append(Constants.JDBC_DB2); sb.append(host).append(":").append(port); + } else if (Constants.PRESTO.equals(type.name())) { + sb.append(Constants.JDBC_PRESTO); + sb.append(host).append(":").append(port); } return sb.toString(); @@ -596,7 +600,7 @@ public class DataSourceService extends BaseService{ /** * delete datasource * - * @param loginUser login user + * @param loginUser login user * @param datasourceId data source id * @return delete result code */ @@ -606,12 +610,12 @@ public class DataSourceService extends BaseService{ try { //query datasource by id DataSource dataSource = dataSourceMapper.selectById(datasourceId); - if(dataSource == null){ + if (dataSource == null) { logger.error("resource id {} not exist", datasourceId); putMsg(result, Status.RESOURCE_NOT_EXIST); return result; } - if(!hasPerm(loginUser, dataSource.getUserId())){ + if (!hasPerm(loginUser, dataSource.getUserId())) { putMsg(result, Status.USER_NO_OPERATION_PERM); return result; } @@ -619,7 +623,7 @@ public class DataSourceService extends BaseService{ datasourceUserMapper.deleteByDatasourceId(datasourceId); putMsg(result, Status.SUCCESS); } catch (Exception e) { - logger.error("delete datasource error",e); + logger.error("delete datasource error", e); throw new RuntimeException("delete datasource error"); } return result; @@ -629,7 +633,7 @@ public class DataSourceService extends BaseService{ * unauthorized datasource * * @param loginUser login user - * @param userId user id + * @param userId user id * @return unauthed data source result code */ public Map unauthDatasource(User loginUser, Integer userId) { @@ -670,11 +674,11 @@ public class DataSourceService extends BaseService{ * authorized datasource * * @param loginUser login user - * @param userId user id + * @param userId user id * @return authorized result code */ public Map authedDatasource(User loginUser, Integer userId) { - Map result = new HashMap<>(5); + Map result = new HashMap<>(); if (!isAdmin(loginUser)) { putMsg(result, Status.USER_NO_OPERATION_PERM); @@ -691,11 +695,11 @@ public class DataSourceService extends BaseService{ /** * get host and port by address * - * @param address address + * @param address address * @return sting array: [host,port] */ private String[] getHostsAndPort(String address) { - return getHostsAndPort(address,Constants.DOUBLE_SLASH); + return getHostsAndPort(address, Constants.DOUBLE_SLASH); } /** @@ -705,7 +709,7 @@ public class DataSourceService extends BaseService{ * @param separator separator * @return sting array: [host,port] */ - private String[] getHostsAndPort(String address,String separator) { + private String[] getHostsAndPort(String address, String separator) { String[] result = new String[2]; String[] tmpArray = address.split(separator); String hostsAndPorts = tmpArray[tmpArray.length - 1]; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java index 6a8dad4f2a..fb735ecf19 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java @@ -98,7 +98,7 @@ public class ExecutorService extends BaseService{ TaskDependType taskDependType, WarningType warningType, int warningGroupId, String receivers, String receiversCc, RunMode runMode, Priority processInstancePriority, String workerGroup, Integer timeout) throws ParseException { - Map result = new HashMap<>(5); + Map result = new HashMap<>(); // timeout is invalid if (timeout <= 0 || timeout > MAX_TASK_TIMEOUT) { putMsg(result,Status.TASK_TIMEOUT_PARAMS_ERROR); @@ -176,7 +176,7 @@ public class ExecutorService extends BaseService{ * @return check result code */ public Map checkProcessDefinitionValid(ProcessDefinition processDefinition, int processDefineId){ - Map result = new HashMap<>(5); + Map result = new HashMap<>(); if (processDefinition == null) { // check process definition exists putMsg(result, Status.PROCESS_DEFINE_NOT_EXIST,processDefineId); @@ -201,7 +201,7 @@ public class ExecutorService extends BaseService{ * @return execute result code */ public Map execute(User loginUser, String projectName, Integer processInstanceId, ExecuteType executeType) { - Map result = new HashMap<>(5); + Map result = new HashMap<>(); Project project = projectMapper.queryByName(projectName); Map checkResult = checkResultAndAuth(loginUser, projectName, project); @@ -294,7 +294,7 @@ public class ExecutorService extends BaseService{ */ private Map checkExecuteType(ProcessInstance processInstance, ExecuteType executeType) { - Map result = new HashMap<>(5); + Map result = new HashMap<>(); ExecutionStatus executionStatus = processInstance.getState(); boolean checkResult = false; switch (executeType) { @@ -339,7 +339,7 @@ public class ExecutorService extends BaseService{ * @return update result */ private Map updateProcessInstancePrepare(ProcessInstance processInstance, CommandType commandType, ExecutionStatus executionStatus) { - Map result = new HashMap<>(5); + Map result = new HashMap<>(); processInstance.setCommandType(commandType); processInstance.addHistoryCmd(commandType); @@ -365,7 +365,7 @@ public class ExecutorService extends BaseService{ * @return insert result code */ private Map insertCommand(User loginUser, Integer instanceId, Integer processDefinitionId, CommandType commandType) { - Map result = new HashMap<>(5); + Map result = new HashMap<>(); Command command = new Command(); command.setCommandType(commandType); command.setProcessDefinitionId(processDefinitionId); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/LoggerService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/LoggerService.java index 3c7b421d5e..14440ee61e 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/LoggerService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/LoggerService.java @@ -16,117 +16,30 @@ */ package org.apache.dolphinscheduler.api.service; -import java.nio.charset.StandardCharsets; -import javax.annotation.PreDestroy; -import org.apache.commons.lang.ArrayUtils; -import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.utils.Result; -import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.common.utils.StringUtils; -import org.apache.dolphinscheduler.dao.entity.TaskInstance; -import org.apache.dolphinscheduler.remote.utils.Host; -import org.apache.dolphinscheduler.service.log.LogClientService; -import org.apache.dolphinscheduler.service.process.ProcessService; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; /** * log service */ -@Service -public class LoggerService { +public interface LoggerService { - private static final Logger logger = LoggerFactory.getLogger(LoggerService.class); - - private static final String LOG_HEAD_FORMAT = "[LOG-PATH]: %s, [HOST]: %s%s"; - - @Autowired - private ProcessService processService; - - private final LogClientService logClient; - - public LoggerService() { - logClient = new LogClientService(); - } - - @PreDestroy - public void close() { - logClient.close(); - } - - /** - * view log - * - * @param taskInstId task instance id - * @param skipLineNum skip line number - * @param limit limit - * @return log string data - */ - public Result queryLog(int taskInstId, int skipLineNum, int limit) { - - TaskInstance taskInstance = processService.findTaskInstanceById(taskInstId); - - if (taskInstance == null || StringUtils.isBlank(taskInstance.getHost())) { - return Result.error(Status.TASK_INSTANCE_NOT_FOUND); - } - - String host = getHost(taskInstance.getHost()); - - Result result = new Result(Status.SUCCESS.getCode(), Status.SUCCESS.getMsg()); - - logger.info("log host : {} , logPath : {} , logServer port : {}", host, taskInstance.getLogPath(), - Constants.RPC_PORT); - - StringBuilder log = new StringBuilder(); - if (skipLineNum == 0) { - String head = String.format(LOG_HEAD_FORMAT, - taskInstance.getLogPath(), - host, - Constants.SYSTEM_LINE_SEPARATOR); - log.append(head); - } - - log.append(logClient - .rollViewLog(host, Constants.RPC_PORT, taskInstance.getLogPath(), skipLineNum, limit)); - - result.setData(log); - return result; - } + /** + * view log + * + * @param taskInstId task instance id + * @param skipLineNum skip line number + * @param limit limit + * @return log string data + */ + Result queryLog(int taskInstId, int skipLineNum, int limit); - /** - * get log size - * - * @param taskInstId task instance id - * @return log byte array - */ - public byte[] getLogBytes(int taskInstId) { - TaskInstance taskInstance = processService.findTaskInstanceById(taskInstId); - if (taskInstance == null || StringUtils.isBlank(taskInstance.getHost())) { - throw new RuntimeException("task instance is null or host is null"); - } - String host = getHost(taskInstance.getHost()); - byte[] head = String.format(LOG_HEAD_FORMAT, - taskInstance.getLogPath(), - host, - Constants.SYSTEM_LINE_SEPARATOR).getBytes(StandardCharsets.UTF_8); - return ArrayUtils.addAll(head, - logClient.getLogBytes(host, Constants.RPC_PORT, taskInstance.getLogPath())); - } + /** + * get log size + * + * @param taskInstId task instance id + * @return log byte array + */ + byte[] getLogBytes(int taskInstId); - - /** - * get host - * - * @param address address - * @return old version return true ,otherwise return false - */ - private String getHost(String address) { - if (Host.isOldVersion(address)) { - return address; - } - return Host.of(address).getIp(); - } } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/MonitorService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/MonitorService.java index 55c4fa113b..e46ca6fcf2 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/MonitorService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/MonitorService.java @@ -56,7 +56,7 @@ public class MonitorService extends BaseService { * @return data base state */ public Map queryDatabaseState(User loginUser) { - Map result = new HashMap<>(5); + Map result = new HashMap<>(); List monitorRecordList = monitorDBDao.queryDatabaseState(); @@ -75,7 +75,7 @@ public class MonitorService extends BaseService { */ public Map queryMaster(User loginUser) { - Map result = new HashMap<>(5); + Map result = new HashMap<>(); List masterServers = getServerListFromZK(true); result.put(Constants.DATA_LIST, masterServers); @@ -91,7 +91,7 @@ public class MonitorService extends BaseService { * @return zookeeper information list */ public Map queryZookeeperState(User loginUser) { - Map result = new HashMap<>(5); + Map result = new HashMap<>(); List zookeeperRecordList = zookeeperMonitor.zookeeperInfoList(); @@ -111,7 +111,7 @@ public class MonitorService extends BaseService { */ public Map queryWorker(User loginUser) { - Map result = new HashMap<>(5); + Map result = new HashMap<>(); List workerServers = getServerListFromZK(false) .stream() .map((Server server) -> { diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java index b3d56e5982..f6f786b6b1 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java @@ -14,1147 +14,205 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.api.service; -import com.baomidou.mybatisplus.core.metadata.IPage; -import com.baomidou.mybatisplus.extension.plugins.pagination.Page; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ArrayNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import org.apache.dolphinscheduler.api.dto.ProcessMeta; -import org.apache.dolphinscheduler.api.dto.treeview.Instance; -import org.apache.dolphinscheduler.api.dto.treeview.TreeViewDto; -import org.apache.dolphinscheduler.api.enums.Status; -import org.apache.dolphinscheduler.api.utils.CheckUtils; -import org.apache.dolphinscheduler.api.utils.FileUtils; -import org.apache.dolphinscheduler.api.utils.PageInfo; -import org.apache.dolphinscheduler.api.utils.exportprocess.ProcessAddTaskParam; -import org.apache.dolphinscheduler.api.utils.exportprocess.TaskNodeParamFactory; -import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.common.enums.*; -import org.apache.dolphinscheduler.common.graph.DAG; -import org.apache.dolphinscheduler.common.model.TaskNode; -import org.apache.dolphinscheduler.common.model.TaskNodeRelation; -import org.apache.dolphinscheduler.common.process.ProcessDag; -import org.apache.dolphinscheduler.common.process.Property; -import org.apache.dolphinscheduler.common.task.AbstractParameters; -import org.apache.dolphinscheduler.common.thread.Stopper; -import org.apache.dolphinscheduler.common.utils.*; -import org.apache.dolphinscheduler.dao.entity.*; -import org.apache.dolphinscheduler.dao.mapper.*; -import org.apache.dolphinscheduler.dao.utils.DagHelper; -import org.apache.dolphinscheduler.service.permission.PermissionCheck; -import org.apache.dolphinscheduler.service.process.ProcessService; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.http.MediaType; -import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Transactional; +import org.apache.dolphinscheduler.dao.entity.ProcessData; +import org.apache.dolphinscheduler.dao.entity.User; + +import java.util.Map; + +import javax.servlet.http.HttpServletResponse; + import org.springframework.web.multipart.MultipartFile; -import javax.servlet.ServletOutputStream; -import javax.servlet.http.HttpServletResponse; -import java.io.BufferedOutputStream; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.util.*; -import java.util.concurrent.ConcurrentHashMap; -import java.util.stream.Collectors; - -import static org.apache.dolphinscheduler.common.Constants.CMDPARAM_SUB_PROCESS_DEFINE_ID; +import com.fasterxml.jackson.core.JsonProcessingException; /** * process definition service */ -@Service -public class ProcessDefinitionService extends BaseDAGService { - - private static final Logger logger = LoggerFactory.getLogger(ProcessDefinitionService.class); - - private static final String PROCESSDEFINITIONID = "processDefinitionId"; - - private static final String RELEASESTATE = "releaseState"; - - private static final String TASKS = "tasks"; - - @Autowired - private ProjectMapper projectMapper; - - @Autowired - private ProjectService projectService; - - @Autowired - private ProcessDefinitionMapper processDefineMapper; - - @Autowired - private ProcessInstanceMapper processInstanceMapper; - - - @Autowired - private TaskInstanceMapper taskInstanceMapper; - - @Autowired - private ScheduleMapper scheduleMapper; - - @Autowired - private ProcessService processService; +public interface ProcessDefinitionService { /** * create process definition * - * @param loginUser login user - * @param projectName project name - * @param name process definition name + * @param loginUser login user + * @param projectName project name + * @param name process definition name * @param processDefinitionJson process definition json - * @param desc description - * @param locations locations for nodes - * @param connects connects for nodes + * @param desc description + * @param locations locations for nodes + * @param connects connects for nodes * @return create result code * @throws JsonProcessingException JsonProcessingException */ - public Map createProcessDefinition(User loginUser, - String projectName, - String name, - String processDefinitionJson, - String desc, - String locations, - String connects) throws JsonProcessingException { - - Map result = new HashMap<>(5); - Project project = projectMapper.queryByName(projectName); - // check project auth - Map checkResult = projectService.checkProjectAndAuth(loginUser, project, projectName); - Status resultStatus = (Status) checkResult.get(Constants.STATUS); - if (resultStatus != Status.SUCCESS) { - return checkResult; - } - - ProcessDefinition processDefine = new ProcessDefinition(); - Date now = new Date(); - - ProcessData processData = JSONUtils.parseObject(processDefinitionJson, ProcessData.class); - Map checkProcessJson = checkProcessNodeList(processData, processDefinitionJson); - if (checkProcessJson.get(Constants.STATUS) != Status.SUCCESS) { - return checkProcessJson; - } - - processDefine.setName(name); - processDefine.setReleaseState(ReleaseState.OFFLINE); - processDefine.setProjectId(project.getId()); - processDefine.setUserId(loginUser.getId()); - processDefine.setProcessDefinitionJson(processDefinitionJson); - processDefine.setDescription(desc); - processDefine.setLocations(locations); - processDefine.setConnects(connects); - processDefine.setTimeout(processData.getTimeout()); - processDefine.setTenantId(processData.getTenantId()); - processDefine.setModifyBy(loginUser.getUserName()); - processDefine.setResourceIds(getResourceIds(processData)); - - //custom global params - List globalParamsList = processData.getGlobalParams(); - if (CollectionUtils.isNotEmpty(globalParamsList)) { - Set globalParamsSet = new HashSet<>(globalParamsList); - globalParamsList = new ArrayList<>(globalParamsSet); - processDefine.setGlobalParamList(globalParamsList); - } - processDefine.setCreateTime(now); - processDefine.setUpdateTime(now); - processDefine.setFlag(Flag.YES); - processDefineMapper.insert(processDefine); - - // return processDefinition object with ID - result.put(Constants.DATA_LIST, processDefineMapper.selectById(processDefine.getId())); - putMsg(result, Status.SUCCESS); - result.put("processDefinitionId", processDefine.getId()); - return result; - } - - /** - * get resource ids - * - * @param processData process data - * @return resource ids - */ - private String getResourceIds(ProcessData processData) { - List tasks = processData.getTasks(); - Set resourceIds = new HashSet<>(); - for (TaskNode taskNode : tasks) { - String taskParameter = taskNode.getParams(); - AbstractParameters params = TaskParametersUtils.getParameters(taskNode.getType(), taskParameter); - if (CollectionUtils.isNotEmpty(params.getResourceFilesList())) { - Set tempSet = params.getResourceFilesList().stream().map(t -> t.getId()).collect(Collectors.toSet()); - resourceIds.addAll(tempSet); - } - } - - StringBuilder sb = new StringBuilder(); - for (int i : resourceIds) { - if (sb.length() > 0) { - sb.append(","); - } - sb.append(i); - } - return sb.toString(); - } - + Map createProcessDefinition(User loginUser, + String projectName, + String name, + String processDefinitionJson, + String desc, + String locations, + String connects) throws JsonProcessingException; /** * query process definition list * - * @param loginUser login user + * @param loginUser login user * @param projectName project name * @return definition list */ - public Map queryProcessDefinitionList(User loginUser, String projectName) { - - HashMap result = new HashMap<>(5); - Project project = projectMapper.queryByName(projectName); - - Map checkResult = projectService.checkProjectAndAuth(loginUser, project, projectName); - Status resultStatus = (Status) checkResult.get(Constants.STATUS); - if (resultStatus != Status.SUCCESS) { - return checkResult; - } - - List resourceList = processDefineMapper.queryAllDefinitionList(project.getId()); - result.put(Constants.DATA_LIST, resourceList); - putMsg(result, Status.SUCCESS); - - return result; - } - + Map queryProcessDefinitionList(User loginUser, + String projectName); /** * query process definition list paging * - * @param loginUser login user + * @param loginUser login user * @param projectName project name - * @param searchVal search value - * @param pageNo page number - * @param pageSize page size - * @param userId user id + * @param searchVal search value + * @param pageNo page number + * @param pageSize page size + * @param userId user id * @return process definition page */ - public Map queryProcessDefinitionListPaging(User loginUser, String projectName, String searchVal, Integer pageNo, Integer pageSize, Integer userId) { - - Map result = new HashMap<>(5); - Project project = projectMapper.queryByName(projectName); - - Map checkResult = projectService.checkProjectAndAuth(loginUser, project, projectName); - Status resultStatus = (Status) checkResult.get(Constants.STATUS); - if (resultStatus != Status.SUCCESS) { - return checkResult; - } - - Page page = new Page(pageNo, pageSize); - IPage processDefinitionIPage = processDefineMapper.queryDefineListPaging( - page, searchVal, userId, project.getId(), isAdmin(loginUser)); - - PageInfo pageInfo = new PageInfo(pageNo, pageSize); - pageInfo.setTotalCount((int) processDefinitionIPage.getTotal()); - pageInfo.setLists(processDefinitionIPage.getRecords()); - result.put(Constants.DATA_LIST, pageInfo); - putMsg(result, Status.SUCCESS); - - return result; - } + Map queryProcessDefinitionListPaging(User loginUser, + String projectName, + String searchVal, + Integer pageNo, + Integer pageSize, + Integer userId); /** * query datail of process definition * - * @param loginUser login user + * @param loginUser login user * @param projectName project name - * @param processId process definition id + * @param processId process definition id * @return process definition detail */ - public Map queryProcessDefinitionById(User loginUser, String projectName, Integer processId) { - - Map result = new HashMap<>(5); - Project project = projectMapper.queryByName(projectName); - - Map checkResult = projectService.checkProjectAndAuth(loginUser, project, projectName); - Status resultStatus = (Status) checkResult.get(Constants.STATUS); - if (resultStatus != Status.SUCCESS) { - return checkResult; - } - - ProcessDefinition processDefinition = processDefineMapper.selectById(processId); - if (processDefinition == null) { - putMsg(result, Status.PROCESS_INSTANCE_NOT_EXIST, processId); - } else { - result.put(Constants.DATA_LIST, processDefinition); - putMsg(result, Status.SUCCESS); - } - return result; - } + Map queryProcessDefinitionById(User loginUser, + String projectName, + Integer processId); /** - * copy process definition + * batch copy process definition * - * @param loginUser login user - * @param projectName project name - * @param processId process definition id - * @return copy result code + * @param loginUser loginUser + * @param projectName projectName + * @param processDefinitionIds processDefinitionIds + * @param targetProjectId targetProjectId */ - public Map copyProcessDefinition(User loginUser, String projectName, Integer processId) throws JsonProcessingException { + Map batchCopyProcessDefinition(User loginUser, + String projectName, + String processDefinitionIds, + int targetProjectId); - Map result = new HashMap<>(5); - Project project = projectMapper.queryByName(projectName); - - Map checkResult = projectService.checkProjectAndAuth(loginUser, project, projectName); - Status resultStatus = (Status) checkResult.get(Constants.STATUS); - if (resultStatus != Status.SUCCESS) { - return checkResult; - } - - ProcessDefinition processDefinition = processDefineMapper.selectById(processId); - if (processDefinition == null) { - putMsg(result, Status.PROCESS_DEFINE_NOT_EXIST, processId); - return result; - } else { - return createProcessDefinition( - loginUser, - projectName, - processDefinition.getName() + "_copy_" + System.currentTimeMillis(), - processDefinition.getProcessDefinitionJson(), - processDefinition.getDescription(), - processDefinition.getLocations(), - processDefinition.getConnects()); - } - } + /** + * batch move process definition + * + * @param loginUser loginUser + * @param projectName projectName + * @param processDefinitionIds processDefinitionIds + * @param targetProjectId targetProjectId + */ + Map batchMoveProcessDefinition(User loginUser, + String projectName, + String processDefinitionIds, + int targetProjectId); /** * update process definition * - * @param loginUser login user - * @param projectName project name - * @param name process definition name - * @param id process definition id + * @param loginUser login user + * @param projectName project name + * @param name process definition name + * @param id process definition id * @param processDefinitionJson process definition json - * @param desc description - * @param locations locations for nodes - * @param connects connects for nodes + * @param desc description + * @param locations locations for nodes + * @param connects connects for nodes * @return update result code */ - public Map updateProcessDefinition(User loginUser, String projectName, int id, String name, - String processDefinitionJson, String desc, - String locations, String connects) { - Map result = new HashMap<>(5); - - Project project = projectMapper.queryByName(projectName); - Map checkResult = projectService.checkProjectAndAuth(loginUser, project, projectName); - Status resultStatus = (Status) checkResult.get(Constants.STATUS); - if (resultStatus != Status.SUCCESS) { - return checkResult; - } - - ProcessData processData = JSONUtils.parseObject(processDefinitionJson, ProcessData.class); - Map checkProcessJson = checkProcessNodeList(processData, processDefinitionJson); - if ((checkProcessJson.get(Constants.STATUS) != Status.SUCCESS)) { - return checkProcessJson; - } - ProcessDefinition processDefine = processService.findProcessDefineById(id); - if (processDefine == null) { - // check process definition exists - putMsg(result, Status.PROCESS_DEFINE_NOT_EXIST, id); - return result; - } else if (processDefine.getReleaseState() == ReleaseState.ONLINE) { - // online can not permit edit - putMsg(result, Status.PROCESS_DEFINE_NOT_ALLOWED_EDIT, processDefine.getName()); - return result; - } else { - putMsg(result, Status.SUCCESS); - } - - Date now = new Date(); - - processDefine.setId(id); - processDefine.setName(name); - processDefine.setReleaseState(ReleaseState.OFFLINE); - processDefine.setProjectId(project.getId()); - processDefine.setProcessDefinitionJson(processDefinitionJson); - processDefine.setDescription(desc); - processDefine.setLocations(locations); - processDefine.setConnects(connects); - processDefine.setTimeout(processData.getTimeout()); - processDefine.setTenantId(processData.getTenantId()); - processDefine.setModifyBy(loginUser.getUserName()); - processDefine.setResourceIds(getResourceIds(processData)); - - //custom global params - List globalParamsList = new ArrayList<>(); - if (CollectionUtils.isNotEmpty(processData.getGlobalParams())) { - Set userDefParamsSet = new HashSet<>(processData.getGlobalParams()); - globalParamsList = new ArrayList<>(userDefParamsSet); - } - processDefine.setGlobalParamList(globalParamsList); - processDefine.setUpdateTime(now); - processDefine.setFlag(Flag.YES); - if (processDefineMapper.updateById(processDefine) > 0) { - putMsg(result, Status.SUCCESS); - - } else { - putMsg(result, Status.UPDATE_PROCESS_DEFINITION_ERROR); - } - return result; - } + Map updateProcessDefinition(User loginUser, + String projectName, + int id, + String name, + String processDefinitionJson, String desc, + String locations, String connects); /** * verify process definition name unique * - * @param loginUser login user + * @param loginUser login user * @param projectName project name - * @param name name + * @param name name * @return true if process definition name not exists, otherwise false */ - public Map verifyProcessDefinitionName(User loginUser, String projectName, String name) { - - Map result = new HashMap<>(); - Project project = projectMapper.queryByName(projectName); - - Map checkResult = projectService.checkProjectAndAuth(loginUser, project, projectName); - Status resultEnum = (Status) checkResult.get(Constants.STATUS); - if (resultEnum != Status.SUCCESS) { - return checkResult; - } - ProcessDefinition processDefinition = processDefineMapper.queryByDefineName(project.getId(), name); - if (processDefinition == null) { - putMsg(result, Status.SUCCESS); - } else { - putMsg(result, Status.PROCESS_INSTANCE_EXIST, name); - } - return result; - } + Map verifyProcessDefinitionName(User loginUser, + String projectName, + String name); /** * delete process definition by id * - * @param loginUser login user - * @param projectName project name + * @param loginUser login user + * @param projectName project name * @param processDefinitionId process definition id * @return delete result code */ - @Transactional(rollbackFor = RuntimeException.class) - public Map deleteProcessDefinitionById(User loginUser, String projectName, Integer processDefinitionId) { - - Map result = new HashMap<>(5); - Project project = projectMapper.queryByName(projectName); - - Map checkResult = projectService.checkProjectAndAuth(loginUser, project, projectName); - Status resultEnum = (Status) checkResult.get(Constants.STATUS); - if (resultEnum != Status.SUCCESS) { - return checkResult; - } - - ProcessDefinition processDefinition = processDefineMapper.selectById(processDefinitionId); - - if (processDefinition == null) { - putMsg(result, Status.PROCESS_DEFINE_NOT_EXIST, processDefinitionId); - return result; - } - - // Determine if the login user is the owner of the process definition - if (loginUser.getId() != processDefinition.getUserId() && loginUser.getUserType() != UserType.ADMIN_USER) { - putMsg(result, Status.USER_NO_OPERATION_PERM); - return result; - } - - // check process definition is already online - if (processDefinition.getReleaseState() == ReleaseState.ONLINE) { - putMsg(result, Status.PROCESS_DEFINE_STATE_ONLINE, processDefinitionId); - return result; - } - - // get the timing according to the process definition - List schedules = scheduleMapper.queryByProcessDefinitionId(processDefinitionId); - if (!schedules.isEmpty() && schedules.size() > 1) { - logger.warn("scheduler num is {},Greater than 1", schedules.size()); - putMsg(result, Status.DELETE_PROCESS_DEFINE_BY_ID_ERROR); - return result; - } else if (schedules.size() == 1) { - Schedule schedule = schedules.get(0); - if (schedule.getReleaseState() == ReleaseState.OFFLINE) { - scheduleMapper.deleteById(schedule.getId()); - } else if (schedule.getReleaseState() == ReleaseState.ONLINE) { - putMsg(result, Status.SCHEDULE_CRON_STATE_ONLINE, schedule.getId()); - return result; - } - } - - int delete = processDefineMapper.deleteById(processDefinitionId); - - if (delete > 0) { - putMsg(result, Status.SUCCESS); - } else { - putMsg(result, Status.DELETE_PROCESS_DEFINE_BY_ID_ERROR); - } - return result; - } + Map deleteProcessDefinitionById(User loginUser, + String projectName, + Integer processDefinitionId); /** * release process definition: online / offline * - * @param loginUser login user - * @param projectName project name - * @param id process definition id + * @param loginUser login user + * @param projectName project name + * @param id process definition id * @param releaseState release state * @return release result code */ - @Transactional(rollbackFor = RuntimeException.class) - public Map releaseProcessDefinition(User loginUser, String projectName, int id, int releaseState) { - HashMap result = new HashMap<>(); - Project project = projectMapper.queryByName(projectName); - - Map checkResult = projectService.checkProjectAndAuth(loginUser, project, projectName); - Status resultEnum = (Status) checkResult.get(Constants.STATUS); - if (resultEnum != Status.SUCCESS) { - return checkResult; - } - - ReleaseState state = ReleaseState.getEnum(releaseState); - - // check state - if (null == state) { - putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, RELEASESTATE); - return result; - } - - ProcessDefinition processDefinition = processDefineMapper.selectById(id); - - switch (state) { - case ONLINE: - // To check resources whether they are already cancel authorized or deleted - String resourceIds = processDefinition.getResourceIds(); - if (StringUtils.isNotBlank(resourceIds)) { - Integer[] resourceIdArray = Arrays.stream(resourceIds.split(",")).map(Integer::parseInt).toArray(Integer[]::new); - PermissionCheck permissionCheck = new PermissionCheck<>(AuthorizationType.RESOURCE_FILE_ID, processService, resourceIdArray, loginUser.getId(), logger); - try { - permissionCheck.checkPermission(); - } catch (Exception e) { - logger.error(e.getMessage(), e); - putMsg(result, Status.RESOURCE_NOT_EXIST_OR_NO_PERMISSION, RELEASESTATE); - return result; - } - } - - processDefinition.setReleaseState(state); - processDefineMapper.updateById(processDefinition); - break; - case OFFLINE: - processDefinition.setReleaseState(state); - processDefineMapper.updateById(processDefinition); - List scheduleList = scheduleMapper.selectAllByProcessDefineArray( - new int[]{processDefinition.getId()} - ); - - for (Schedule schedule : scheduleList) { - logger.info("set schedule offline, project id: {}, schedule id: {}, process definition id: {}", project.getId(), schedule.getId(), id); - // set status - schedule.setReleaseState(ReleaseState.OFFLINE); - scheduleMapper.updateById(schedule); - SchedulerService.deleteSchedule(project.getId(), schedule.getId()); - } - break; - default: - putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, RELEASESTATE); - return result; - } - - putMsg(result, Status.SUCCESS); - return result; - } + Map releaseProcessDefinition(User loginUser, + String projectName, + int id, + int releaseState); /** * batch export process definition by ids * - * @param loginUser - * @param projectName - * @param processDefinitionIds - * @param response + * @param loginUser login user + * @param projectName project name + * @param processDefinitionIds process definition ids + * @param response http servlet response */ - public void batchExportProcessDefinitionByIds(User loginUser, String projectName, String processDefinitionIds, HttpServletResponse response) { - - if (StringUtils.isEmpty(processDefinitionIds)) { - return; - } - - //export project info - Project project = projectMapper.queryByName(projectName); - - //check user access for project - Map checkResult = projectService.checkProjectAndAuth(loginUser, project, projectName); - Status resultStatus = (Status) checkResult.get(Constants.STATUS); - - if (resultStatus != Status.SUCCESS) { - return; - } - - List processDefinitionList = - getProcessDefinitionList(processDefinitionIds); - - if (CollectionUtils.isNotEmpty(processDefinitionList)) { - downloadProcessDefinitionFile(response, processDefinitionList); - } - } - - /** - * get process definition list by ids - * - * @param processDefinitionIds - * @return - */ - private List getProcessDefinitionList(String processDefinitionIds) { - List processDefinitionList = new ArrayList<>(); - String[] processDefinitionIdArray = processDefinitionIds.split(","); - for (String strProcessDefinitionId : processDefinitionIdArray) { - //get workflow info - int processDefinitionId = Integer.parseInt(strProcessDefinitionId); - ProcessDefinition processDefinition = processDefineMapper.queryByDefineId(processDefinitionId); - if (null != processDefinition) { - processDefinitionList.add(exportProcessMetaData(processDefinitionId, processDefinition)); - } - } - - return processDefinitionList; - } - - /** - * download the process definition file - * - * @param response - * @param processDefinitionList - */ - private void downloadProcessDefinitionFile(HttpServletResponse response, List processDefinitionList) { - response.setContentType(MediaType.APPLICATION_JSON_UTF8_VALUE); - BufferedOutputStream buff = null; - ServletOutputStream out = null; - try { - out = response.getOutputStream(); - buff = new BufferedOutputStream(out); - buff.write(JSONUtils.toJsonString(processDefinitionList).getBytes(StandardCharsets.UTF_8)); - buff.flush(); - buff.close(); - } catch (IOException e) { - logger.warn("export process fail", e); - } finally { - if (null != buff) { - try { - buff.close(); - } catch (Exception e) { - logger.warn("export process buffer not close", e); - } - } - if (null != out) { - try { - out.close(); - } catch (Exception e) { - logger.warn("export process output stream not close", e); - } - } - } - } - - /** - * get export process metadata string - * - * @param processDefinitionId process definition id - * @param processDefinition process definition - * @return export process metadata string - */ - public String exportProcessMetaDataStr(Integer processDefinitionId, ProcessDefinition processDefinition) { - //create workflow json file - return JSONUtils.toJsonString(exportProcessMetaData(processDefinitionId, processDefinition)); - } - - /** - * get export process metadata string - * - * @param processDefinitionId process definition id - * @param processDefinition process definition - * @return export process metadata string - */ - public ProcessMeta exportProcessMetaData(Integer processDefinitionId, ProcessDefinition processDefinition) { - //correct task param which has data source or dependent param - String correctProcessDefinitionJson = addExportTaskNodeSpecialParam(processDefinition.getProcessDefinitionJson()); - processDefinition.setProcessDefinitionJson(correctProcessDefinitionJson); - - //export process metadata - ProcessMeta exportProcessMeta = new ProcessMeta(); - exportProcessMeta.setProjectName(processDefinition.getProjectName()); - exportProcessMeta.setProcessDefinitionName(processDefinition.getName()); - exportProcessMeta.setProcessDefinitionJson(processDefinition.getProcessDefinitionJson()); - exportProcessMeta.setProcessDefinitionLocations(processDefinition.getLocations()); - exportProcessMeta.setProcessDefinitionConnects(processDefinition.getConnects()); - - //schedule info - List schedules = scheduleMapper.queryByProcessDefinitionId(processDefinitionId); - if (!schedules.isEmpty()) { - Schedule schedule = schedules.get(0); - exportProcessMeta.setScheduleWarningType(schedule.getWarningType().toString()); - exportProcessMeta.setScheduleWarningGroupId(schedule.getWarningGroupId()); - exportProcessMeta.setScheduleStartTime(DateUtils.dateToString(schedule.getStartTime())); - exportProcessMeta.setScheduleEndTime(DateUtils.dateToString(schedule.getEndTime())); - exportProcessMeta.setScheduleCrontab(schedule.getCrontab()); - exportProcessMeta.setScheduleFailureStrategy(String.valueOf(schedule.getFailureStrategy())); - exportProcessMeta.setScheduleReleaseState(String.valueOf(ReleaseState.OFFLINE)); - exportProcessMeta.setScheduleProcessInstancePriority(String.valueOf(schedule.getProcessInstancePriority())); - exportProcessMeta.setScheduleWorkerGroupName(schedule.getWorkerGroup()); - } - //create workflow json file - return exportProcessMeta; - } - - /** - * correct task param which has datasource or dependent - * - * @param processDefinitionJson processDefinitionJson - * @return correct processDefinitionJson - */ - public String addExportTaskNodeSpecialParam(String processDefinitionJson) { - ObjectNode jsonObject = JSONUtils.parseObject(processDefinitionJson); - ArrayNode jsonArray = (ArrayNode) jsonObject.path(TASKS); - - for (int i = 0; i < jsonArray.size(); i++) { - JsonNode taskNode = jsonArray.path(i); - if (StringUtils.isNotEmpty(taskNode.path("type").asText())) { - String taskType = taskNode.path("type").asText(); - - ProcessAddTaskParam addTaskParam = TaskNodeParamFactory.getByTaskType(taskType); - if (null != addTaskParam) { - addTaskParam.addExportSpecialParam(taskNode); - } - } - } - jsonObject.set(TASKS, jsonArray); - return jsonObject.toString(); - } - - /** - * check task if has sub process - * - * @param taskType task type - * @return if task has sub process return true else false - */ - private boolean checkTaskHasSubProcess(String taskType) { - return taskType.equals(TaskType.SUB_PROCESS.name()); - } + void batchExportProcessDefinitionByIds(User loginUser, + String projectName, + String processDefinitionIds, + HttpServletResponse response); /** * import process definition * - * @param loginUser login user - * @param file process metadata json file + * @param loginUser login user + * @param file process metadata json file * @param currentProjectName current project name * @return import process */ - @Transactional(rollbackFor = RuntimeException.class) - public Map importProcessDefinition(User loginUser, MultipartFile file, String currentProjectName) { - Map result = new HashMap<>(5); - String processMetaJson = FileUtils.file2String(file); - List processMetaList = JSONUtils.toList(processMetaJson, ProcessMeta.class); - - //check file content - if (CollectionUtils.isEmpty(processMetaList)) { - putMsg(result, Status.DATA_IS_NULL, "fileContent"); - return result; - } - - for (ProcessMeta processMeta : processMetaList) { - - if (!checkAndImportProcessDefinition(loginUser, currentProjectName, result, processMeta)) { - return result; - } - } - - return result; - } - - /** - * check and import process definition - * - * @param loginUser - * @param currentProjectName - * @param result - * @param processMeta - * @return - */ - private boolean checkAndImportProcessDefinition(User loginUser, String currentProjectName, Map result, ProcessMeta processMeta) { - - if (!checkImportanceParams(processMeta, result)) { - return false; - } - - //deal with process name - String processDefinitionName = processMeta.getProcessDefinitionName(); - //use currentProjectName to query - Project targetProject = projectMapper.queryByName(currentProjectName); - if (null != targetProject) { - processDefinitionName = recursionProcessDefinitionName(targetProject.getId(), - processDefinitionName, 1); - } - - //unique check - Map checkResult = verifyProcessDefinitionName(loginUser, currentProjectName, processDefinitionName); - Status status = (Status) checkResult.get(Constants.STATUS); - if (Status.SUCCESS.equals(status)) { - putMsg(result, Status.SUCCESS); - } else { - result.putAll(checkResult); - return false; - } - - // get create process result - Map createProcessResult = - getCreateProcessResult(loginUser, - currentProjectName, - result, - processMeta, - processDefinitionName, - addImportTaskNodeParam(loginUser, processMeta.getProcessDefinitionJson(), targetProject)); - - if (createProcessResult == null) { - return false; - } - - //create process definition - Integer processDefinitionId = - Objects.isNull(createProcessResult.get(PROCESSDEFINITIONID)) ? - null : Integer.parseInt(createProcessResult.get(PROCESSDEFINITIONID).toString()); - - //scheduler param - return getImportProcessScheduleResult(loginUser, - currentProjectName, - result, - processMeta, - processDefinitionName, - processDefinitionId); - - } - - /** - * get create process result - * - * @param loginUser - * @param currentProjectName - * @param result - * @param processMeta - * @param processDefinitionName - * @param importProcessParam - * @return - */ - private Map getCreateProcessResult(User loginUser, - String currentProjectName, - Map result, - ProcessMeta processMeta, - String processDefinitionName, - String importProcessParam) { - Map createProcessResult = null; - try { - createProcessResult = createProcessDefinition(loginUser - , currentProjectName, - processDefinitionName + "_import_" + System.currentTimeMillis(), - importProcessParam, - processMeta.getProcessDefinitionDescription(), - processMeta.getProcessDefinitionLocations(), - processMeta.getProcessDefinitionConnects()); - putMsg(result, Status.SUCCESS); - } catch (JsonProcessingException e) { - logger.error("import process meta json data: {}", e.getMessage(), e); - putMsg(result, Status.IMPORT_PROCESS_DEFINE_ERROR); - } - - return createProcessResult; - } - - /** - * get import process schedule result - * - * @param loginUser - * @param currentProjectName - * @param result - * @param processMeta - * @param processDefinitionName - * @param processDefinitionId - * @return - */ - private boolean getImportProcessScheduleResult(User loginUser, - String currentProjectName, - Map result, - ProcessMeta processMeta, - String processDefinitionName, - Integer processDefinitionId) { - if (null != processMeta.getScheduleCrontab() && null != processDefinitionId) { - int scheduleInsert = importProcessSchedule(loginUser, - currentProjectName, - processMeta, - processDefinitionName, - processDefinitionId); - - if (0 == scheduleInsert) { - putMsg(result, Status.IMPORT_PROCESS_DEFINE_ERROR); - return false; - } - } - return true; - } - - /** - * check importance params - * - * @param processMeta - * @param result - * @return - */ - private boolean checkImportanceParams(ProcessMeta processMeta, Map result) { - if (StringUtils.isEmpty(processMeta.getProjectName())) { - putMsg(result, Status.DATA_IS_NULL, "projectName"); - return false; - } - if (StringUtils.isEmpty(processMeta.getProcessDefinitionName())) { - putMsg(result, Status.DATA_IS_NULL, "processDefinitionName"); - return false; - } - if (StringUtils.isEmpty(processMeta.getProcessDefinitionJson())) { - putMsg(result, Status.DATA_IS_NULL, "processDefinitionJson"); - return false; - } - - return true; - } - - /** - * import process add special task param - * - * @param loginUser login user - * @param processDefinitionJson process definition json - * @param targetProject target project - * @return import process param - */ - private String addImportTaskNodeParam(User loginUser, String processDefinitionJson, Project targetProject) { - ObjectNode jsonObject = JSONUtils.parseObject(processDefinitionJson); - ArrayNode jsonArray = (ArrayNode) jsonObject.get(TASKS); - //add sql and dependent param - for (int i = 0; i < jsonArray.size(); i++) { - JsonNode taskNode = jsonArray.path(i); - String taskType = taskNode.path("type").asText(); - ProcessAddTaskParam addTaskParam = TaskNodeParamFactory.getByTaskType(taskType); - if (null != addTaskParam) { - addTaskParam.addImportSpecialParam(taskNode); - } - } - - //recursive sub-process parameter correction map key for old process id value for new process id - Map subProcessIdMap = new HashMap<>(20); - - List subProcessList = StreamUtils.asStream(jsonArray.elements()) - .filter(elem -> checkTaskHasSubProcess(JSONUtils.parseObject(elem.toString()).path("type").asText())) - .collect(Collectors.toList()); - - if (CollectionUtils.isNotEmpty(subProcessList)) { - importSubProcess(loginUser, targetProject, jsonArray, subProcessIdMap); - } - - jsonObject.set(TASKS, jsonArray); - return jsonObject.toString(); - } - - /** - * import process schedule - * - * @param loginUser login user - * @param currentProjectName current project name - * @param processMeta process meta data - * @param processDefinitionName process definition name - * @param processDefinitionId process definition id - * @return insert schedule flag - */ - public int importProcessSchedule(User loginUser, String currentProjectName, ProcessMeta processMeta, - String processDefinitionName, Integer processDefinitionId) { - Date now = new Date(); - Schedule scheduleObj = new Schedule(); - scheduleObj.setProjectName(currentProjectName); - scheduleObj.setProcessDefinitionId(processDefinitionId); - scheduleObj.setProcessDefinitionName(processDefinitionName); - scheduleObj.setCreateTime(now); - scheduleObj.setUpdateTime(now); - scheduleObj.setUserId(loginUser.getId()); - scheduleObj.setUserName(loginUser.getUserName()); - - scheduleObj.setCrontab(processMeta.getScheduleCrontab()); - - if (null != processMeta.getScheduleStartTime()) { - scheduleObj.setStartTime(DateUtils.stringToDate(processMeta.getScheduleStartTime())); - } - if (null != processMeta.getScheduleEndTime()) { - scheduleObj.setEndTime(DateUtils.stringToDate(processMeta.getScheduleEndTime())); - } - if (null != processMeta.getScheduleWarningType()) { - scheduleObj.setWarningType(WarningType.valueOf(processMeta.getScheduleWarningType())); - } - if (null != processMeta.getScheduleWarningGroupId()) { - scheduleObj.setWarningGroupId(processMeta.getScheduleWarningGroupId()); - } - if (null != processMeta.getScheduleFailureStrategy()) { - scheduleObj.setFailureStrategy(FailureStrategy.valueOf(processMeta.getScheduleFailureStrategy())); - } - if (null != processMeta.getScheduleReleaseState()) { - scheduleObj.setReleaseState(ReleaseState.valueOf(processMeta.getScheduleReleaseState())); - } - if (null != processMeta.getScheduleProcessInstancePriority()) { - scheduleObj.setProcessInstancePriority(Priority.valueOf(processMeta.getScheduleProcessInstancePriority())); - } - - if (null != processMeta.getScheduleWorkerGroupName()) { - scheduleObj.setWorkerGroup(processMeta.getScheduleWorkerGroupName()); - } - - return scheduleMapper.insert(scheduleObj); - } - - /** - * check import process has sub process - * recursion create sub process - * - * @param loginUser login user - * @param targetProject target project - * @param jsonArray process task array - * @param subProcessIdMap correct sub process id map - */ - public void importSubProcess(User loginUser, Project targetProject, ArrayNode jsonArray, Map subProcessIdMap) { - for (int i = 0; i < jsonArray.size(); i++) { - ObjectNode taskNode = (ObjectNode) jsonArray.path(i); - String taskType = taskNode.path("type").asText(); - - if (!checkTaskHasSubProcess(taskType)) { - continue; - } - //get sub process info - ObjectNode subParams = (ObjectNode) taskNode.path("params"); - Integer subProcessId = subParams.path(PROCESSDEFINITIONID).asInt(); - ProcessDefinition subProcess = processDefineMapper.queryByDefineId(subProcessId); - //check is sub process exist in db - if (null == subProcess) { - continue; - } - String subProcessJson = subProcess.getProcessDefinitionJson(); - //check current project has sub process - ProcessDefinition currentProjectSubProcess = processDefineMapper.queryByDefineName(targetProject.getId(), subProcess.getName()); - - if (null == currentProjectSubProcess) { - ArrayNode subJsonArray = (ArrayNode) JSONUtils.parseObject(subProcess.getProcessDefinitionJson()).get(TASKS); - - List subProcessList = StreamUtils.asStream(subJsonArray.elements()) - .filter(item -> checkTaskHasSubProcess(JSONUtils.parseObject(item.toString()).path("type").asText())) - .collect(Collectors.toList()); - - if (CollectionUtils.isNotEmpty(subProcessList)) { - importSubProcess(loginUser, targetProject, subJsonArray, subProcessIdMap); - //sub process processId correct - if (!subProcessIdMap.isEmpty()) { - - for (Map.Entry entry : subProcessIdMap.entrySet()) { - String oldSubProcessId = "\"processDefinitionId\":" + entry.getKey(); - String newSubProcessId = "\"processDefinitionId\":" + entry.getValue(); - subProcessJson = subProcessJson.replaceAll(oldSubProcessId, newSubProcessId); - } - - subProcessIdMap.clear(); - } - } - - //if sub-process recursion - Date now = new Date(); - //create sub process in target project - ProcessDefinition processDefine = new ProcessDefinition(); - processDefine.setName(subProcess.getName()); - processDefine.setVersion(subProcess.getVersion()); - processDefine.setReleaseState(subProcess.getReleaseState()); - processDefine.setProjectId(targetProject.getId()); - processDefine.setUserId(loginUser.getId()); - processDefine.setProcessDefinitionJson(subProcessJson); - processDefine.setDescription(subProcess.getDescription()); - processDefine.setLocations(subProcess.getLocations()); - processDefine.setConnects(subProcess.getConnects()); - processDefine.setTimeout(subProcess.getTimeout()); - processDefine.setTenantId(subProcess.getTenantId()); - processDefine.setGlobalParams(subProcess.getGlobalParams()); - processDefine.setCreateTime(now); - processDefine.setUpdateTime(now); - processDefine.setFlag(subProcess.getFlag()); - processDefine.setReceivers(subProcess.getReceivers()); - processDefine.setReceiversCc(subProcess.getReceiversCc()); - processDefineMapper.insert(processDefine); - - logger.info("create sub process, project: {}, process name: {}", targetProject.getName(), processDefine.getName()); - - //modify task node - ProcessDefinition newSubProcessDefine = processDefineMapper.queryByDefineName(processDefine.getProjectId(), processDefine.getName()); - - if (null != newSubProcessDefine) { - subProcessIdMap.put(subProcessId, newSubProcessDefine.getId()); - subParams.put(PROCESSDEFINITIONID, newSubProcessDefine.getId()); - taskNode.set("params", subParams); - } - } - } - } - + Map importProcessDefinition(User loginUser, + MultipartFile file, + String currentProjectName); /** * check the process definition node meets the specifications * - * @param processData process data + * @param processData process data * @param processDefinitionJson process definition json * @return check result code */ - public Map checkProcessNodeList(ProcessData processData, String processDefinitionJson) { - - Map result = new HashMap<>(5); - try { - if (processData == null) { - logger.error("process data is null"); - putMsg(result, Status.DATA_IS_NOT_VALID, processDefinitionJson); - return result; - } - - // Check whether the task node is normal - List taskNodes = processData.getTasks(); - - if (taskNodes == null) { - logger.error("process node info is empty"); - putMsg(result, Status.DATA_IS_NULL, processDefinitionJson); - return result; - } - - // check has cycle - if (graphHasCycle(taskNodes)) { - logger.error("process DAG has cycle"); - putMsg(result, Status.PROCESS_NODE_HAS_CYCLE); - return result; - } - - // check whether the process definition json is normal - for (TaskNode taskNode : taskNodes) { - if (!CheckUtils.checkTaskNodeParameters(taskNode.getParams(), taskNode.getType())) { - logger.error("task node {} parameter invalid", taskNode.getName()); - putMsg(result, Status.PROCESS_NODE_S_PARAMETER_INVALID, taskNode.getName()); - return result; - } - - // check extra params - CheckUtils.checkOtherParams(taskNode.getExtras()); - } - putMsg(result, Status.SUCCESS); - } catch (Exception e) { - result.put(Constants.STATUS, Status.REQUEST_PARAMS_NOT_VALID_ERROR); - result.put(Constants.MSG, e.getMessage()); - } - return result; - } + Map checkProcessNodeList(ProcessData processData, + String processDefinitionJson); /** * get task node details based on process definition @@ -1162,36 +220,7 @@ public class ProcessDefinitionService extends BaseDAGService { * @param defineId define id * @return task node list */ - public Map getTaskNodeListByDefinitionId(Integer defineId) { - Map result = new HashMap<>(); - - ProcessDefinition processDefinition = processDefineMapper.selectById(defineId); - if (processDefinition == null) { - logger.info("process define not exists"); - putMsg(result, Status.PROCESS_DEFINE_NOT_EXIST, defineId); - return result; - } - - - String processDefinitionJson = processDefinition.getProcessDefinitionJson(); - - ProcessData processData = JSONUtils.parseObject(processDefinitionJson, ProcessData.class); - - //process data check - if (null == processData) { - logger.error("process data is null"); - putMsg(result, Status.DATA_IS_NOT_VALID, processDefinitionJson); - return result; - } - - List taskNodeList = (processData.getTasks() == null) ? new ArrayList<>() : processData.getTasks(); - - result.put(Constants.DATA_LIST, taskNodeList); - putMsg(result, Status.SUCCESS); - - return result; - - } + Map getTaskNodeListByDefinitionId(Integer defineId); /** * get task node details based on process definition @@ -1199,37 +228,7 @@ public class ProcessDefinitionService extends BaseDAGService { * @param defineIdList define id list * @return task node list */ - public Map getTaskNodeListByDefinitionIdList(String defineIdList) { - Map result = new HashMap<>(); - - Map> taskNodeMap = new HashMap<>(); - String[] idList = defineIdList.split(","); - List idIntList = new ArrayList<>(); - for (String definitionId : idList) { - idIntList.add(Integer.parseInt(definitionId)); - } - Integer[] idArray = idIntList.toArray(new Integer[idIntList.size()]); - List processDefinitionList = processDefineMapper.queryDefinitionListByIdList(idArray); - if (CollectionUtils.isEmpty(processDefinitionList)) { - logger.info("process definition not exists"); - putMsg(result, Status.PROCESS_DEFINE_NOT_EXIST, defineIdList); - return result; - } - - for (ProcessDefinition processDefinition : processDefinitionList) { - String processDefinitionJson = processDefinition.getProcessDefinitionJson(); - ProcessData processData = JSONUtils.parseObject(processDefinitionJson, ProcessData.class); - List taskNodeList = (processData.getTasks() == null) ? new ArrayList<>() : processData.getTasks(); - taskNodeMap.put(processDefinition.getId(), taskNodeList); - } - - result.put(Constants.DATA_LIST, taskNodeMap); - putMsg(result, Status.SUCCESS); - - return result; - - } - + Map getTaskNodeListByDefinitionIdList(String defineIdList); /** * query process definition all by project id @@ -1237,219 +236,29 @@ public class ProcessDefinitionService extends BaseDAGService { * @param projectId project id * @return process definitions in the project */ - public Map queryProcessDefinitionAllByProjectId(Integer projectId) { - - HashMap result = new HashMap<>(5); - - List resourceList = processDefineMapper.queryAllDefinitionList(projectId); - result.put(Constants.DATA_LIST, resourceList); - putMsg(result, Status.SUCCESS); - - return result; - } + Map queryProcessDefinitionAllByProjectId(Integer projectId); /** * Encapsulates the TreeView structure * * @param processId process definition id - * @param limit limit + * @param limit limit * @return tree view json data * @throws Exception exception */ - public Map viewTree(Integer processId, Integer limit) throws Exception { - Map result = new HashMap<>(); - - ProcessDefinition processDefinition = processDefineMapper.selectById(processId); - if (null == processDefinition) { - logger.info("process define not exists"); - putMsg(result, Status.PROCESS_DEFINE_NOT_EXIST, processDefinition); - return result; - } - DAG dag = genDagGraph(processDefinition); - /** - * nodes that is running - */ - Map> runningNodeMap = new ConcurrentHashMap<>(); - - /** - * nodes that is waiting torun - */ - Map> waitingRunningNodeMap = new ConcurrentHashMap<>(); - - /** - * List of process instances - */ - List processInstanceList = processInstanceMapper.queryByProcessDefineId(processId, limit); - - for (ProcessInstance processInstance : processInstanceList) { - processInstance.setDuration(DateUtils.differSec(processInstance.getStartTime(), processInstance.getEndTime())); - } - - if (limit > processInstanceList.size()) { - limit = processInstanceList.size(); - } - - TreeViewDto parentTreeViewDto = new TreeViewDto(); - parentTreeViewDto.setName("DAG"); - parentTreeViewDto.setType(""); - // Specify the process definition, because it is a TreeView for a process definition - - for (int i = limit - 1; i >= 0; i--) { - ProcessInstance processInstance = processInstanceList.get(i); - - Date endTime = processInstance.getEndTime() == null ? new Date() : processInstance.getEndTime(); - parentTreeViewDto.getInstances().add(new Instance(processInstance.getId(), processInstance.getName(), "", processInstance.getState().toString() - , processInstance.getStartTime(), endTime, processInstance.getHost(), DateUtils.format2Readable(endTime.getTime() - processInstance.getStartTime().getTime()))); - } - - List parentTreeViewDtoList = new ArrayList<>(); - parentTreeViewDtoList.add(parentTreeViewDto); - // Here is the encapsulation task instance - for (String startNode : dag.getBeginNode()) { - runningNodeMap.put(startNode, parentTreeViewDtoList); - } - - while (Stopper.isRunning()) { - Set postNodeList = null; - Iterator>> iter = runningNodeMap.entrySet().iterator(); - while (iter.hasNext()) { - Map.Entry> en = iter.next(); - String nodeName = en.getKey(); - parentTreeViewDtoList = en.getValue(); - - TreeViewDto treeViewDto = new TreeViewDto(); - treeViewDto.setName(nodeName); - TaskNode taskNode = dag.getNode(nodeName); - treeViewDto.setType(taskNode.getType()); - - - //set treeViewDto instances - for (int i = limit - 1; i >= 0; i--) { - ProcessInstance processInstance = processInstanceList.get(i); - TaskInstance taskInstance = taskInstanceMapper.queryByInstanceIdAndName(processInstance.getId(), nodeName); - if (taskInstance == null) { - treeViewDto.getInstances().add(new Instance(-1, "not running", "null")); - } else { - Date startTime = taskInstance.getStartTime() == null ? new Date() : taskInstance.getStartTime(); - Date endTime = taskInstance.getEndTime() == null ? new Date() : taskInstance.getEndTime(); - - int subProcessId = 0; - /** - * if process is sub process, the return sub id, or sub id=0 - */ - if (taskInstance.getTaskType().equals(TaskType.SUB_PROCESS.name())) { - String taskJson = taskInstance.getTaskJson(); - taskNode = JSONUtils.parseObject(taskJson, TaskNode.class); - subProcessId = Integer.parseInt(JSONUtils.parseObject( - taskNode.getParams()).path(CMDPARAM_SUB_PROCESS_DEFINE_ID).asText()); - } - treeViewDto.getInstances().add(new Instance(taskInstance.getId(), taskInstance.getName(), taskInstance.getTaskType(), taskInstance.getState().toString() - , taskInstance.getStartTime(), taskInstance.getEndTime(), taskInstance.getHost(), DateUtils.format2Readable(endTime.getTime() - startTime.getTime()), subProcessId)); - } - } - for (TreeViewDto pTreeViewDto : parentTreeViewDtoList) { - pTreeViewDto.getChildren().add(treeViewDto); - } - postNodeList = dag.getSubsequentNodes(nodeName); - if (CollectionUtils.isNotEmpty(postNodeList)) { - for (String nextNodeName : postNodeList) { - List treeViewDtoList = waitingRunningNodeMap.get(nextNodeName); - if (CollectionUtils.isNotEmpty(treeViewDtoList)) { - treeViewDtoList.add(treeViewDto); - waitingRunningNodeMap.put(nextNodeName, treeViewDtoList); - } else { - treeViewDtoList = new ArrayList<>(); - treeViewDtoList.add(treeViewDto); - waitingRunningNodeMap.put(nextNodeName, treeViewDtoList); - } - } - } - runningNodeMap.remove(nodeName); - } - if (waitingRunningNodeMap == null || waitingRunningNodeMap.size() == 0) { - break; - } else { - runningNodeMap.putAll(waitingRunningNodeMap); - waitingRunningNodeMap.clear(); - } - } - result.put(Constants.DATA_LIST, parentTreeViewDto); - result.put(Constants.STATUS, Status.SUCCESS); - result.put(Constants.MSG, Status.SUCCESS.getMsg()); - return result; - } - + Map viewTree(Integer processId, + Integer limit) throws Exception; /** - * Generate the DAG Graph based on the process definition id + * switch the defined process definition verison * - * @param processDefinition process definition - * @return dag graph + * @param loginUser login user + * @param projectName project name + * @param processDefinitionId process definition id + * @param version the version user want to switch + * @return switch process definition version result code */ - private DAG genDagGraph(ProcessDefinition processDefinition) { - - String processDefinitionJson = processDefinition.getProcessDefinitionJson(); - - ProcessData processData = JSONUtils.parseObject(processDefinitionJson, ProcessData.class); - - //check process data - if (null != processData) { - List taskNodeList = processData.getTasks(); - processDefinition.setGlobalParamList(processData.getGlobalParams()); - ProcessDag processDag = DagHelper.getProcessDag(taskNodeList); - - // Generate concrete Dag to be executed - return DagHelper.buildDagGraph(processDag); - } - - return new DAG<>(); - } - - - /** - * whether the graph has a ring - * - * @param taskNodeResponseList task node response list - * @return if graph has cycle flag - */ - private boolean graphHasCycle(List taskNodeResponseList) { - DAG graph = new DAG<>(); - - // Fill the vertices - for (TaskNode taskNodeResponse : taskNodeResponseList) { - graph.addNode(taskNodeResponse.getName(), taskNodeResponse); - } - - // Fill edge relations - for (TaskNode taskNodeResponse : taskNodeResponseList) { - taskNodeResponse.getPreTasks(); - List preTasks = JSONUtils.toList(taskNodeResponse.getPreTasks(), String.class); - if (CollectionUtils.isNotEmpty(preTasks)) { - for (String preTask : preTasks) { - if (!graph.addEdge(preTask, taskNodeResponse.getName())) { - return true; - } - } - } - } - - return graph.hasCycle(); - } - - private String recursionProcessDefinitionName(Integer projectId, String processDefinitionName, int num) { - ProcessDefinition processDefinition = processDefineMapper.queryByDefineName(projectId, processDefinitionName); - if (processDefinition != null) { - if (num > 1) { - String str = processDefinitionName.substring(0, processDefinitionName.length() - 3); - processDefinitionName = str + "(" + num + ")"; - } else { - processDefinitionName = processDefinition.getName() + "(" + num + ")"; - } - } else { - return processDefinitionName; - } - return recursionProcessDefinitionName(projectId, processDefinitionName, num + 1); - } - + Map switchProcessDefinitionVersion(User loginUser, String projectName + , int processDefinitionId, long version); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionVersionService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionVersionService.java new file mode 100644 index 0000000000..5538194db7 --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionVersionService.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.service; + +import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; +import org.apache.dolphinscheduler.dao.entity.ProcessDefinitionVersion; +import org.apache.dolphinscheduler.dao.entity.User; + +import java.util.Map; + +public interface ProcessDefinitionVersionService { + + /** + * add the newest version of one process definition + * + * @param processDefinition the process definition that need to record version + * @return the newest version number of this process definition + */ + long addProcessDefinitionVersion(ProcessDefinition processDefinition); + + /** + * query the pagination versions info by one certain process definition id + * + * @param loginUser login user info to check auth + * @param projectName process definition project name + * @param pageNo page number + * @param pageSize page size + * @param processDefinitionId process definition id + * @return the pagination process definition versions info of the certain process definition + */ + Map queryProcessDefinitionVersions(User loginUser, String projectName, + int pageNo, int pageSize, int processDefinitionId); + + /** + * query one certain process definition version by version number and process definition id + * + * @param processDefinitionId process definition id + * @param version version number + * @return the process definition version info + */ + ProcessDefinitionVersion queryByProcessDefinitionIdAndVersion(int processDefinitionId, + long version); + + /** + * delete one certain process definition by version number and process definition id + * + * @param loginUser login user info to check auth + * @param projectName process definition project name + * @param processDefinitionId process definition id + * @param version version number + * @return delele result code + */ + Map deleteByProcessDefinitionIdAndVersion(User loginUser, String projectName, + int processDefinitionId, long version); +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessInstanceService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessInstanceService.java index e4a00f3895..c8d3c74da0 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessInstanceService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessInstanceService.java @@ -14,10 +14,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.api.service; -import com.baomidou.mybatisplus.core.metadata.IPage; -import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import static org.apache.dolphinscheduler.common.Constants.DATA_LIST; +import static org.apache.dolphinscheduler.common.Constants.DEPENDENT_SPLIT; +import static org.apache.dolphinscheduler.common.Constants.GLOBAL_PARAMS; +import static org.apache.dolphinscheduler.common.Constants.LOCAL_PARAMS; +import static org.apache.dolphinscheduler.common.Constants.PROCESS_INSTANCE_STATE; +import static org.apache.dolphinscheduler.common.Constants.TASK_LIST; + import org.apache.dolphinscheduler.api.dto.gantt.GanttDto; import org.apache.dolphinscheduler.api.dto.gantt.Task; import org.apache.dolphinscheduler.api.enums.Status; @@ -31,20 +37,27 @@ import org.apache.dolphinscheduler.common.enums.TaskType; import org.apache.dolphinscheduler.common.graph.DAG; import org.apache.dolphinscheduler.common.model.TaskNode; import org.apache.dolphinscheduler.common.model.TaskNodeRelation; +import org.apache.dolphinscheduler.common.process.ProcessDag; import org.apache.dolphinscheduler.common.process.Property; -import org.apache.dolphinscheduler.common.utils.*; +import org.apache.dolphinscheduler.common.utils.CollectionUtils; +import org.apache.dolphinscheduler.common.utils.DateUtils; +import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.ParameterUtils; +import org.apache.dolphinscheduler.common.utils.StringUtils; import org.apache.dolphinscheduler.common.utils.placeholder.BusinessTimeUtils; -import org.apache.dolphinscheduler.dao.entity.*; +import org.apache.dolphinscheduler.dao.entity.ProcessData; +import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; +import org.apache.dolphinscheduler.dao.entity.ProcessInstance; +import org.apache.dolphinscheduler.dao.entity.Project; +import org.apache.dolphinscheduler.dao.entity.TaskInstance; +import org.apache.dolphinscheduler.dao.entity.Tenant; +import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper; import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper; import org.apache.dolphinscheduler.dao.mapper.ProjectMapper; import org.apache.dolphinscheduler.dao.mapper.TaskInstanceMapper; +import org.apache.dolphinscheduler.dao.utils.DagHelper; import org.apache.dolphinscheduler.service.process.ProcessService; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Transactional; import java.io.BufferedReader; import java.io.ByteArrayInputStream; @@ -52,16 +65,28 @@ import java.io.IOException; import java.io.InputStreamReader; import java.nio.charset.StandardCharsets; import java.text.ParseException; -import java.util.*; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; import java.util.stream.Collectors; -import static org.apache.dolphinscheduler.common.Constants.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; /** * process instance service */ @Service -public class ProcessInstanceService extends BaseDAGService { +public class ProcessInstanceService extends BaseService { private static final Logger logger = LoggerFactory.getLogger(ProcessInstanceService.class); @@ -84,6 +109,9 @@ public class ProcessInstanceService extends BaseDAGService { @Autowired ProcessDefinitionService processDefinitionService; + @Autowired + ProcessDefinitionVersionService processDefinitionVersionService; + @Autowired ExecutorService execService; @@ -94,18 +122,11 @@ public class ProcessInstanceService extends BaseDAGService { LoggerService loggerService; - @Autowired UsersService usersService; /** * return top n SUCCESS process instance order by running time which started between startTime and endTime - * @param loginUser - * @param projectName - * @param size - * @param startTime - * @param endTime - * @return */ public Map queryTopNLongestRunningProcessInstance(User loginUser, String projectName, int size, String startTime, String endTime) { Map result = new HashMap<>(); @@ -131,7 +152,7 @@ public class ProcessInstanceService extends BaseDAGService { return result; } Date end = DateUtils.stringToDate(endTime); - if(start == null || end == null) { + if (start == null || end == null) { putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, "startDate,endDate"); return result; } @@ -145,6 +166,7 @@ public class ProcessInstanceService extends BaseDAGService { putMsg(result, Status.SUCCESS); return result; } + /** * query process instance by id * @@ -154,7 +176,7 @@ public class ProcessInstanceService extends BaseDAGService { * @return process instance detail */ public Map queryProcessInstanceById(User loginUser, String projectName, Integer processId) { - Map result = new HashMap<>(5); + Map result = new HashMap<>(); Project project = projectMapper.queryByName(projectName); Map checkResult = projectService.checkProjectAndAuth(loginUser, project, projectName); @@ -167,7 +189,7 @@ public class ProcessInstanceService extends BaseDAGService { ProcessDefinition processDefinition = processService.findProcessDefineById(processInstance.getProcessDefinitionId()); processInstance.setReceivers(processDefinition.getReceivers()); processInstance.setReceiversCc(processDefinition.getReceiversCc()); - result.put(Constants.DATA_LIST, processInstance); + result.put(DATA_LIST, processInstance); putMsg(result, Status.SUCCESS); return result; @@ -190,10 +212,10 @@ public class ProcessInstanceService extends BaseDAGService { */ public Map queryProcessInstanceList(User loginUser, String projectName, Integer processDefineId, String startDate, String endDate, - String searchVal, String executorName,ExecutionStatus stateType, String host, + String searchVal, String executorName, ExecutionStatus stateType, String host, Integer pageNo, Integer pageSize) { - Map result = new HashMap<>(5); + Map result = new HashMap<>(); Project project = projectMapper.queryByName(projectName); Map checkResult = projectService.checkProjectAndAuth(loginUser, project, projectName); @@ -222,18 +244,18 @@ public class ProcessInstanceService extends BaseDAGService { return result; } - Page page = new Page(pageNo, pageSize); + Page page = new Page<>(pageNo, pageSize); PageInfo pageInfo = new PageInfo(pageNo, pageSize); int executorId = usersService.getUserIdByName(executorName); IPage processInstanceList = processInstanceMapper.queryProcessInstanceListPaging(page, - project.getId(), processDefineId, searchVal, executorId,statusArray, host, start, end); + project.getId(), processDefineId, searchVal, executorId, statusArray, host, start, end); List processInstances = processInstanceList.getRecords(); - for(ProcessInstance processInstance: processInstances){ - processInstance.setDuration(DateUtils.differSec(processInstance.getStartTime(),processInstance.getEndTime())); + for (ProcessInstance processInstance : processInstances) { + processInstance.setDuration(DateUtils.differSec(processInstance.getStartTime(), processInstance.getEndTime())); User executor = usersService.queryUser(processInstance.getExecutorId()); if (null != executor) { processInstance.setExecutorName(executor.getUserName()); @@ -242,13 +264,11 @@ public class ProcessInstanceService extends BaseDAGService { pageInfo.setTotalCount((int) processInstanceList.getTotal()); pageInfo.setLists(processInstances); - result.put(Constants.DATA_LIST, pageInfo); + result.put(DATA_LIST, pageInfo); putMsg(result, Status.SUCCESS); return result; } - - /** * query task list by process instance id * @@ -273,7 +293,7 @@ public class ProcessInstanceService extends BaseDAGService { Map resultMap = new HashMap<>(); resultMap.put(PROCESS_INSTANCE_STATE, processInstance.getState().toString()); resultMap.put(TASK_LIST, taskInstanceList); - result.put(Constants.DATA_LIST, resultMap); + result.put(DATA_LIST, resultMap); putMsg(result, Status.SUCCESS); return result; @@ -281,14 +301,13 @@ public class ProcessInstanceService extends BaseDAGService { /** * add dependent result for dependent task - * @param taskInstanceList */ private void addDependResultForTaskList(List taskInstanceList) throws IOException { - for(TaskInstance taskInstance: taskInstanceList){ - if(taskInstance.getTaskType().equalsIgnoreCase(TaskType.DEPENDENT.toString())){ + for (TaskInstance taskInstance : taskInstanceList) { + if (taskInstance.getTaskType().equalsIgnoreCase(TaskType.DEPENDENT.toString())) { Result logResult = loggerService.queryLog( taskInstance.getId(), 0, 4098); - if(logResult.getCode() == Status.SUCCESS.ordinal()){ + if (logResult.getCode() == Status.SUCCESS.ordinal()) { String log = (String) logResult.getData(); Map resultMap = parseLogForDependentResult(log); taskInstance.setDependentResult(JSONUtils.toJsonString(resultMap)); @@ -297,24 +316,24 @@ public class ProcessInstanceService extends BaseDAGService { } } - public Map parseLogForDependentResult(String log) throws IOException { + public Map parseLogForDependentResult(String log) throws IOException { Map resultMap = new HashMap<>(); - if(StringUtils.isEmpty(log)){ + if (StringUtils.isEmpty(log)) { return resultMap; } BufferedReader br = new BufferedReader(new InputStreamReader(new ByteArrayInputStream(log.getBytes( - StandardCharsets.UTF_8)), StandardCharsets.UTF_8)); + StandardCharsets.UTF_8)), StandardCharsets.UTF_8)); String line; while ((line = br.readLine()) != null) { - if(line.contains(DEPENDENT_SPLIT)){ + if (line.contains(DEPENDENT_SPLIT)) { String[] tmpStringArray = line.split(":\\|\\|"); - if(tmpStringArray.length != 2){ + if (tmpStringArray.length != 2) { continue; } String dependResultString = tmpStringArray[1]; String[] dependStringArray = dependResultString.split(","); - if(dependStringArray.length != 2){ + if (dependStringArray.length != 2) { continue; } String key = dependStringArray[0].trim(); @@ -325,7 +344,6 @@ public class ProcessInstanceService extends BaseDAGService { return resultMap; } - /** * query sub process instance detail info by task id * @@ -362,7 +380,7 @@ public class ProcessInstanceService extends BaseDAGService { } Map dataMap = new HashMap<>(); dataMap.put("subProcessInstanceId", subWorkflowInstance.getId()); - result.put(Constants.DATA_LIST, dataMap); + result.put(DATA_LIST, dataMap); putMsg(result, Status.SUCCESS); return result; } @@ -438,7 +456,7 @@ public class ProcessInstanceService extends BaseDAGService { processInstance.setTimeout(timeout); Tenant tenant = processService.getTenantForProcess(processData.getTenantId(), processDefinition.getUserId()); - if(tenant != null){ + if (tenant != null) { processInstance.setTenantCode(tenant.getTenantCode()); } processInstance.setProcessInstanceJson(processInstanceJson); @@ -453,6 +471,11 @@ public class ProcessInstanceService extends BaseDAGService { processDefinition.setLocations(locations); processDefinition.setConnects(connects); processDefinition.setTimeout(timeout); + processDefinition.setUpdateTime(new Date()); + + // add process definition version + long version = processDefinitionVersionService.addProcessDefinitionVersion(processDefinition); + processDefinition.setVersion(version); updateDefine = processDefineMapper.updateById(processDefinition); } if (update > 0 && updateDefine > 0) { @@ -461,7 +484,6 @@ public class ProcessInstanceService extends BaseDAGService { putMsg(result, Status.UPDATE_PROCESS_INSTANCE_ERROR); } - return result; } @@ -501,13 +523,14 @@ public class ProcessInstanceService extends BaseDAGService { } Map dataMap = new HashMap<>(); dataMap.put("parentWorkflowInstance", parentWorkflowInstance.getId()); - result.put(Constants.DATA_LIST, dataMap); + result.put(DATA_LIST, dataMap); putMsg(result, Status.SUCCESS); return result; } /** * delete process instance by id, at the same time，delete task instance and their mapping relation data + * * @param loginUser login user * @param projectName project name * @param processInstanceId process instance id @@ -516,7 +539,7 @@ public class ProcessInstanceService extends BaseDAGService { @Transactional(rollbackFor = RuntimeException.class) public Map deleteProcessInstanceById(User loginUser, String projectName, Integer processInstanceId) { - Map result = new HashMap<>(5); + Map result = new HashMap<>(); Project project = projectMapper.queryByName(projectName); Map checkResult = projectService.checkProjectAndAuth(loginUser, project, projectName); @@ -530,13 +553,10 @@ public class ProcessInstanceService extends BaseDAGService { return result; } - - processService.removeTaskLogFile(processInstanceId); // delete database cascade int delete = processService.deleteWorkProcessInstanceById(processInstanceId); - processService.deleteAllSubWorkProcessByParentId(processInstanceId); processService.deleteWorkProcessMapByParentId(processInstanceId); @@ -556,7 +576,7 @@ public class ProcessInstanceService extends BaseDAGService { * @return variables data */ public Map viewVariables(Integer processInstanceId) { - Map result = new HashMap<>(5); + Map result = new HashMap<>(); ProcessInstance processInstance = processInstanceMapper.queryDetailById(processInstanceId); @@ -568,7 +588,6 @@ public class ProcessInstanceService extends BaseDAGService { .getBusinessTime(processInstance.getCmdTypeIfComplement(), processInstance.getScheduleTime()); - String workflowInstanceJson = processInstance.getProcessInstanceJson(); ProcessData workflowData = JSONUtils.parseObject(workflowInstanceJson, ProcessData.class); @@ -579,10 +598,9 @@ public class ProcessInstanceService extends BaseDAGService { List globalParams = new ArrayList<>(); if (userDefinedParams != null && userDefinedParams.length() > 0) { - globalParams = JSONUtils.toList(userDefinedParams, Property.class); + globalParams = JSONUtils.toList(userDefinedParams, Property.class); } - List taskNodeList = workflowData.getTasks(); // global param string @@ -594,7 +612,7 @@ public class ProcessInstanceService extends BaseDAGService { } // local params - Map> localUserDefParams = new HashMap<>(); + Map> localUserDefParams = new HashMap<>(); for (TaskNode taskNode : taskNodeList) { String parameter = taskNode.getParams(); Map map = JSONUtils.toMap(parameter); @@ -603,9 +621,9 @@ public class ProcessInstanceService extends BaseDAGService { localParams = ParameterUtils.convertParameterPlaceholders(localParams, timeParams); List localParamsList = JSONUtils.toList(localParams, Property.class); - Map localParamsMap = new HashMap<>(); - localParamsMap.put("taskType",taskNode.getType()); - localParamsMap.put("localParamsList",localParamsList); + Map localParamsMap = new HashMap<>(); + localParamsMap.put("taskType", taskNode.getType()); + localParamsMap.put("localParamsList", localParamsList); if (CollectionUtils.isNotEmpty(localParamsList)) { localUserDefParams.put(taskNode.getName(), localParamsMap); } @@ -618,7 +636,7 @@ public class ProcessInstanceService extends BaseDAGService { resultMap.put(GLOBAL_PARAMS, globalParams); resultMap.put(LOCAL_PARAMS, localUserDefParams); - result.put(Constants.DATA_LIST, resultMap); + result.put(DATA_LIST, resultMap); putMsg(result, Status.SUCCESS); return result; } @@ -668,9 +686,48 @@ public class ProcessInstanceService extends BaseDAGService { } ganttDto.setTasks(taskList); - result.put(Constants.DATA_LIST, ganttDto); + result.put(DATA_LIST, ganttDto); putMsg(result, Status.SUCCESS); return result; } + /** + * process instance to DAG + * + * @param processInstance input process instance + * @return process instance dag. + */ + private static DAG processInstance2DAG(ProcessInstance processInstance) { + + String processDefinitionJson = processInstance.getProcessInstanceJson(); + + ProcessData processData = JSONUtils.parseObject(processDefinitionJson, ProcessData.class); + + List taskNodeList = processData.getTasks(); + + ProcessDag processDag = DagHelper.getProcessDag(taskNodeList); + + return DagHelper.buildDagGraph(processDag); + } + + /** + * query process instance by processDefinitionId and stateArray + * @param processDefinitionId processDefinitionId + * @param states states array + * @return process instance list + */ + public List queryByProcessDefineIdAndStatus(int processDefinitionId, int[] states) { + return processInstanceMapper.queryByProcessDefineIdAndStatus(processDefinitionId, states); + } + + /** + * query process instance by processDefinitionId + * @param processDefinitionId processDefinitionId + * @param size size + * @return process instance list + */ + public List queryByProcessDefineId(int processDefinitionId,int size) { + return processInstanceMapper.queryByProcessDefineId(processDefinitionId, size); + } + } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProjectService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProjectService.java index 6d3650b77f..ca0e1fc0ec 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProjectService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProjectService.java @@ -16,45 +16,15 @@ */ package org.apache.dolphinscheduler.api.service; -import org.apache.dolphinscheduler.api.enums.Status; -import org.apache.dolphinscheduler.api.utils.PageInfo; -import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.common.enums.UserType; -import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; import org.apache.dolphinscheduler.dao.entity.Project; -import org.apache.dolphinscheduler.dao.entity.ProjectUser; import org.apache.dolphinscheduler.dao.entity.User; -import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper; -import org.apache.dolphinscheduler.dao.mapper.ProjectMapper; -import org.apache.dolphinscheduler.dao.mapper.ProjectUserMapper; -import com.baomidou.mybatisplus.core.metadata.IPage; -import com.baomidou.mybatisplus.extension.plugins.pagination.Page; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; -import java.util.*; - -import static org.apache.dolphinscheduler.api.utils.CheckUtils.checkDesc; +import java.util.Map; /** * project service - *HttpTask./ **/ -@Service -public class ProjectService extends BaseService{ - - private static final Logger logger = LoggerFactory.getLogger(ProjectService.class); - - @Autowired - private ProjectMapper projectMapper; - - @Autowired - private ProjectUserMapper projectUserMapper; - - @Autowired - private ProcessDefinitionMapper processDefinitionMapper; +public interface ProjectService { /** * create project @@ -64,38 +34,7 @@ public class ProjectService extends BaseService{ * @param desc description * @return returns an error if it exists */ - public Map createProject(User loginUser, String name, String desc) { - - Map result = new HashMap<>(5); - Map descCheck = checkDesc(desc); - if (descCheck.get(Constants.STATUS) != Status.SUCCESS) { - return descCheck; - } - - Project project = projectMapper.queryByName(name); - if (project != null) { - putMsg(result, Status.PROJECT_ALREADY_EXISTS, name); - return result; - } - project = new Project(); - Date now = new Date(); - - project.setName(name); - project.setDescription(desc); - project.setUserId(loginUser.getId()); - project.setUserName(loginUser.getUserName()); - project.setCreateTime(now); - project.setUpdateTime(now); - - if (projectMapper.insert(project) > 0) { - Project insertedProject = projectMapper.queryByName(name); - result.put(Constants.DATA_LIST, insertedProject); - putMsg(result, Status.SUCCESS); - } else { - putMsg(result, Status.CREATE_PROJECT_ERROR); - } - return result; - } + Map createProject(User loginUser, String name, String desc); /** * query project details by id @@ -103,19 +42,7 @@ public class ProjectService extends BaseService{ * @param projectId project id * @return project detail information */ - public Map queryById(Integer projectId) { - - Map result = new HashMap<>(5); - Project project = projectMapper.selectById(projectId); - - if (project != null) { - result.put(Constants.DATA_LIST, project); - putMsg(result, Status.SUCCESS); - } else { - putMsg(result, Status.PROJECT_NOT_FOUNT, projectId); - } - return result; - } + Map queryById(Integer projectId); /** * check project and authorization @@ -125,30 +52,9 @@ public class ProjectService extends BaseService{ * @param projectName project name * @return true if the login user have permission to see the project */ - public Map checkProjectAndAuth(User loginUser, Project project, String projectName) { - Map result = new HashMap<>(5); - if (project == null) { - putMsg(result, Status.PROJECT_NOT_FOUNT, projectName); - } else if (!checkReadPermission(loginUser, project)) { - // check read permission - putMsg(result, Status.USER_NO_OPERATION_PROJECT_PERM, loginUser.getUserName(), projectName); - }else { - putMsg(result, Status.SUCCESS); - } - return result; - } + Map checkProjectAndAuth(User loginUser, Project project, String projectName); - public boolean hasProjectAndPerm(User loginUser, Project project, Map result) { - boolean checkResult = false; - if (project == null) { - putMsg(result, Status.PROJECT_NOT_FOUNT, ""); - } else if (!checkReadPermission(loginUser, project)) { - putMsg(result, Status.USER_NO_OPERATION_PROJECT_PERM, loginUser.getUserName(), project.getName()); - } else { - checkResult = true; - } - return checkResult; - } + boolean hasProjectAndPerm(User loginUser, Project project, Map result); /** * admin can view all projects @@ -159,29 +65,7 @@ public class ProjectService extends BaseService{ * @param pageNo page number * @return project list which the login user have permission to see */ - public Map queryProjectListPaging(User loginUser, Integer pageSize, Integer pageNo, String searchVal) { - Map result = new HashMap<>(); - PageInfo pageInfo = new PageInfo(pageNo, pageSize); - - Page page = new Page(pageNo, pageSize); - - int userId = loginUser.getUserType() == UserType.ADMIN_USER ? 0 : loginUser.getId(); - IPage projectIPage = projectMapper.queryProjectListPaging(page, userId, searchVal); - - List projectList = projectIPage.getRecords(); - if(userId != 0){ - for (Project project : projectList) { - project.setPerm(org.apache.dolphinscheduler.common.Constants.DEFAULT_ADMIN_PERMISSION); - } - } - pageInfo.setTotalCount((int)projectIPage.getTotal()); - pageInfo.setLists(projectList); - result.put(Constants.COUNT, (int)projectIPage.getTotal()); - result.put(Constants.DATA_LIST, pageInfo); - putMsg(result, Status.SUCCESS); - - return result; - } + Map queryProjectListPaging(User loginUser, Integer pageSize, Integer pageNo, String searchVal); /** * delete project by id @@ -190,50 +74,7 @@ public class ProjectService extends BaseService{ * @param projectId project id * @return delete result code */ - public Map deleteProject(User loginUser, Integer projectId) { - Map result = new HashMap<>(5); - Project project = projectMapper.selectById(projectId); - Map checkResult = getCheckResult(loginUser, project); - if (checkResult != null) { - return checkResult; - } - - if (!hasPerm(loginUser, project.getUserId())) { - putMsg(result, Status.USER_NO_OPERATION_PERM); - return result; - } - - List processDefinitionList = processDefinitionMapper.queryAllDefinitionList(projectId); - - if(processDefinitionList.size() > 0){ - putMsg(result, Status.DELETE_PROJECT_ERROR_DEFINES_NOT_NULL); - return result; - } - int delete = projectMapper.deleteById(projectId); - if (delete > 0) { - putMsg(result, Status.SUCCESS); - } else { - putMsg(result, Status.DELETE_PROJECT_ERROR); - } - return result; - } - - /** - * get check result - * - * @param loginUser login user - * @param project project - * @return check result - */ - private Map getCheckResult(User loginUser, Project project) { - String projectName = project == null ? null:project.getName(); - Map checkResult = checkProjectAndAuth(loginUser, project, projectName); - Status status = (Status) checkResult.get(Constants.STATUS); - if (status != Status.SUCCESS) { - return checkResult; - } - return null; - } + Map deleteProject(User loginUser, Integer projectId); /** * updateProcessInstance project @@ -244,37 +85,7 @@ public class ProjectService extends BaseService{ * @param desc description * @return update result code */ - public Map update(User loginUser, Integer projectId, String projectName, String desc) { - Map result = new HashMap<>(5); - - Map descCheck = checkDesc(desc); - if (descCheck.get(Constants.STATUS) != Status.SUCCESS) { - return descCheck; - } - - Project project = projectMapper.selectById(projectId); - boolean hasProjectAndPerm = hasProjectAndPerm(loginUser, project, result); - if (!hasProjectAndPerm) { - return result; - } - Project tempProject = projectMapper.queryByName(projectName); - if (tempProject != null && tempProject.getId() != projectId) { - putMsg(result, Status.PROJECT_ALREADY_EXISTS, projectName); - return result; - } - project.setName(projectName); - project.setDescription(desc); - project.setUpdateTime(new Date()); - - int update = projectMapper.updateById(project); - if (update > 0) { - putMsg(result, Status.SUCCESS); - } else { - putMsg(result, Status.UPDATE_PROJECT_ERROR); - } - return result; - } - + Map update(User loginUser, Integer projectId, String projectName, String desc); /** * query unauthorized project @@ -283,48 +94,7 @@ public class ProjectService extends BaseService{ * @param userId user id * @return the projects which user have not permission to see */ - public Map queryUnauthorizedProject(User loginUser, Integer userId) { - Map result = new HashMap<>(5); - if (checkAdmin(loginUser, result)) { - return result; - } - /** - * query all project list except specified userId - */ - List projectList = projectMapper.queryProjectExceptUserId(userId); - List resultList = new ArrayList<>(); - Set projectSet = null; - if (projectList != null && projectList.size() > 0) { - projectSet = new HashSet<>(projectList); - - List authedProjectList = projectMapper.queryAuthedProjectListByUserId(userId); - - resultList = getUnauthorizedProjects(projectSet, authedProjectList); - } - result.put(Constants.DATA_LIST, resultList); - putMsg(result,Status.SUCCESS); - return result; - } - - /** - * get unauthorized project - * - * @param projectSet project set - * @param authedProjectList authed project list - * @return project list that authorization - */ - private List getUnauthorizedProjects(Set projectSet, List authedProjectList) { - List resultList; - Set authedProjectSet = null; - if (authedProjectList != null && authedProjectList.size() > 0) { - authedProjectSet = new HashSet<>(authedProjectList); - projectSet.removeAll(authedProjectSet); - - } - resultList = new ArrayList<>(projectSet); - return resultList; - } - + Map queryUnauthorizedProject(User loginUser, Integer userId); /** * query authorized project @@ -333,83 +103,21 @@ public class ProjectService extends BaseService{ * @param userId user id * @return projects which the user have permission to see, Except for items created by this user */ - public Map queryAuthorizedProject(User loginUser, Integer userId) { - Map result = new HashMap<>(); - - if (checkAdmin(loginUser, result)) { - return result; - } - - List projects = projectMapper.queryAuthedProjectListByUserId(userId); - result.put(Constants.DATA_LIST, projects); - putMsg(result,Status.SUCCESS); - - return result; - } - + Map queryAuthorizedProject(User loginUser, Integer userId); /** - * check whether have read permission + * query authorized project * - * @param user user - * @param project project - * @return true if the user have permission to see the project, otherwise return false + * @param loginUser login user + * @return projects which the user have permission to see, Except for items created by this user */ - private boolean checkReadPermission(User user, Project project) { - int permissionId = queryPermission(user, project); - return (permissionId & Constants.READ_PERMISSION) != 0; - } - - /** - * query permission id - * - * @param user user - * @param project project - * @return permission - */ - private int queryPermission(User user, Project project) { - if (user.getUserType() == UserType.ADMIN_USER) { - return Constants.READ_PERMISSION; - } - - if (project.getUserId() == user.getId()) { - return Constants.ALL_PERMISSIONS; - } - - ProjectUser projectUser = projectUserMapper.queryProjectRelation(project.getId(), user.getId()); - - if (projectUser == null) { - return 0; - } - - return projectUser.getPerm(); - - } + Map queryProjectCreatedByUser(User loginUser); /** * query all project list that have one or more process definitions. + * * @return project list */ - public Map queryAllProjectList() { - Map result = new HashMap<>(); - List projects = projectMapper.selectList(null); - List processDefinitions = processDefinitionMapper.selectList(null); - if(projects != null){ - Set set = new HashSet<>(); - for (ProcessDefinition processDefinition : processDefinitions){ - set.add(processDefinition.getProjectId()); - } - List tempDeletelist = new ArrayList(); - for (Project project : projects) { - if(!set.contains(project.getId())){ - tempDeletelist.add(project); - } - } - projects.removeAll(tempDeletelist); - } - result.put(Constants.DATA_LIST, projects); - putMsg(result,Status.SUCCESS); - return result; - } + Map queryAllProjectList(); -} +} \ No newline at end of file diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/QueueService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/QueueService.java index cba1b5f2bb..caffeabd5c 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/QueueService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/QueueService.java @@ -59,7 +59,7 @@ public class QueueService extends BaseService { * @return queue list */ public Map queryList(User loginUser) { - Map result = new HashMap<>(5); + Map result = new HashMap<>(); if (checkAdmin(loginUser, result)) { return result; } @@ -81,7 +81,7 @@ public class QueueService extends BaseService { * @return queue list */ public Map queryList(User loginUser, String searchVal, Integer pageNo, Integer pageSize) { - Map result = new HashMap<>(5); + Map result = new HashMap<>(); if (checkAdmin(loginUser, result)) { return result; } @@ -110,7 +110,7 @@ public class QueueService extends BaseService { * @return create result */ public Map createQueue(User loginUser, String queue, String queueName) { - Map result = new HashMap<>(5); + Map result = new HashMap<>(); if (checkAdmin(loginUser, result)) { return result; } @@ -159,7 +159,7 @@ public class QueueService extends BaseService { * @return update result code */ public Map updateQueue(User loginUser, int id, String queue, String queueName) { - Map result = new HashMap<>(5); + Map result = new HashMap<>(); if (checkAdmin(loginUser, result)) { return result; } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java index 3cb715b964..56d40d9cab 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java @@ -272,10 +272,7 @@ public class ResourcesService extends BaseService { private boolean checkResourceExists(String fullName, int userId, int type ){ List resources = resourcesMapper.queryResourceList(fullName, userId, type); - if (resources != null && resources.size() > 0) { - return true; - } - return false; + return resources != null && resources.size() > 0; } @@ -402,7 +399,7 @@ public class ResourcesService extends BaseService { putMsg(result, Status.SUCCESS); Map dataMap = new BeanMap(resource); - Map resultMap = new HashMap<>(5); + Map resultMap = new HashMap<>(); for (Map.Entry entry: dataMap.entrySet()) { if (!Constants.CLASS.equalsIgnoreCase(entry.getKey().toString())) { resultMap.put(entry.getKey().toString(), entry.getValue()); @@ -447,7 +444,7 @@ public class ResourcesService extends BaseService { */ public Map queryResourceListPaging(User loginUser, int direcotryId, ResourceType type, String searchVal, Integer pageNo, Integer pageSize) { - HashMap result = new HashMap<>(5); + HashMap result = new HashMap<>(); Page page = new Page(pageNo, pageSize); int userId = loginUser.getId(); if (isAdmin(loginUser)) { @@ -548,7 +545,7 @@ public class ResourcesService extends BaseService { */ public Map queryResourceList(User loginUser, ResourceType type) { - Map result = new HashMap<>(5); + Map result = new HashMap<>(); int userId = loginUser.getId(); if(isAdmin(loginUser)){ @@ -571,7 +568,7 @@ public class ResourcesService extends BaseService { */ public Map queryResourceJarList(User loginUser, ResourceType type) { - Map result = new HashMap<>(5); + Map result = new HashMap<>(); int userId = loginUser.getId(); if(isAdmin(loginUser)){ userId = 0; @@ -1094,7 +1091,7 @@ public class ResourcesService extends BaseService { * @return unauthorized result code */ public Map unauthorizedUDFFunction(User loginUser, Integer userId) { - Map result = new HashMap<>(5); + Map result = new HashMap<>(); //only admin can operate if (checkAdmin(loginUser, result)) { return result; @@ -1146,7 +1143,7 @@ public class ResourcesService extends BaseService { * @return authorized result */ public Map authorizedFile(User loginUser, Integer userId) { - Map result = new HashMap<>(5); + Map result = new HashMap<>(); if (checkAdmin(loginUser, result)){ return result; } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/SchedulerService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/SchedulerService.java index 78a36c639a..93fa14872a 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/SchedulerService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/SchedulerService.java @@ -452,7 +452,7 @@ public class SchedulerService extends BaseService { * @return schedule list */ public Map queryScheduleList(User loginUser, String projectName) { - Map result = new HashMap<>(5); + Map result = new HashMap<>(); Project project = projectMapper.queryByName(projectName); // check project auth @@ -534,7 +534,7 @@ public class SchedulerService extends BaseService { */ public Map deleteScheduleById(User loginUser, String projectName, Integer scheduleId) { - Map result = new HashMap<>(5); + Map result = new HashMap<>(); Project project = projectMapper.queryByName(projectName); Map checkResult = projectService.checkProjectAndAuth(loginUser, project, projectName); @@ -583,7 +583,7 @@ public class SchedulerService extends BaseService { * @return the next five fire time */ public Map previewSchedule(User loginUser, String projectName, String schedule) { - Map result = new HashMap<>(5); + Map result = new HashMap<>(); CronExpression cronExpression; ScheduleParam scheduleParam = JSONUtils.parseObject(schedule, ScheduleParam.class); Date now = new Date(); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/SessionService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/SessionService.java index b4aab962ef..dc911f51e3 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/SessionService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/SessionService.java @@ -16,36 +16,15 @@ */ package org.apache.dolphinscheduler.api.service; +import javax.servlet.http.HttpServletRequest; -import org.apache.dolphinscheduler.api.controller.BaseController; -import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.common.utils.CollectionUtils; import org.apache.dolphinscheduler.dao.entity.Session; import org.apache.dolphinscheduler.dao.entity.User; -import org.apache.dolphinscheduler.dao.mapper.SessionMapper; -import org.apache.commons.lang.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Transactional; - -import javax.servlet.http.Cookie; -import javax.servlet.http.HttpServletRequest; -import java.util.Date; -import java.util.List; -import java.util.UUID; /** * session service */ -@Service -public class SessionService extends BaseService{ - - private static final Logger logger = LoggerFactory.getLogger(SessionService.class); - - @Autowired - private SessionMapper sessionMapper; +public interface SessionService { /** * get user session from request @@ -53,26 +32,7 @@ public class SessionService extends BaseService{ * @param request request * @return session */ - public Session getSession(HttpServletRequest request) { - String sessionId = request.getHeader(Constants.SESSION_ID); - - if(StringUtils.isBlank(sessionId)) { - Cookie cookie = getCookie(request, Constants.SESSION_ID); - - if (cookie != null) { - sessionId = cookie.getValue(); - } - } - - if(StringUtils.isBlank(sessionId)) { - return null; - } - - String ip = BaseController.getClientIpAddress(request); - logger.debug("get session: {}, ip: {}", sessionId, ip); - - return sessionMapper.selectById(sessionId); - } + Session getSession(HttpServletRequest request); /** * create session @@ -81,55 +41,7 @@ public class SessionService extends BaseService{ * @param ip ip * @return session string */ - @Transactional(rollbackFor = RuntimeException.class) - public String createSession(User user, String ip) { - Session session = null; - - // logined - List sessionList = sessionMapper.queryByUserId(user.getId()); - - Date now = new Date(); - - /** - * if you have logged in and are still valid, return directly - */ - if (CollectionUtils.isNotEmpty(sessionList)) { - // is session list greater 1 ， delete other ，get one - if (sessionList.size() > 1){ - for (int i=1 ; i < sessionList.size();i++){ - sessionMapper.deleteById(sessionList.get(i).getId()); - } - } - session = sessionList.get(0); - if (now.getTime() - session.getLastLoginTime().getTime() <= Constants.SESSION_TIME_OUT * 1000) { - /** - * updateProcessInstance the latest login time - */ - session.setLastLoginTime(now); - sessionMapper.updateById(session); - - return session.getId(); - - } else { - /** - * session expired, then delete this session first - */ - sessionMapper.deleteById(session.getId()); - } - } - - // assign new session - session = new Session(); - - session.setId(UUID.randomUUID().toString()); - session.setIp(ip); - session.setUserId(user.getId()); - session.setLastLoginTime(now); - - sessionMapper.insert(session); - - return session.getId(); - } + String createSession(User user, String ip); /** * sign out @@ -138,17 +50,5 @@ public class SessionService extends BaseService{ * @param ip no use * @param loginUser login user */ - public void signOut(String ip, User loginUser) { - try { - /** - * query session by user id and ip - */ - Session session = sessionMapper.queryByUserIdAndIp(loginUser.getId(),ip); - - //delete session - sessionMapper.deleteById(session.getId()); - }catch (Exception e){ - logger.warn("userId : {} , ip : {} , find more one session",loginUser.getId(),ip); - } - } + void signOut(String ip, User loginUser); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/TaskInstanceService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/TaskInstanceService.java index 170278e02f..695b76b2bc 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/TaskInstanceService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/TaskInstanceService.java @@ -17,8 +17,6 @@ package org.apache.dolphinscheduler.api.service; -import com.baomidou.mybatisplus.core.metadata.IPage; -import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.utils.PageInfo; import org.apache.dolphinscheduler.common.Constants; @@ -32,11 +30,20 @@ import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.mapper.ProjectMapper; import org.apache.dolphinscheduler.dao.mapper.TaskInstanceMapper; import org.apache.dolphinscheduler.service.process.ProcessService; + +import java.text.MessageFormat; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; -import java.text.MessageFormat; -import java.util.*; +import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; /** * task instance service @@ -79,11 +86,11 @@ public class TaskInstanceService extends BaseService { * @param pageSize page size * @return task list page */ - public Map queryTaskListPaging(User loginUser, String projectName, - Integer processInstanceId, String taskName, String executorName, String startDate, - String endDate, String searchVal, ExecutionStatus stateType,String host, - Integer pageNo, Integer pageSize) { - Map result = new HashMap<>(5); + public Map queryTaskListPaging(User loginUser, String projectName, + Integer processInstanceId, String taskName, String executorName, String startDate, + String endDate, String searchVal, ExecutionStatus stateType, String host, + Integer pageNo, Integer pageSize) { + Map result = new HashMap<>(); Project project = projectMapper.queryByName(projectName); Map checkResult = projectService.checkProjectAndAuth(loginUser, project, projectName); @@ -93,23 +100,23 @@ public class TaskInstanceService extends BaseService { } int[] statusArray = null; - if(stateType != null){ + if (stateType != null) { statusArray = new int[]{stateType.ordinal()}; } Date start = null; Date end = null; - try { - if(StringUtils.isNotEmpty(startDate)){ - start = DateUtils.getScheduleDate(startDate); + if (StringUtils.isNotEmpty(startDate)) { + start = DateUtils.getScheduleDate(startDate); + if (start == null) { + return generateInvalidParamRes(result, "startDate"); } - if(StringUtils.isNotEmpty( endDate)){ - end = DateUtils.getScheduleDate(endDate); + } + if (StringUtils.isNotEmpty(endDate)) { + end = DateUtils.getScheduleDate(endDate); + if (end == null) { + return generateInvalidParamRes(result, "endDate"); } - } catch (Exception e) { - result.put(Constants.STATUS, Status.REQUEST_PARAMS_NOT_VALID_ERROR); - result.put(Constants.MSG, MessageFormat.format(Status.REQUEST_PARAMS_NOT_VALID_ERROR.getMsg(), "startDate,endDate")); - return result; } Page page = new Page(pageNo, pageSize); @@ -124,18 +131,30 @@ public class TaskInstanceService extends BaseService { exclusionSet.add("taskJson"); List taskInstanceList = taskInstanceIPage.getRecords(); - for(TaskInstance taskInstance : taskInstanceList){ + for (TaskInstance taskInstance : taskInstanceList) { taskInstance.setDuration(DateUtils.differSec(taskInstance.getStartTime(), taskInstance.getEndTime())); User executor = usersService.queryUser(taskInstance.getExecutorId()); if (null != executor) { taskInstance.setExecutorName(executor.getUserName()); } } - pageInfo.setTotalCount((int)taskInstanceIPage.getTotal()); - pageInfo.setLists(CollectionUtils.getListByExclusion(taskInstanceIPage.getRecords(),exclusionSet)); + pageInfo.setTotalCount((int) taskInstanceIPage.getTotal()); + pageInfo.setLists(CollectionUtils.getListByExclusion(taskInstanceIPage.getRecords(), exclusionSet)); result.put(Constants.DATA_LIST, pageInfo); putMsg(result, Status.SUCCESS); return result; } + + /*** + * generate {@link org.apache.dolphinscheduler.api.enums.Status#REQUEST_PARAMS_NOT_VALID_ERROR} res with param name + * @param result exist result map + * @param params invalid params name + * @return update result map + */ + private Map generateInvalidParamRes(Map result, String params) { + result.put(Constants.STATUS, Status.REQUEST_PARAMS_NOT_VALID_ERROR); + result.put(Constants.MSG, MessageFormat.format(Status.REQUEST_PARAMS_NOT_VALID_ERROR.getMsg(), params)); + return result; + } } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/TenantService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/TenantService.java index a78c951d34..8e83e22a3d 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/TenantService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/TenantService.java @@ -14,338 +14,85 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.api.service; -import com.baomidou.mybatisplus.core.metadata.IPage; -import com.baomidou.mybatisplus.extension.plugins.pagination.Page; -import org.apache.dolphinscheduler.api.enums.Status; -import org.apache.dolphinscheduler.api.utils.PageInfo; import org.apache.dolphinscheduler.api.utils.Result; -import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.common.utils.CollectionUtils; -import org.apache.dolphinscheduler.common.utils.HadoopUtils; -import org.apache.dolphinscheduler.common.utils.PropertyUtils; -import org.apache.dolphinscheduler.common.utils.StringUtils; -import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; -import org.apache.dolphinscheduler.dao.entity.ProcessInstance; -import org.apache.dolphinscheduler.dao.entity.Tenant; import org.apache.dolphinscheduler.dao.entity.User; -import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper; -import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper; -import org.apache.dolphinscheduler.dao.mapper.TenantMapper; -import org.apache.dolphinscheduler.dao.mapper.UserMapper; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Transactional; -import java.util.Date; -import java.util.HashMap; -import java.util.List; import java.util.Map; /** * tenant service */ -@Service -public class TenantService extends BaseService{ +public interface TenantService { - private static final Logger logger = LoggerFactory.getLogger(TenantService.class); - - @Autowired - private TenantMapper tenantMapper; - - @Autowired - private ProcessInstanceMapper processInstanceMapper; - - @Autowired - private ProcessDefinitionMapper processDefinitionMapper; - - @Autowired - private UserMapper userMapper; - - - - /** - * create tenant - * - * - * @param loginUser login user - * @param tenantCode tenant code - * @param tenantName tenant name - * @param queueId queue id - * @param desc description - * @return create result code - * @throws Exception exception - */ - @Transactional(rollbackFor = Exception.class) - public Map createTenant(User loginUser, - String tenantCode, - String tenantName, - int queueId, - String desc) throws Exception { - - Map result = new HashMap<>(5); - result.put(Constants.STATUS, false); - if (checkAdmin(loginUser, result)) { - return result; - } - - if (checkTenantExists(tenantCode)){ - putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, tenantCode); - return result; - } - - - Tenant tenant = new Tenant(); - Date now = new Date(); - - if (!tenantCode.matches("^[0-9a-zA-Z_.-]{1,}$") || tenantCode.startsWith("-") || tenantCode.startsWith(".")){ - putMsg(result, Status.VERIFY_TENANT_CODE_ERROR); - return result; - } - tenant.setTenantCode(tenantCode); - tenant.setTenantName(tenantName); - tenant.setQueueId(queueId); - tenant.setDescription(desc); - tenant.setCreateTime(now); - tenant.setUpdateTime(now); - - // save - tenantMapper.insert(tenant); - - // if hdfs startup - if (PropertyUtils.getResUploadStartupState()){ - createTenantDirIfNotExists(tenantCode); - } - - putMsg(result, Status.SUCCESS); - - return result; -} - - - - /** - * query tenant list paging - * - * @param loginUser login user - * @param searchVal search value - * @param pageNo page number - * @param pageSize page size - * @return tenant list page - */ - public Map queryTenantList(User loginUser, String searchVal, Integer pageNo, Integer pageSize) { - - Map result = new HashMap<>(5); - if (checkAdmin(loginUser, result)) { - return result; - } - - Page page = new Page(pageNo, pageSize); - IPage tenantIPage = tenantMapper.queryTenantPaging(page, searchVal); - PageInfo pageInfo = new PageInfo<>(pageNo, pageSize); - pageInfo.setTotalCount((int)tenantIPage.getTotal()); - pageInfo.setLists(tenantIPage.getRecords()); - result.put(Constants.DATA_LIST, pageInfo); - - putMsg(result, Status.SUCCESS); - - return result; - } - - /** - * updateProcessInstance tenant - * - * @param loginUser login user - * @param id tennat id - * @param tenantCode tennat code - * @param tenantName tennat name - * @param queueId queue id - * @param desc description - * @return update result code - * @throws Exception exception - */ - public Map updateTenant(User loginUser,int id,String tenantCode, String tenantName, int queueId, String desc) throws Exception { - - Map result = new HashMap<>(5); - result.put(Constants.STATUS, false); - - if (checkAdmin(loginUser, result)) { - return result; - } - - Tenant tenant = tenantMapper.queryById(id); - - if (tenant == null){ - putMsg(result, Status.TENANT_NOT_EXIST); - return result; - } - - // updateProcessInstance tenant /** - * if the tenant code is modified, the original resource needs to be copied to the new tenant. + * create tenant + * + * @param loginUser login user + * @param tenantCode tenant code + * @param tenantName tenant name + * @param queueId queue id + * @param desc description + * @return create result code + * @throws Exception exception */ - if (!tenant.getTenantCode().equals(tenantCode)){ - if (checkTenantExists(tenantCode)){ - // if hdfs startup - if (PropertyUtils.getResUploadStartupState()){ - String resourcePath = HadoopUtils.getHdfsDataBasePath() + "/" + tenantCode + "/resources"; - String udfsPath = HadoopUtils.getHdfsUdfDir(tenantCode); - //init hdfs resource - HadoopUtils.getInstance().mkdir(resourcePath); - HadoopUtils.getInstance().mkdir(udfsPath); - } - }else { - putMsg(result, Status.TENANT_CODE_HAS_ALREADY_EXISTS); - return result; - } - } + Map createTenant(User loginUser, + String tenantCode, + String tenantName, + int queueId, + String desc) throws Exception; - Date now = new Date(); + /** + * query tenant list paging + * + * @param loginUser login user + * @param searchVal search value + * @param pageNo page number + * @param pageSize page size + * @return tenant list page + */ + Map queryTenantList(User loginUser, String searchVal, Integer pageNo, Integer pageSize); - if (StringUtils.isNotEmpty(tenantCode)){ - tenant.setTenantCode(tenantCode); - } + /** + * updateProcessInstance tenant + * + * @param loginUser login user + * @param id tennat id + * @param tenantCode tennat code + * @param tenantName tennat name + * @param queueId queue id + * @param desc description + * @return update result code + * @throws Exception exception + */ + Map updateTenant(User loginUser, int id, String tenantCode, String tenantName, int queueId, + String desc) throws Exception; - if (StringUtils.isNotEmpty(tenantName)){ - tenant.setTenantName(tenantName); - } + /** + * delete tenant + * + * @param loginUser login user + * @param id tenant id + * @return delete result code + * @throws Exception exception + */ + Map deleteTenantById(User loginUser, int id) throws Exception; - if (queueId != 0){ - tenant.setQueueId(queueId); - } - tenant.setDescription(desc); - tenant.setUpdateTime(now); - tenantMapper.updateById(tenant); + /** + * query tenant list + * + * @param loginUser login user + * @return tenant list + */ + Map queryTenantList(User loginUser); - result.put(Constants.STATUS, Status.SUCCESS); - result.put(Constants.MSG, Status.SUCCESS.getMsg()); - return result; - } - - /** - * delete tenant - * - * @param loginUser login user - * @param id tenant id - * @return delete result code - * @throws Exception exception - */ - @Transactional(rollbackFor = Exception.class) - public Map deleteTenantById(User loginUser, int id) throws Exception { - Map result = new HashMap<>(5); - - if (checkAdmin(loginUser, result)) { - return result; - } - - Tenant tenant = tenantMapper.queryById(id); - if (tenant == null){ - putMsg(result, Status.TENANT_NOT_EXIST); - return result; - } - - List processInstances = getProcessInstancesByTenant(tenant); - if(CollectionUtils.isNotEmpty(processInstances)){ - putMsg(result, Status.DELETE_TENANT_BY_ID_FAIL, processInstances.size()); - return result; - } - - List processDefinitions = processDefinitionMapper.queryDefinitionListByTenant(tenant.getId()); - if(CollectionUtils.isNotEmpty(processDefinitions)){ - putMsg(result, Status.DELETE_TENANT_BY_ID_FAIL_DEFINES, processDefinitions.size()); - return result; - } - - List userList = userMapper.queryUserListByTenant(tenant.getId()); - if(CollectionUtils.isNotEmpty(userList)){ - putMsg(result, Status.DELETE_TENANT_BY_ID_FAIL_USERS, userList.size()); - return result; - } - - // if resource upload startup - if (PropertyUtils.getResUploadStartupState()){ - String tenantPath = HadoopUtils.getHdfsDataBasePath() + "/" + tenant.getTenantCode(); - - if (HadoopUtils.getInstance().exists(tenantPath)){ - HadoopUtils.getInstance().delete(tenantPath, true); - } - } - - tenantMapper.deleteById(id); - processInstanceMapper.updateProcessInstanceByTenantId(id, -1); - putMsg(result, Status.SUCCESS); - return result; - } - - private List getProcessInstancesByTenant(Tenant tenant) { - return processInstanceMapper.queryByTenantIdAndStatus(tenant.getId(), org.apache.dolphinscheduler.common.Constants.NOT_TERMINATED_STATES); - } - - /** - * query tenant list - * - * @param loginUser login user - * @return tenant list - */ - public Map queryTenantList(User loginUser) { - - Map result = new HashMap<>(5); - - List resourceList = tenantMapper.selectList(null); - result.put(Constants.DATA_LIST, resourceList); - putMsg(result, Status.SUCCESS); - - return result; - } - - /** - * query tenant list via tenant code - * @param tenantCode tenant code - * @return tenant list - */ - public Map queryTenantList(String tenantCode) { - Map result = new HashMap<>(5); - - List resourceList = tenantMapper.queryByTenantCode(tenantCode); - if (CollectionUtils.isNotEmpty(resourceList)) { - result.put(Constants.DATA_LIST, resourceList); - putMsg(result, Status.SUCCESS); - } else { - putMsg(result, Status.TENANT_NOT_EXIST); - } - - return result; - } - - /** - * verify tenant code - * - * @param tenantCode tenant code - * @return true if tenant code can user, otherwise return false - */ - public Result verifyTenantCode(String tenantCode) { - Result result = new Result(); - if (checkTenantExists(tenantCode)) { - logger.error("tenant {} has exist, can't create again.", tenantCode); - putMsg(result, Status.TENANT_NAME_EXIST, tenantCode); - } else { - putMsg(result, Status.SUCCESS); - } - return result; - } - - - /** - * check tenant exists - * - * @param tenantCode tenant code - * @return ture if the tenant code exists, otherwise return false - */ - private boolean checkTenantExists(String tenantCode) { - List tenants = tenantMapper.queryByTenantCode(tenantCode); - return CollectionUtils.isNotEmpty(tenants); - } + /** + * verify tenant code + * + * @param tenantCode tenant code + * @return true if tenant code can user, otherwise return false + */ + Result verifyTenantCode(String tenantCode); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UdfFuncService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UdfFuncService.java index bb92c9ccf7..04f641f279 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UdfFuncService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UdfFuncService.java @@ -136,10 +136,7 @@ public class UdfFuncService extends BaseService{ */ private boolean checkUdfFuncNameExists(String name){ List resource = udfFuncMapper.queryUdfByIdStr(null, name); - if(resource != null && resource.size() > 0){ - return true; - } - return false; + return resource != null && resource.size() > 0; } @@ -151,7 +148,7 @@ public class UdfFuncService extends BaseService{ */ public Map queryUdfFuncDetail(int id) { - Map result = new HashMap<>(5); + Map result = new HashMap<>(); UdfFunc udfFunc = udfFuncMapper.selectById(id); if (udfFunc == null) { putMsg(result, Status.RESOURCE_NOT_EXIST); @@ -247,7 +244,7 @@ public class UdfFuncService extends BaseService{ * @return udf function list page */ public Map queryUdfFuncListPaging(User loginUser, String searchVal, Integer pageNo, Integer pageSize) { - Map result = new HashMap<>(5); + Map result = new HashMap<>(); PageInfo pageInfo = new PageInfo(pageNo, pageSize); @@ -286,7 +283,7 @@ public class UdfFuncService extends BaseService{ * @return resource list */ public Map queryResourceList(User loginUser, Integer type) { - Map result = new HashMap<>(5); + Map result = new HashMap<>(); List udfFuncList = udfFuncMapper.getUdfFuncByType(loginUser.getId(), type); result.put(Constants.DATA_LIST, udfFuncList); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UsersService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UsersService.java index cbd795cce4..89038ad09f 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UsersService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UsersService.java @@ -26,6 +26,7 @@ import org.apache.dolphinscheduler.api.utils.CheckUtils; import org.apache.dolphinscheduler.api.utils.PageInfo; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.enums.Flag; import org.apache.dolphinscheduler.common.enums.ResourceType; import org.apache.dolphinscheduler.common.enums.UserType; import org.apache.dolphinscheduler.common.utils.*; @@ -39,6 +40,7 @@ import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import java.io.IOException; +import java.text.MessageFormat; import java.util.*; import java.util.stream.Collectors; @@ -101,7 +103,7 @@ public class UsersService extends BaseService { String queue, int state) throws Exception { - Map result = new HashMap<>(5); + Map result = new HashMap<>(); //check all user params String msg = this.checkUserParams(userName, userPassword, email, phone); @@ -229,7 +231,7 @@ public class UsersService extends BaseService { * @return user list page */ public Map queryUserList(User loginUser, String searchVal, Integer pageNo, Integer pageSize) { - Map result = new HashMap<>(5); + Map result = new HashMap<>(); if (check(result, !isAdmin(loginUser), Status.USER_NO_OPERATION_PERM)) { return result; @@ -269,7 +271,7 @@ public class UsersService extends BaseService { String phone, String queue, int state) throws Exception { - Map result = new HashMap<>(5); + Map result = new HashMap<>(); result.put(Constants.STATUS, false); User user = userMapper.selectById(userId); @@ -392,7 +394,7 @@ public class UsersService extends BaseService { * @throws Exception exception when operate hdfs */ public Map deleteUserById(User loginUser, int id) throws Exception { - Map result = new HashMap<>(5); + Map result = new HashMap<>(); //only admin can operate if (!isAdmin(loginUser)) { putMsg(result, Status.USER_NO_OPERATION_PERM, id); @@ -432,7 +434,7 @@ public class UsersService extends BaseService { */ @Transactional(rollbackFor = RuntimeException.class) public Map grantProject(User loginUser, int userId, String projectIds) { - Map result = new HashMap<>(5); + Map result = new HashMap<>(); result.put(Constants.STATUS, false); //only admin can operate @@ -482,7 +484,7 @@ public class UsersService extends BaseService { */ @Transactional(rollbackFor = RuntimeException.class) public Map grantResources(User loginUser, int userId, String resourceIds) { - Map result = new HashMap<>(5); + Map result = new HashMap<>(); //only admin can operate if (check(result, !isAdmin(loginUser), Status.USER_NO_OPERATION_PERM)) { return result; @@ -579,7 +581,7 @@ public class UsersService extends BaseService { */ @Transactional(rollbackFor = RuntimeException.class) public Map grantUDFFunction(User loginUser, int userId, String udfIds) { - Map result = new HashMap<>(5); + Map result = new HashMap<>(); //only admin can operate if (check(result, !isAdmin(loginUser), Status.USER_NO_OPERATION_PERM)) { @@ -626,7 +628,7 @@ public class UsersService extends BaseService { */ @Transactional(rollbackFor = RuntimeException.class) public Map grantDataSource(User loginUser, int userId, String datasourceIds) { - Map result = new HashMap<>(5); + Map result = new HashMap<>(); result.put(Constants.STATUS, false); //only admin can operate @@ -706,7 +708,7 @@ public class UsersService extends BaseService { * @return user list */ public Map queryAllGeneralUsers(User loginUser) { - Map result = new HashMap<>(5); + Map result = new HashMap<>(); //only admin can operate if (check(result, !isAdmin(loginUser), Status.USER_NO_OPERATION_PERM)) { return result; @@ -727,7 +729,7 @@ public class UsersService extends BaseService { * @return user list */ public Map queryUserList(User loginUser) { - Map result = new HashMap<>(5); + Map result = new HashMap<>(); //only admin can operate if (check(result, !isAdmin(loginUser), Status.USER_NO_OPERATION_PERM)) { return result; @@ -771,7 +773,7 @@ public class UsersService extends BaseService { */ public Map unauthorizedUser(User loginUser, Integer alertgroupId) { - Map result = new HashMap<>(5); + Map result = new HashMap<>(); //only admin can operate if (check(result, !isAdmin(loginUser), Status.USER_NO_OPERATION_PERM)) { return result; @@ -807,7 +809,7 @@ public class UsersService extends BaseService { * @return authorized result code */ public Map authorizedUser(User loginUser, Integer alertgroupId) { - Map result = new HashMap<>(5); + Map result = new HashMap<>(); //only admin can operate if (check(result, !isAdmin(loginUser), Status.USER_NO_OPERATION_PERM)) { return result; @@ -917,10 +919,11 @@ public class UsersService extends BaseService { * @param repeatPassword repeat password * @param email email * @return register result code + * @throws Exception exception */ @Transactional(rollbackFor = RuntimeException.class) public Map registerUser(String userName, String userPassword, String repeatPassword, String email) { - Map result = new HashMap<>(5); + Map result = new HashMap<>(); //check user params String msg = this.checkUserParams(userName, userPassword, email, ""); @@ -934,10 +937,100 @@ public class UsersService extends BaseService { putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, "two passwords are not same"); return result; } - - createUser(userName, userPassword, email, 1, "", "", 0); + User user = createUser(userName, userPassword, email, 1, "", "", Flag.NO.ordinal()); putMsg(result, Status.SUCCESS); + result.put(Constants.DATA_LIST, user); return result; } + /** + * activate user, only system admin have permission, change user state code 0 to 1 + * + * @param loginUser login user + * @param userName user name + * @return create result code + */ + public Map activateUser(User loginUser, String userName) { + Map result = new HashMap<>(); + result.put(Constants.STATUS, false); + + if (!isAdmin(loginUser)) { + putMsg(result, Status.USER_NO_OPERATION_PERM); + return result; + } + + if (!CheckUtils.checkUserName(userName)){ + putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, userName); + return result; + } + + User user = userMapper.queryByUserNameAccurately(userName); + + if (user == null) { + putMsg(result, Status.USER_NOT_EXIST, userName); + return result; + } + + if (user.getState() != Flag.NO.ordinal()) { + putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, userName); + return result; + } + + user.setState(Flag.YES.ordinal()); + Date now = new Date(); + user.setUpdateTime(now); + userMapper.updateById(user); + User responseUser = userMapper.queryByUserNameAccurately(userName); + putMsg(result, Status.SUCCESS); + result.put(Constants.DATA_LIST, responseUser); + return result; + } + + /** + * activate user, only system admin have permission, change users state code 0 to 1 + * + * @param loginUser login user + * @param userNames user name + * @return create result code + */ + public Map batchActivateUser(User loginUser, List userNames) { + Map result = new HashMap<>(); + + if (!isAdmin(loginUser)) { + putMsg(result, Status.USER_NO_OPERATION_PERM); + return result; + } + + int totalSuccess = 0; + List successUserNames = new ArrayList<>(); + Map successRes = new HashMap<>(); + int totalFailed = 0; + List> failedInfo = new ArrayList<>(); + Map failedRes = new HashMap<>(); + for (String userName : userNames) { + Map tmpResult = activateUser(loginUser, userName); + if (tmpResult.get(Constants.STATUS) != Status.SUCCESS) { + totalFailed++; + Map failedBody = new HashMap<>(); + failedBody.put("userName", userName); + Status status = (Status) tmpResult.get(Constants.STATUS); + String errorMessage = MessageFormat.format(status.getMsg(), userName); + failedBody.put("msg", errorMessage); + failedInfo.add(failedBody); + } else { + totalSuccess++; + successUserNames.add(userName); + } + } + successRes.put("sum", totalSuccess); + successRes.put("userName", successUserNames); + failedRes.put("sum", totalFailed); + failedRes.put("info", failedInfo); + Map res = new HashMap<>(); + res.put("success", successRes); + res.put("failed", failedRes); + putMsg(result, Status.SUCCESS); + result.put(Constants.DATA_LIST, res); + return result; + } } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/WorkerGroupService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/WorkerGroupService.java index 374fd6e718..95257e8c8a 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/WorkerGroupService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/WorkerGroupService.java @@ -63,7 +63,7 @@ public class WorkerGroupService extends BaseService { // list to index Integer toIndex = (pageNo - 1) * pageSize + pageSize; - Map result = new HashMap<>(5); + Map result = new HashMap<>(); if (checkAdmin(loginUser, result)) { return result; } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AccessTokenServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AccessTokenServiceImpl.java new file mode 100644 index 0000000000..54151d902f --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AccessTokenServiceImpl.java @@ -0,0 +1,186 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.api.service.impl; + +import org.apache.dolphinscheduler.api.enums.Status; +import org.apache.dolphinscheduler.api.service.AccessTokenService; +import org.apache.dolphinscheduler.api.service.BaseService; +import org.apache.dolphinscheduler.api.utils.PageInfo; +import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.enums.UserType; +import org.apache.dolphinscheduler.common.utils.DateUtils; +import org.apache.dolphinscheduler.common.utils.EncryptionUtils; +import org.apache.dolphinscheduler.dao.entity.AccessToken; +import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.dao.mapper.AccessTokenMapper; + +import java.util.Date; +import java.util.HashMap; +import java.util.Map; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; + +/** + * access token service impl + */ +@Service +public class AccessTokenServiceImpl extends BaseService implements AccessTokenService { + + private static final Logger logger = LoggerFactory.getLogger(AccessTokenServiceImpl.class); + + @Autowired + private AccessTokenMapper accessTokenMapper; + + /** + * query access token list + * + * @param loginUser login user + * @param searchVal search value + * @param pageNo page number + * @param pageSize page size + * @return token list for page number and page size + */ + public Map queryAccessTokenList(User loginUser, String searchVal, Integer pageNo, Integer pageSize) { + Map result = new HashMap<>(5); + + PageInfo pageInfo = new PageInfo<>(pageNo, pageSize); + Page page = new Page<>(pageNo, pageSize); + int userId = loginUser.getId(); + if (loginUser.getUserType() == UserType.ADMIN_USER) { + userId = 0; + } + IPage accessTokenList = accessTokenMapper.selectAccessTokenPage(page, searchVal, userId); + pageInfo.setTotalCount((int) accessTokenList.getTotal()); + pageInfo.setLists(accessTokenList.getRecords()); + result.put(Constants.DATA_LIST, pageInfo); + putMsg(result, Status.SUCCESS); + + return result; + } + + /** + * create token + * + * @param userId token for user + * @param expireTime token expire time + * @param token token string + * @return create result code + */ + public Map createToken(int userId, String expireTime, String token) { + Map result = new HashMap<>(5); + + if (userId <= 0) { + throw new IllegalArgumentException("User id should not less than or equals to 0."); + } + AccessToken accessToken = new AccessToken(); + accessToken.setUserId(userId); + accessToken.setExpireTime(DateUtils.stringToDate(expireTime)); + accessToken.setToken(token); + accessToken.setCreateTime(new Date()); + accessToken.setUpdateTime(new Date()); + + // insert + int insert = accessTokenMapper.insert(accessToken); + + if (insert > 0) { + putMsg(result, Status.SUCCESS); + } else { + putMsg(result, Status.CREATE_ACCESS_TOKEN_ERROR); + } + + return result; + } + + /** + * generate token + * + * @param userId token for user + * @param expireTime token expire time + * @return token string + */ + public Map generateToken(int userId, String expireTime) { + Map result = new HashMap<>(5); + String token = EncryptionUtils.getMd5(userId + expireTime + System.currentTimeMillis()); + result.put(Constants.DATA_LIST, token); + putMsg(result, Status.SUCCESS); + return result; + } + + /** + * delete access token + * + * @param loginUser login user + * @param id token id + * @return delete result code + */ + public Map delAccessTokenById(User loginUser, int id) { + Map result = new HashMap<>(5); + + AccessToken accessToken = accessTokenMapper.selectById(id); + + if (accessToken == null) { + logger.error("access token not exist, access token id {}", id); + putMsg(result, Status.ACCESS_TOKEN_NOT_EXIST); + return result; + } + + if (loginUser.getId() != accessToken.getUserId() && + loginUser.getUserType() != UserType.ADMIN_USER) { + putMsg(result, Status.USER_NO_OPERATION_PERM); + return result; + } + + accessTokenMapper.deleteById(id); + putMsg(result, Status.SUCCESS); + return result; + } + + /** + * update token by id + * + * @param id token id + * @param userId token for user + * @param expireTime token expire time + * @param token token string + * @return update result code + */ + public Map updateToken(int id, int userId, String expireTime, String token) { + Map result = new HashMap<>(5); + + AccessToken accessToken = accessTokenMapper.selectById(id); + if (accessToken == null) { + logger.error("access token not exist, access token id {}", id); + putMsg(result, Status.ACCESS_TOKEN_NOT_EXIST); + return result; + } + accessToken.setUserId(userId); + accessToken.setExpireTime(DateUtils.stringToDate(expireTime)); + accessToken.setToken(token); + accessToken.setUpdateTime(new Date()); + + accessTokenMapper.updateById(accessToken); + + putMsg(result, Status.SUCCESS); + return result; + } +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataAnalysisServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataAnalysisServiceImpl.java new file mode 100644 index 0000000000..21313b96d3 --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataAnalysisServiceImpl.java @@ -0,0 +1,384 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.api.service.impl; + + +import org.apache.dolphinscheduler.api.dto.CommandStateCount; +import org.apache.dolphinscheduler.api.dto.DefineUserDto; +import org.apache.dolphinscheduler.api.dto.TaskCountDto; +import org.apache.dolphinscheduler.api.enums.Status; +import org.apache.dolphinscheduler.api.service.BaseService; +import org.apache.dolphinscheduler.api.service.DataAnalysisService; +import org.apache.dolphinscheduler.api.service.ProjectService; +import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.enums.CommandType; +import org.apache.dolphinscheduler.common.enums.UserType; +import org.apache.dolphinscheduler.common.utils.DateUtils; +import org.apache.dolphinscheduler.common.utils.StringUtils; +import org.apache.dolphinscheduler.common.utils.TriFunction; +import org.apache.dolphinscheduler.dao.entity.CommandCount; +import org.apache.dolphinscheduler.dao.entity.DefinitionGroupByUser; +import org.apache.dolphinscheduler.dao.entity.ExecuteStatusCount; +import org.apache.dolphinscheduler.dao.entity.Project; +import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.dao.mapper.CommandMapper; +import org.apache.dolphinscheduler.dao.mapper.ErrorCommandMapper; +import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper; +import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper; +import org.apache.dolphinscheduler.dao.mapper.ProjectMapper; +import org.apache.dolphinscheduler.dao.mapper.TaskInstanceMapper; +import org.apache.dolphinscheduler.service.process.ProcessService; + +import java.text.MessageFormat; +import java.util.ArrayList; +import java.util.Date; +import java.util.EnumMap; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +/** + * data analysis service impl + */ +@Service +public class DataAnalysisServiceImpl extends BaseService implements DataAnalysisService { + + private static final Logger logger = LoggerFactory.getLogger(DataAnalysisServiceImpl.class); + + @Autowired + private ProjectMapper projectMapper; + + @Autowired + private ProjectService projectService; + + @Autowired + private ProcessInstanceMapper processInstanceMapper; + + @Autowired + private ProcessDefinitionMapper processDefinitionMapper; + + @Autowired + private CommandMapper commandMapper; + + @Autowired + private ErrorCommandMapper errorCommandMapper; + + @Autowired + private TaskInstanceMapper taskInstanceMapper; + + @Autowired + private ProcessService processService; + + private static final String COMMAND_STATE = "commandState"; + + private static final String ERROR_COMMAND_STATE = "errorCommandState"; + + /** + * statistical task instance status data + * + * @param loginUser login user + * @param projectId project id + * @param startDate start date + * @param endDate end date + * @return task state count data + */ + public Map countTaskStateByProject(User loginUser, int projectId, String startDate, String endDate) { + + return countStateByProject( + loginUser, + projectId, + startDate, + endDate, + (start, end, projectIds) -> this.taskInstanceMapper.countTaskInstanceStateByUser(start, end, projectIds)); + } + + /** + * statistical process instance status data + * + * @param loginUser login user + * @param projectId project id + * @param startDate start date + * @param endDate end date + * @return process instance state count data + */ + public Map countProcessInstanceStateByProject(User loginUser, int projectId, String startDate, String endDate) { + return this.countStateByProject( + loginUser, + projectId, + startDate, + endDate, + (start, end, projectIds) -> this.processInstanceMapper.countInstanceStateByUser(start, end, projectIds)); + } + + private Map countStateByProject(User loginUser, int projectId, String startDate, String endDate + , TriFunction> instanceStateCounter) { + Map result = new HashMap<>(5); + boolean checkProject = checkProject(loginUser, projectId, result); + if (!checkProject) { + return result; + } + + Date start; + Date end; + try { + start = DateUtils.getScheduleDate(startDate); + end = DateUtils.getScheduleDate(endDate); + } catch (Exception e) { + logger.error(e.getMessage(), e); + putErrorRequestParamsMsg(result); + return result; + } + Integer[] projectIdArray = getProjectIdsArrays(loginUser, projectId); + List processInstanceStateCounts = + instanceStateCounter.apply(start, end, projectIdArray); + + if (processInstanceStateCounts != null) { + TaskCountDto taskCountResult = new TaskCountDto(processInstanceStateCounts); + result.put(Constants.DATA_LIST, taskCountResult); + putMsg(result, Status.SUCCESS); + } + return result; + } + + + /** + * statistics the process definition quantities of certain person + * + * @param loginUser login user + * @param projectId project id + * @return definition count data + */ + public Map countDefinitionByUser(User loginUser, int projectId) { + Map result = new HashMap<>(); + + + Integer[] projectIdArray = getProjectIdsArrays(loginUser, projectId); + List defineGroupByUsers = processDefinitionMapper.countDefinitionGroupByUser( + loginUser.getId(), projectIdArray, isAdmin(loginUser)); + + DefineUserDto dto = new DefineUserDto(defineGroupByUsers); + result.put(Constants.DATA_LIST, dto); + putMsg(result, Status.SUCCESS); + return result; + } + + + /** + * statistical command status data + * + * @param loginUser login user + * @param projectId project id + * @param startDate start date + * @param endDate end date + * @return command state count data + */ + public Map countCommandState(User loginUser, int projectId, String startDate, String endDate) { + + Map result = new HashMap<>(5); + boolean checkProject = checkProject(loginUser, projectId, result); + if (!checkProject) { + return result; + } + + /** + * find all the task lists in the project under the user + * statistics based on task status execution, failure, completion, wait, total + */ + Date start = null; + Date end = null; + + if (startDate != null && endDate != null) { + try { + start = DateUtils.getScheduleDate(startDate); + end = DateUtils.getScheduleDate(endDate); + } catch (Exception e) { + logger.error(e.getMessage(), e); + putErrorRequestParamsMsg(result); + return result; + } + } + + + Integer[] projectIdArray = getProjectIdsArrays(loginUser, projectId); + // count command state + List commandStateCounts = + commandMapper.countCommandState( + loginUser.getId(), + start, + end, + projectIdArray); + + // count error command state + List errorCommandStateCounts = + errorCommandMapper.countCommandState( + start, end, projectIdArray); + + // enumMap + Map> dataMap = new EnumMap<>(CommandType.class); + + Map commonCommand = new HashMap<>(); + commonCommand.put(COMMAND_STATE, 0); + commonCommand.put(ERROR_COMMAND_STATE, 0); + + + // init data map + /** + * START_PROCESS, START_CURRENT_TASK_PROCESS, RECOVER_TOLERANCE_FAULT_PROCESS, RECOVER_SUSPENDED_PROCESS, + START_FAILURE_TASK_PROCESS,COMPLEMENT_DATA,SCHEDULER, REPEAT_RUNNING,PAUSE,STOP,RECOVER_WAITTING_THREAD; + */ + dataMap.put(CommandType.START_PROCESS, commonCommand); + dataMap.put(CommandType.START_CURRENT_TASK_PROCESS, commonCommand); + dataMap.put(CommandType.RECOVER_TOLERANCE_FAULT_PROCESS, commonCommand); + dataMap.put(CommandType.RECOVER_SUSPENDED_PROCESS, commonCommand); + dataMap.put(CommandType.START_FAILURE_TASK_PROCESS, commonCommand); + dataMap.put(CommandType.COMPLEMENT_DATA, commonCommand); + dataMap.put(CommandType.SCHEDULER, commonCommand); + dataMap.put(CommandType.REPEAT_RUNNING, commonCommand); + dataMap.put(CommandType.PAUSE, commonCommand); + dataMap.put(CommandType.STOP, commonCommand); + dataMap.put(CommandType.RECOVER_WAITTING_THREAD, commonCommand); + + // put command state + for (CommandCount executeStatusCount : commandStateCounts) { + Map commandStateCountsMap = new HashMap<>(dataMap.get(executeStatusCount.getCommandType())); + commandStateCountsMap.put(COMMAND_STATE, executeStatusCount.getCount()); + dataMap.put(executeStatusCount.getCommandType(), commandStateCountsMap); + } + + // put error command state + for (CommandCount errorExecutionStatus : errorCommandStateCounts) { + Map errorCommandStateCountsMap = new HashMap<>(dataMap.get(errorExecutionStatus.getCommandType())); + errorCommandStateCountsMap.put(ERROR_COMMAND_STATE, errorExecutionStatus.getCount()); + dataMap.put(errorExecutionStatus.getCommandType(), errorCommandStateCountsMap); + } + + List list = new ArrayList<>(); + for (Map.Entry> next : dataMap.entrySet()) { + CommandStateCount commandStateCount = new CommandStateCount(next.getValue().get(ERROR_COMMAND_STATE), + next.getValue().get(COMMAND_STATE), next.getKey()); + list.add(commandStateCount); + } + + result.put(Constants.DATA_LIST, list); + putMsg(result, Status.SUCCESS); + return result; + } + + private Integer[] getProjectIdsArrays(User loginUser, int projectId) { + List projectIds = new ArrayList<>(); + if (projectId != 0) { + projectIds.add(projectId); + } else if (loginUser.getUserType() == UserType.GENERAL_USER) { + projectIds = processService.getProjectIdListHavePerm(loginUser.getId()); + if (projectIds.isEmpty()) { + projectIds.add(0); + } + } + return projectIds.toArray(new Integer[0]); + } + + /** + * count queue state + * + * @param loginUser login user + * @param projectId project id + * @return queue state count data + */ + public Map countQueueState(User loginUser, int projectId) { + Map result = new HashMap<>(5); + + boolean checkProject = checkProject(loginUser, projectId, result); + if (!checkProject) { + return result; + } + + // TODO tasksQueueList and tasksKillList is never updated. + List tasksQueueList = new ArrayList<>(); + List tasksKillList = new ArrayList<>(); + + Map dataMap = new HashMap<>(); + if (loginUser.getUserType() == UserType.ADMIN_USER) { + dataMap.put("taskQueue", tasksQueueList.size()); + dataMap.put("taskKill", tasksKillList.size()); + + result.put(Constants.DATA_LIST, dataMap); + putMsg(result, Status.SUCCESS); + return result; + } + + int[] tasksQueueIds = new int[tasksQueueList.size()]; + int[] tasksKillIds = new int[tasksKillList.size()]; + + int i = 0; + for (String taskQueueStr : tasksQueueList) { + if (StringUtils.isNotEmpty(taskQueueStr)) { + String[] splits = taskQueueStr.split("_"); + if (splits.length >= 4) { + tasksQueueIds[i++] = Integer.parseInt(splits[3]); + } + } + } + + i = 0; + for (String taskKillStr : tasksKillList) { + if (StringUtils.isNotEmpty(taskKillStr)) { + String[] splits = taskKillStr.split("-"); + if (splits.length == 2) { + tasksKillIds[i++] = Integer.parseInt(splits[1]); + } + } + } + Integer taskQueueCount = 0; + Integer taskKillCount = 0; + + Integer[] projectIds = getProjectIdsArrays(loginUser, projectId); + if (tasksQueueIds.length != 0) { + taskQueueCount = taskInstanceMapper.countTask( + projectIds, + tasksQueueIds); + } + + if (tasksKillIds.length != 0) { + taskKillCount = taskInstanceMapper.countTask(projectIds, tasksKillIds); + } + + dataMap.put("taskQueue", taskQueueCount); + dataMap.put("taskKill", taskKillCount); + + result.put(Constants.DATA_LIST, dataMap); + putMsg(result, Status.SUCCESS); + return result; + } + + private boolean checkProject(User loginUser, int projectId, Map result) { + if (projectId != 0) { + Project project = projectMapper.selectById(projectId); + return projectService.hasProjectAndPerm(loginUser, project, result); + } + return true; + } + + private void putErrorRequestParamsMsg(Map result) { + result.put(Constants.STATUS, Status.REQUEST_PARAMS_NOT_VALID_ERROR); + result.put(Constants.MSG, MessageFormat.format(Status.REQUEST_PARAMS_NOT_VALID_ERROR.getMsg(), "startDate,endDate")); + } +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/LoggerServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/LoggerServiceImpl.java new file mode 100644 index 0000000000..c71f2980f5 --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/LoggerServiceImpl.java @@ -0,0 +1,146 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.api.service.impl; + +import org.apache.dolphinscheduler.api.enums.Status; +import org.apache.dolphinscheduler.api.exceptions.ServiceException; +import org.apache.dolphinscheduler.api.service.LoggerService; +import org.apache.dolphinscheduler.api.utils.Result; +import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.utils.StringUtils; +import org.apache.dolphinscheduler.dao.entity.TaskInstance; +import org.apache.dolphinscheduler.remote.utils.Host; +import org.apache.dolphinscheduler.service.log.LogClientService; +import org.apache.dolphinscheduler.service.process.ProcessService; + +import org.apache.commons.lang.ArrayUtils; + +import java.nio.charset.StandardCharsets; +import java.util.Objects; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +/** + * log service + */ +@Service +public class LoggerServiceImpl implements LoggerService { + + private static final Logger logger = LoggerFactory.getLogger(LoggerServiceImpl.class); + + private static final String LOG_HEAD_FORMAT = "[LOG-PATH]: %s, [HOST]: %s%s"; + + @Autowired + private ProcessService processService; + + private LogClientService logClient; + + @PostConstruct + public void init() { + if (Objects.isNull(this.logClient)) { + this.logClient = new LogClientService(); + } + } + + @PreDestroy + public void close() { + if (Objects.nonNull(this.logClient) && this.logClient.isRunning()) { + logClient.close(); + } + } + + /** + * view log + * + * @param taskInstId task instance id + * @param skipLineNum skip line number + * @param limit limit + * @return log string data + */ + @SuppressWarnings("unchecked") + public Result queryLog(int taskInstId, int skipLineNum, int limit) { + + TaskInstance taskInstance = processService.findTaskInstanceById(taskInstId); + + if (taskInstance == null || StringUtils.isBlank(taskInstance.getHost())) { + return Result.error(Status.TASK_INSTANCE_NOT_FOUND); + } + + String host = getHost(taskInstance.getHost()); + + Result result = new Result<>(Status.SUCCESS.getCode(), Status.SUCCESS.getMsg()); + + logger.info("log host : {} , logPath : {} , logServer port : {}", host, taskInstance.getLogPath(), + Constants.RPC_PORT); + + StringBuilder log = new StringBuilder(); + if (skipLineNum == 0) { + String head = String.format(LOG_HEAD_FORMAT, + taskInstance.getLogPath(), + host, + Constants.SYSTEM_LINE_SEPARATOR); + log.append(head); + } + + log.append(logClient + .rollViewLog(host, Constants.RPC_PORT, taskInstance.getLogPath(), skipLineNum, limit)); + + result.setData(log.toString()); + return result; + } + + + /** + * get log size + * + * @param taskInstId task instance id + * @return log byte array + */ + public byte[] getLogBytes(int taskInstId) { + TaskInstance taskInstance = processService.findTaskInstanceById(taskInstId); + if (taskInstance == null || StringUtils.isBlank(taskInstance.getHost())) { + throw new ServiceException("task instance is null or host is null"); + } + String host = getHost(taskInstance.getHost()); + byte[] head = String.format(LOG_HEAD_FORMAT, + taskInstance.getLogPath(), + host, + Constants.SYSTEM_LINE_SEPARATOR).getBytes(StandardCharsets.UTF_8); + return ArrayUtils.addAll(head, + logClient.getLogBytes(host, Constants.RPC_PORT, taskInstance.getLogPath())); + } + + + /** + * get host + * + * @param address address + * @return old version return true ,otherwise return false + */ + private String getHost(String address) { + if (Boolean.TRUE.equals(Host.isOldVersion(address))) { + return address; + } + return Host.of(address).getIp(); + } +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java new file mode 100644 index 0000000000..a5e297072c --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java @@ -0,0 +1,1731 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.service.impl; + +import static org.apache.dolphinscheduler.common.Constants.CMDPARAM_SUB_PROCESS_DEFINE_ID; + +import org.apache.dolphinscheduler.api.dto.ProcessMeta; +import org.apache.dolphinscheduler.api.dto.treeview.Instance; +import org.apache.dolphinscheduler.api.dto.treeview.TreeViewDto; +import org.apache.dolphinscheduler.api.enums.Status; +import org.apache.dolphinscheduler.api.service.BaseService; +import org.apache.dolphinscheduler.api.service.ProcessDefinitionService; +import org.apache.dolphinscheduler.api.service.ProcessDefinitionVersionService; +import org.apache.dolphinscheduler.api.service.ProcessInstanceService; +import org.apache.dolphinscheduler.api.service.ProjectService; +import org.apache.dolphinscheduler.api.service.SchedulerService; +import org.apache.dolphinscheduler.api.utils.CheckUtils; +import org.apache.dolphinscheduler.api.utils.FileUtils; +import org.apache.dolphinscheduler.api.utils.PageInfo; +import org.apache.dolphinscheduler.api.utils.exportprocess.ProcessAddTaskParam; +import org.apache.dolphinscheduler.api.utils.exportprocess.TaskNodeParamFactory; +import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.enums.AuthorizationType; +import org.apache.dolphinscheduler.common.enums.FailureStrategy; +import org.apache.dolphinscheduler.common.enums.Flag; +import org.apache.dolphinscheduler.common.enums.Priority; +import org.apache.dolphinscheduler.common.enums.ReleaseState; +import org.apache.dolphinscheduler.common.enums.TaskType; +import org.apache.dolphinscheduler.common.enums.UserType; +import org.apache.dolphinscheduler.common.enums.WarningType; +import org.apache.dolphinscheduler.common.graph.DAG; +import org.apache.dolphinscheduler.common.model.TaskNode; +import org.apache.dolphinscheduler.common.model.TaskNodeRelation; +import org.apache.dolphinscheduler.common.process.ProcessDag; +import org.apache.dolphinscheduler.common.process.Property; +import org.apache.dolphinscheduler.common.task.AbstractParameters; +import org.apache.dolphinscheduler.common.thread.Stopper; +import org.apache.dolphinscheduler.common.utils.CollectionUtils; +import org.apache.dolphinscheduler.common.utils.DateUtils; +import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.StreamUtils; +import org.apache.dolphinscheduler.common.utils.StringUtils; +import org.apache.dolphinscheduler.common.utils.TaskParametersUtils; +import org.apache.dolphinscheduler.dao.entity.ProcessData; +import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; +import org.apache.dolphinscheduler.dao.entity.ProcessDefinitionVersion; +import org.apache.dolphinscheduler.dao.entity.ProcessInstance; +import org.apache.dolphinscheduler.dao.entity.Project; +import org.apache.dolphinscheduler.dao.entity.Schedule; +import org.apache.dolphinscheduler.dao.entity.TaskInstance; +import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper; +import org.apache.dolphinscheduler.dao.mapper.ProjectMapper; +import org.apache.dolphinscheduler.dao.mapper.ScheduleMapper; +import org.apache.dolphinscheduler.dao.mapper.TaskInstanceMapper; +import org.apache.dolphinscheduler.dao.utils.DagHelper; +import org.apache.dolphinscheduler.service.permission.PermissionCheck; +import org.apache.dolphinscheduler.service.process.ProcessService; + +import java.io.BufferedOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; + +import javax.servlet.ServletOutputStream; +import javax.servlet.http.HttpServletResponse; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.MediaType; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; +import org.springframework.web.multipart.MultipartFile; + +import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; + +/** + * process definition service impl + */ +@Service +public class ProcessDefinitionServiceImpl extends BaseService implements + ProcessDefinitionService { + + private static final Logger logger = LoggerFactory.getLogger(ProcessDefinitionServiceImpl.class); + + private static final String PROCESSDEFINITIONID = "processDefinitionId"; + + private static final String RELEASESTATE = "releaseState"; + + private static final String TASKS = "tasks"; + + @Autowired + private ProjectMapper projectMapper; + + @Autowired + private ProjectService projectService; + + @Autowired + private ProcessDefinitionVersionService processDefinitionVersionService; + + @Autowired + private ProcessDefinitionMapper processDefineMapper; + + @Autowired + private ProcessInstanceService processInstanceService; + + @Autowired + private TaskInstanceMapper taskInstanceMapper; + + @Autowired + private ScheduleMapper scheduleMapper; + + @Autowired + private ProcessService processService; + + /** + * create process definition + * + * @param loginUser login user + * @param projectName project name + * @param name process definition name + * @param processDefinitionJson process definition json + * @param desc description + * @param locations locations for nodes + * @param connects connects for nodes + * @return create result code + * @throws JsonProcessingException JsonProcessingException + */ + public Map createProcessDefinition(User loginUser, + String projectName, + String name, + String processDefinitionJson, + String desc, + String locations, + String connects) throws JsonProcessingException { + + Map result = new HashMap<>(); + Project project = projectMapper.queryByName(projectName); + // check project auth + Map checkResult = projectService.checkProjectAndAuth(loginUser, project, projectName); + Status resultStatus = (Status) checkResult.get(Constants.STATUS); + if (resultStatus != Status.SUCCESS) { + return checkResult; + } + + ProcessDefinition processDefine = new ProcessDefinition(); + Date now = new Date(); + + ProcessData processData = JSONUtils.parseObject(processDefinitionJson, ProcessData.class); + Map checkProcessJson = checkProcessNodeList(processData, processDefinitionJson); + if (checkProcessJson.get(Constants.STATUS) != Status.SUCCESS) { + return checkProcessJson; + } + + processDefine.setName(name); + processDefine.setReleaseState(ReleaseState.OFFLINE); + processDefine.setProjectId(project.getId()); + processDefine.setUserId(loginUser.getId()); + processDefine.setProcessDefinitionJson(processDefinitionJson); + processDefine.setDescription(desc); + processDefine.setLocations(locations); + processDefine.setConnects(connects); + processDefine.setTimeout(processData.getTimeout()); + processDefine.setTenantId(processData.getTenantId()); + processDefine.setModifyBy(loginUser.getUserName()); + processDefine.setResourceIds(getResourceIds(processData)); + + //custom global params + List globalParamsList = processData.getGlobalParams(); + if (CollectionUtils.isNotEmpty(globalParamsList)) { + Set globalParamsSet = new HashSet<>(globalParamsList); + globalParamsList = new ArrayList<>(globalParamsSet); + processDefine.setGlobalParamList(globalParamsList); + } + processDefine.setCreateTime(now); + processDefine.setUpdateTime(now); + processDefine.setFlag(Flag.YES); + + // save the new process definition + processDefineMapper.insert(processDefine); + + // add process definition version + long version = processDefinitionVersionService.addProcessDefinitionVersion(processDefine); + + processDefine.setVersion(version); + + processDefineMapper.updateVersionByProcessDefinitionId(processDefine.getId(), version); + + // return processDefinition object with ID + result.put(Constants.DATA_LIST, processDefineMapper.selectById(processDefine.getId())); + putMsg(result, Status.SUCCESS); + result.put("processDefinitionId", processDefine.getId()); + return result; + } + + /** + * get resource ids + * + * @param processData process data + * @return resource ids + */ + private String getResourceIds(ProcessData processData) { + List tasks = processData.getTasks(); + Set resourceIds = new HashSet<>(); + for (TaskNode taskNode : tasks) { + String taskParameter = taskNode.getParams(); + AbstractParameters params = TaskParametersUtils.getParameters(taskNode.getType(), taskParameter); + if (CollectionUtils.isNotEmpty(params.getResourceFilesList())) { + Set tempSet = params.getResourceFilesList().stream().map(t -> t.getId()).collect(Collectors.toSet()); + resourceIds.addAll(tempSet); + } + } + + StringBuilder sb = new StringBuilder(); + for (int i : resourceIds) { + if (sb.length() > 0) { + sb.append(","); + } + sb.append(i); + } + return sb.toString(); + } + + /** + * query process definition list + * + * @param loginUser login user + * @param projectName project name + * @return definition list + */ + public Map queryProcessDefinitionList(User loginUser, String projectName) { + + HashMap result = new HashMap<>(); + Project project = projectMapper.queryByName(projectName); + + Map checkResult = projectService.checkProjectAndAuth(loginUser, project, projectName); + Status resultStatus = (Status) checkResult.get(Constants.STATUS); + if (resultStatus != Status.SUCCESS) { + return checkResult; + } + + List resourceList = processDefineMapper.queryAllDefinitionList(project.getId()); + result.put(Constants.DATA_LIST, resourceList); + putMsg(result, Status.SUCCESS); + + return result; + } + + /** + * query process definition list paging + * + * @param loginUser login user + * @param projectName project name + * @param searchVal search value + * @param pageNo page number + * @param pageSize page size + * @param userId user id + * @return process definition page + */ + public Map queryProcessDefinitionListPaging(User loginUser, String projectName, String searchVal, Integer pageNo, Integer pageSize, Integer userId) { + + Map result = new HashMap<>(); + Project project = projectMapper.queryByName(projectName); + + Map checkResult = projectService.checkProjectAndAuth(loginUser, project, projectName); + Status resultStatus = (Status) checkResult.get(Constants.STATUS); + if (resultStatus != Status.SUCCESS) { + return checkResult; + } + + Page page = new Page<>(pageNo, pageSize); + IPage processDefinitionIPage = processDefineMapper.queryDefineListPaging( + page, searchVal, userId, project.getId(), isAdmin(loginUser)); + + PageInfo pageInfo = new PageInfo<>(pageNo, pageSize); + pageInfo.setTotalCount((int) processDefinitionIPage.getTotal()); + pageInfo.setLists(processDefinitionIPage.getRecords()); + result.put(Constants.DATA_LIST, pageInfo); + putMsg(result, Status.SUCCESS); + + return result; + } + + /** + * query datail of process definition + * + * @param loginUser login user + * @param projectName project name + * @param processId process definition id + * @return process definition detail + */ + public Map queryProcessDefinitionById(User loginUser, String projectName, Integer processId) { + + Map result = new HashMap<>(); + Project project = projectMapper.queryByName(projectName); + + Map checkResult = projectService.checkProjectAndAuth(loginUser, project, projectName); + Status resultStatus = (Status) checkResult.get(Constants.STATUS); + if (resultStatus != Status.SUCCESS) { + return checkResult; + } + + ProcessDefinition processDefinition = processDefineMapper.selectById(processId); + if (processDefinition == null) { + putMsg(result, Status.PROCESS_INSTANCE_NOT_EXIST, processId); + } else { + result.put(Constants.DATA_LIST, processDefinition); + putMsg(result, Status.SUCCESS); + } + return result; + } + + /** + * update process definition + * + * @param loginUser login user + * @param projectName project name + * @param name process definition name + * @param id process definition id + * @param processDefinitionJson process definition json + * @param desc description + * @param locations locations for nodes + * @param connects connects for nodes + * @return update result code + */ + public Map updateProcessDefinition(User loginUser, String projectName, int id, String name, + String processDefinitionJson, String desc, + String locations, String connects) { + Map result = new HashMap<>(); + + Project project = projectMapper.queryByName(projectName); + Map checkResult = projectService.checkProjectAndAuth(loginUser, project, projectName); + Status resultStatus = (Status) checkResult.get(Constants.STATUS); + if (resultStatus != Status.SUCCESS) { + return checkResult; + } + + ProcessData processData = JSONUtils.parseObject(processDefinitionJson, ProcessData.class); + Map checkProcessJson = checkProcessNodeList(processData, processDefinitionJson); + if ((checkProcessJson.get(Constants.STATUS) != Status.SUCCESS)) { + return checkProcessJson; + } + ProcessDefinition processDefine = processService.findProcessDefineById(id); + if (processDefine == null) { + // check process definition exists + putMsg(result, Status.PROCESS_DEFINE_NOT_EXIST, id); + return result; + } else if (processDefine.getReleaseState() == ReleaseState.ONLINE) { + // online can not permit edit + putMsg(result, Status.PROCESS_DEFINE_NOT_ALLOWED_EDIT, processDefine.getName()); + return result; + } else { + putMsg(result, Status.SUCCESS); + } + + Date now = new Date(); + + processDefine.setId(id); + processDefine.setName(name); + processDefine.setReleaseState(ReleaseState.OFFLINE); + processDefine.setProjectId(project.getId()); + processDefine.setProcessDefinitionJson(processDefinitionJson); + processDefine.setDescription(desc); + processDefine.setLocations(locations); + processDefine.setConnects(connects); + processDefine.setTimeout(processData.getTimeout()); + processDefine.setTenantId(processData.getTenantId()); + processDefine.setModifyBy(loginUser.getUserName()); + processDefine.setResourceIds(getResourceIds(processData)); + + //custom global params + List globalParamsList = new ArrayList<>(); + if (CollectionUtils.isNotEmpty(processData.getGlobalParams())) { + Set userDefParamsSet = new HashSet<>(processData.getGlobalParams()); + globalParamsList = new ArrayList<>(userDefParamsSet); + } + processDefine.setGlobalParamList(globalParamsList); + processDefine.setUpdateTime(now); + processDefine.setFlag(Flag.YES); + + // add process definition version + long version = processDefinitionVersionService.addProcessDefinitionVersion(processDefine); + processDefine.setVersion(version); + + if (processDefineMapper.updateById(processDefine) > 0) { + putMsg(result, Status.SUCCESS); + result.put(Constants.DATA_LIST, processDefineMapper.queryByDefineId(id)); + } else { + putMsg(result, Status.UPDATE_PROCESS_DEFINITION_ERROR); + } + return result; + } + + /** + * verify process definition name unique + * + * @param loginUser login user + * @param projectName project name + * @param name name + * @return true if process definition name not exists, otherwise false + */ + public Map verifyProcessDefinitionName(User loginUser, String projectName, String name) { + + Map result = new HashMap<>(); + Project project = projectMapper.queryByName(projectName); + + Map checkResult = projectService.checkProjectAndAuth(loginUser, project, projectName); + Status resultEnum = (Status) checkResult.get(Constants.STATUS); + if (resultEnum != Status.SUCCESS) { + return checkResult; + } + ProcessDefinition processDefinition = processDefineMapper.queryByDefineName(project.getId(), name); + if (processDefinition == null) { + putMsg(result, Status.SUCCESS); + } else { + putMsg(result, Status.PROCESS_INSTANCE_EXIST, name); + } + return result; + } + + /** + * delete process definition by id + * + * @param loginUser login user + * @param projectName project name + * @param processDefinitionId process definition id + * @return delete result code + */ + @Transactional(rollbackFor = RuntimeException.class) + public Map deleteProcessDefinitionById(User loginUser, String projectName, Integer processDefinitionId) { + + Map result = new HashMap<>(); + Project project = projectMapper.queryByName(projectName); + + Map checkResult = projectService.checkProjectAndAuth(loginUser, project, projectName); + Status resultEnum = (Status) checkResult.get(Constants.STATUS); + if (resultEnum != Status.SUCCESS) { + return checkResult; + } + + ProcessDefinition processDefinition = processDefineMapper.selectById(processDefinitionId); + + if (processDefinition == null) { + putMsg(result, Status.PROCESS_DEFINE_NOT_EXIST, processDefinitionId); + return result; + } + + // Determine if the login user is the owner of the process definition + if (loginUser.getId() != processDefinition.getUserId() && loginUser.getUserType() != UserType.ADMIN_USER) { + putMsg(result, Status.USER_NO_OPERATION_PERM); + return result; + } + + // check process definition is already online + if (processDefinition.getReleaseState() == ReleaseState.ONLINE) { + putMsg(result, Status.PROCESS_DEFINE_STATE_ONLINE, processDefinitionId); + return result; + } + // check process instances is already running + List processInstances = processInstanceService.queryByProcessDefineIdAndStatus(processDefinitionId, Constants.NOT_TERMINATED_STATES); + if (CollectionUtils.isNotEmpty(processInstances)) { + putMsg(result, Status.DELETE_PROCESS_DEFINITION_BY_ID_FAIL,processInstances.size()); + return result; + } + + // get the timing according to the process definition + List schedules = scheduleMapper.queryByProcessDefinitionId(processDefinitionId); + if (!schedules.isEmpty() && schedules.size() > 1) { + logger.warn("scheduler num is {},Greater than 1", schedules.size()); + putMsg(result, Status.DELETE_PROCESS_DEFINE_BY_ID_ERROR); + return result; + } else if (schedules.size() == 1) { + Schedule schedule = schedules.get(0); + if (schedule.getReleaseState() == ReleaseState.OFFLINE) { + scheduleMapper.deleteById(schedule.getId()); + } else if (schedule.getReleaseState() == ReleaseState.ONLINE) { + putMsg(result, Status.SCHEDULE_CRON_STATE_ONLINE, schedule.getId()); + return result; + } + } + + int delete = processDefineMapper.deleteById(processDefinitionId); + + if (delete > 0) { + putMsg(result, Status.SUCCESS); + } else { + putMsg(result, Status.DELETE_PROCESS_DEFINE_BY_ID_ERROR); + } + return result; + } + + /** + * release process definition: online / offline + * + * @param loginUser login user + * @param projectName project name + * @param id process definition id + * @param releaseState release state + * @return release result code + */ + @Transactional(rollbackFor = RuntimeException.class) + public Map releaseProcessDefinition(User loginUser, String projectName, int id, int releaseState) { + HashMap result = new HashMap<>(); + Project project = projectMapper.queryByName(projectName); + + Map checkResult = projectService.checkProjectAndAuth(loginUser, project, projectName); + Status resultEnum = (Status) checkResult.get(Constants.STATUS); + if (resultEnum != Status.SUCCESS) { + return checkResult; + } + + ReleaseState state = ReleaseState.getEnum(releaseState); + + // check state + if (null == state) { + putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, RELEASESTATE); + return result; + } + + ProcessDefinition processDefinition = processDefineMapper.selectById(id); + + switch (state) { + case ONLINE: + // To check resources whether they are already cancel authorized or deleted + String resourceIds = processDefinition.getResourceIds(); + if (StringUtils.isNotBlank(resourceIds)) { + Integer[] resourceIdArray = Arrays.stream(resourceIds.split(",")).map(Integer::parseInt).toArray(Integer[]::new); + PermissionCheck permissionCheck = new PermissionCheck<>(AuthorizationType.RESOURCE_FILE_ID, processService, resourceIdArray, loginUser.getId(), logger); + try { + permissionCheck.checkPermission(); + } catch (Exception e) { + logger.error(e.getMessage(), e); + putMsg(result, Status.RESOURCE_NOT_EXIST_OR_NO_PERMISSION, RELEASESTATE); + return result; + } + } + + processDefinition.setReleaseState(state); + processDefineMapper.updateById(processDefinition); + break; + case OFFLINE: + processDefinition.setReleaseState(state); + processDefineMapper.updateById(processDefinition); + List scheduleList = scheduleMapper.selectAllByProcessDefineArray( + new int[]{processDefinition.getId()} + ); + + for (Schedule schedule : scheduleList) { + logger.info("set schedule offline, project id: {}, schedule id: {}, process definition id: {}", project.getId(), schedule.getId(), id); + // set status + schedule.setReleaseState(ReleaseState.OFFLINE); + scheduleMapper.updateById(schedule); + SchedulerService.deleteSchedule(project.getId(), schedule.getId()); + } + break; + default: + putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, RELEASESTATE); + return result; + } + + putMsg(result, Status.SUCCESS); + return result; + } + + /** + * batch export process definition by ids + */ + public void batchExportProcessDefinitionByIds(User loginUser, String projectName, String processDefinitionIds, HttpServletResponse response) { + + if (StringUtils.isEmpty(processDefinitionIds)) { + return; + } + + //export project info + Project project = projectMapper.queryByName(projectName); + + //check user access for project + Map checkResult = projectService.checkProjectAndAuth(loginUser, project, projectName); + Status resultStatus = (Status) checkResult.get(Constants.STATUS); + + if (resultStatus != Status.SUCCESS) { + return; + } + + List processDefinitionList = + getProcessDefinitionList(processDefinitionIds); + + if (CollectionUtils.isNotEmpty(processDefinitionList)) { + downloadProcessDefinitionFile(response, processDefinitionList); + } + } + + /** + * get process definition list by ids + */ + private List getProcessDefinitionList(String processDefinitionIds) { + List processDefinitionList = new ArrayList<>(); + String[] processDefinitionIdArray = processDefinitionIds.split(","); + for (String strProcessDefinitionId : processDefinitionIdArray) { + //get workflow info + int processDefinitionId = Integer.parseInt(strProcessDefinitionId); + ProcessDefinition processDefinition = processDefineMapper.queryByDefineId(processDefinitionId); + if (null != processDefinition) { + processDefinitionList.add(exportProcessMetaData(processDefinitionId, processDefinition)); + } + } + + return processDefinitionList; + } + + /** + * download the process definition file + */ + private void downloadProcessDefinitionFile(HttpServletResponse response, List processDefinitionList) { + response.setContentType(MediaType.APPLICATION_JSON_UTF8_VALUE); + BufferedOutputStream buff = null; + ServletOutputStream out = null; + try { + out = response.getOutputStream(); + buff = new BufferedOutputStream(out); + buff.write(JSONUtils.toJsonString(processDefinitionList).getBytes(StandardCharsets.UTF_8)); + buff.flush(); + buff.close(); + } catch (IOException e) { + logger.warn("export process fail", e); + } finally { + if (null != buff) { + try { + buff.close(); + } catch (Exception e) { + logger.warn("export process buffer not close", e); + } + } + if (null != out) { + try { + out.close(); + } catch (Exception e) { + logger.warn("export process output stream not close", e); + } + } + } + } + + /** + * get export process metadata string + * + * @param processDefinitionId process definition id + * @param processDefinition process definition + * @return export process metadata string + */ + public ProcessMeta exportProcessMetaData(Integer processDefinitionId, ProcessDefinition processDefinition) { + //correct task param which has data source or dependent param + String correctProcessDefinitionJson = addExportTaskNodeSpecialParam(processDefinition.getProcessDefinitionJson()); + processDefinition.setProcessDefinitionJson(correctProcessDefinitionJson); + + //export process metadata + ProcessMeta exportProcessMeta = new ProcessMeta(); + exportProcessMeta.setProjectName(processDefinition.getProjectName()); + exportProcessMeta.setProcessDefinitionName(processDefinition.getName()); + exportProcessMeta.setProcessDefinitionJson(processDefinition.getProcessDefinitionJson()); + exportProcessMeta.setProcessDefinitionLocations(processDefinition.getLocations()); + exportProcessMeta.setProcessDefinitionConnects(processDefinition.getConnects()); + + //schedule info + List schedules = scheduleMapper.queryByProcessDefinitionId(processDefinitionId); + if (!schedules.isEmpty()) { + Schedule schedule = schedules.get(0); + exportProcessMeta.setScheduleWarningType(schedule.getWarningType().toString()); + exportProcessMeta.setScheduleWarningGroupId(schedule.getWarningGroupId()); + exportProcessMeta.setScheduleStartTime(DateUtils.dateToString(schedule.getStartTime())); + exportProcessMeta.setScheduleEndTime(DateUtils.dateToString(schedule.getEndTime())); + exportProcessMeta.setScheduleCrontab(schedule.getCrontab()); + exportProcessMeta.setScheduleFailureStrategy(String.valueOf(schedule.getFailureStrategy())); + exportProcessMeta.setScheduleReleaseState(String.valueOf(ReleaseState.OFFLINE)); + exportProcessMeta.setScheduleProcessInstancePriority(String.valueOf(schedule.getProcessInstancePriority())); + exportProcessMeta.setScheduleWorkerGroupName(schedule.getWorkerGroup()); + } + //create workflow json file + return exportProcessMeta; + } + + /** + * correct task param which has datasource or dependent + * + * @param processDefinitionJson processDefinitionJson + * @return correct processDefinitionJson + */ + private String addExportTaskNodeSpecialParam(String processDefinitionJson) { + ObjectNode jsonObject = JSONUtils.parseObject(processDefinitionJson); + ArrayNode jsonArray = (ArrayNode) jsonObject.path(TASKS); + + for (int i = 0; i < jsonArray.size(); i++) { + JsonNode taskNode = jsonArray.path(i); + if (StringUtils.isNotEmpty(taskNode.path("type").asText())) { + String taskType = taskNode.path("type").asText(); + + ProcessAddTaskParam addTaskParam = TaskNodeParamFactory.getByTaskType(taskType); + if (null != addTaskParam) { + addTaskParam.addExportSpecialParam(taskNode); + } + } + } + jsonObject.set(TASKS, jsonArray); + return jsonObject.toString(); + } + + /** + * check task if has sub process + * + * @param taskType task type + * @return if task has sub process return true else false + */ + private boolean checkTaskHasSubProcess(String taskType) { + return taskType.equals(TaskType.SUB_PROCESS.name()); + } + + /** + * import process definition + * + * @param loginUser login user + * @param file process metadata json file + * @param currentProjectName current project name + * @return import process + */ + @Transactional(rollbackFor = RuntimeException.class) + public Map importProcessDefinition(User loginUser, MultipartFile file, String currentProjectName) { + Map result = new HashMap<>(); + String processMetaJson = FileUtils.file2String(file); + List processMetaList = JSONUtils.toList(processMetaJson, ProcessMeta.class); + + //check file content + if (CollectionUtils.isEmpty(processMetaList)) { + putMsg(result, Status.DATA_IS_NULL, "fileContent"); + return result; + } + + for (ProcessMeta processMeta : processMetaList) { + + if (!checkAndImportProcessDefinition(loginUser, currentProjectName, result, processMeta)) { + return result; + } + } + + return result; + } + + /** + * check and import process definition + */ + private boolean checkAndImportProcessDefinition(User loginUser, String currentProjectName, Map result, ProcessMeta processMeta) { + + if (!checkImportanceParams(processMeta, result)) { + return false; + } + + //deal with process name + String processDefinitionName = processMeta.getProcessDefinitionName(); + //use currentProjectName to query + Project targetProject = projectMapper.queryByName(currentProjectName); + if (null != targetProject) { + processDefinitionName = recursionProcessDefinitionName(targetProject.getId(), + processDefinitionName, 1); + } + + //unique check + Map checkResult = verifyProcessDefinitionName(loginUser, currentProjectName, processDefinitionName); + Status status = (Status) checkResult.get(Constants.STATUS); + if (Status.SUCCESS.equals(status)) { + putMsg(result, Status.SUCCESS); + } else { + result.putAll(checkResult); + return false; + } + + // get create process result + Map createProcessResult = + getCreateProcessResult(loginUser, + currentProjectName, + result, + processMeta, + processDefinitionName, + addImportTaskNodeParam(loginUser, processMeta.getProcessDefinitionJson(), targetProject)); + + if (createProcessResult == null) { + return false; + } + + //create process definition + Integer processDefinitionId = + Objects.isNull(createProcessResult.get(PROCESSDEFINITIONID)) + ? null : Integer.parseInt(createProcessResult.get(PROCESSDEFINITIONID).toString()); + + //scheduler param + return getImportProcessScheduleResult(loginUser, + currentProjectName, + result, + processMeta, + processDefinitionName, + processDefinitionId); + + } + + /** + * get create process result + */ + private Map getCreateProcessResult(User loginUser, + String currentProjectName, + Map result, + ProcessMeta processMeta, + String processDefinitionName, + String importProcessParam) { + Map createProcessResult = null; + try { + createProcessResult = createProcessDefinition(loginUser + , currentProjectName, + processDefinitionName + "_import_" + System.currentTimeMillis(), + importProcessParam, + processMeta.getProcessDefinitionDescription(), + processMeta.getProcessDefinitionLocations(), + processMeta.getProcessDefinitionConnects()); + putMsg(result, Status.SUCCESS); + } catch (JsonProcessingException e) { + logger.error("import process meta json data: {}", e.getMessage(), e); + putMsg(result, Status.IMPORT_PROCESS_DEFINE_ERROR); + } + + return createProcessResult; + } + + /** + * get import process schedule result + */ + private boolean getImportProcessScheduleResult(User loginUser, + String currentProjectName, + Map result, + ProcessMeta processMeta, + String processDefinitionName, + Integer processDefinitionId) { + if (null != processMeta.getScheduleCrontab() && null != processDefinitionId) { + int scheduleInsert = importProcessSchedule(loginUser, + currentProjectName, + processMeta, + processDefinitionName, + processDefinitionId); + + if (0 == scheduleInsert) { + putMsg(result, Status.IMPORT_PROCESS_DEFINE_ERROR); + return false; + } + } + return true; + } + + /** + * check importance params + */ + private boolean checkImportanceParams(ProcessMeta processMeta, Map result) { + if (StringUtils.isEmpty(processMeta.getProjectName())) { + putMsg(result, Status.DATA_IS_NULL, "projectName"); + return false; + } + if (StringUtils.isEmpty(processMeta.getProcessDefinitionName())) { + putMsg(result, Status.DATA_IS_NULL, "processDefinitionName"); + return false; + } + if (StringUtils.isEmpty(processMeta.getProcessDefinitionJson())) { + putMsg(result, Status.DATA_IS_NULL, "processDefinitionJson"); + return false; + } + + return true; + } + + /** + * import process add special task param + * + * @param loginUser login user + * @param processDefinitionJson process definition json + * @param targetProject target project + * @return import process param + */ + private String addImportTaskNodeParam(User loginUser, String processDefinitionJson, Project targetProject) { + ObjectNode jsonObject = JSONUtils.parseObject(processDefinitionJson); + ArrayNode jsonArray = (ArrayNode) jsonObject.get(TASKS); + //add sql and dependent param + for (int i = 0; i < jsonArray.size(); i++) { + JsonNode taskNode = jsonArray.path(i); + String taskType = taskNode.path("type").asText(); + ProcessAddTaskParam addTaskParam = TaskNodeParamFactory.getByTaskType(taskType); + if (null != addTaskParam) { + addTaskParam.addImportSpecialParam(taskNode); + } + } + + //recursive sub-process parameter correction map key for old process id value for new process id + Map subProcessIdMap = new HashMap<>(); + + List subProcessList = StreamUtils.asStream(jsonArray.elements()) + .filter(elem -> checkTaskHasSubProcess(JSONUtils.parseObject(elem.toString()).path("type").asText())) + .collect(Collectors.toList()); + + if (CollectionUtils.isNotEmpty(subProcessList)) { + importSubProcess(loginUser, targetProject, jsonArray, subProcessIdMap); + } + + jsonObject.set(TASKS, jsonArray); + return jsonObject.toString(); + } + + /** + * import process schedule + * + * @param loginUser login user + * @param currentProjectName current project name + * @param processMeta process meta data + * @param processDefinitionName process definition name + * @param processDefinitionId process definition id + * @return insert schedule flag + */ + public int importProcessSchedule(User loginUser, String currentProjectName, ProcessMeta processMeta, + String processDefinitionName, Integer processDefinitionId) { + Date now = new Date(); + Schedule scheduleObj = new Schedule(); + scheduleObj.setProjectName(currentProjectName); + scheduleObj.setProcessDefinitionId(processDefinitionId); + scheduleObj.setProcessDefinitionName(processDefinitionName); + scheduleObj.setCreateTime(now); + scheduleObj.setUpdateTime(now); + scheduleObj.setUserId(loginUser.getId()); + scheduleObj.setUserName(loginUser.getUserName()); + + scheduleObj.setCrontab(processMeta.getScheduleCrontab()); + + if (null != processMeta.getScheduleStartTime()) { + scheduleObj.setStartTime(DateUtils.stringToDate(processMeta.getScheduleStartTime())); + } + if (null != processMeta.getScheduleEndTime()) { + scheduleObj.setEndTime(DateUtils.stringToDate(processMeta.getScheduleEndTime())); + } + if (null != processMeta.getScheduleWarningType()) { + scheduleObj.setWarningType(WarningType.valueOf(processMeta.getScheduleWarningType())); + } + if (null != processMeta.getScheduleWarningGroupId()) { + scheduleObj.setWarningGroupId(processMeta.getScheduleWarningGroupId()); + } + if (null != processMeta.getScheduleFailureStrategy()) { + scheduleObj.setFailureStrategy(FailureStrategy.valueOf(processMeta.getScheduleFailureStrategy())); + } + if (null != processMeta.getScheduleReleaseState()) { + scheduleObj.setReleaseState(ReleaseState.valueOf(processMeta.getScheduleReleaseState())); + } + if (null != processMeta.getScheduleProcessInstancePriority()) { + scheduleObj.setProcessInstancePriority(Priority.valueOf(processMeta.getScheduleProcessInstancePriority())); + } + + if (null != processMeta.getScheduleWorkerGroupName()) { + scheduleObj.setWorkerGroup(processMeta.getScheduleWorkerGroupName()); + } + + return scheduleMapper.insert(scheduleObj); + } + + /** + * check import process has sub process + * recursion create sub process + * + * @param loginUser login user + * @param targetProject target project + * @param jsonArray process task array + * @param subProcessIdMap correct sub process id map + */ + private void importSubProcess(User loginUser, Project targetProject, ArrayNode jsonArray, Map subProcessIdMap) { + for (int i = 0; i < jsonArray.size(); i++) { + ObjectNode taskNode = (ObjectNode) jsonArray.path(i); + String taskType = taskNode.path("type").asText(); + + if (!checkTaskHasSubProcess(taskType)) { + continue; + } + //get sub process info + ObjectNode subParams = (ObjectNode) taskNode.path("params"); + Integer subProcessId = subParams.path(PROCESSDEFINITIONID).asInt(); + ProcessDefinition subProcess = processDefineMapper.queryByDefineId(subProcessId); + //check is sub process exist in db + if (null == subProcess) { + continue; + } + String subProcessJson = subProcess.getProcessDefinitionJson(); + //check current project has sub process + ProcessDefinition currentProjectSubProcess = processDefineMapper.queryByDefineName(targetProject.getId(), subProcess.getName()); + + if (null == currentProjectSubProcess) { + ArrayNode subJsonArray = (ArrayNode) JSONUtils.parseObject(subProcess.getProcessDefinitionJson()).get(TASKS); + + List subProcessList = StreamUtils.asStream(subJsonArray.elements()) + .filter(item -> checkTaskHasSubProcess(JSONUtils.parseObject(item.toString()).path("type").asText())) + .collect(Collectors.toList()); + + if (CollectionUtils.isNotEmpty(subProcessList)) { + importSubProcess(loginUser, targetProject, subJsonArray, subProcessIdMap); + //sub process processId correct + if (!subProcessIdMap.isEmpty()) { + + for (Map.Entry entry : subProcessIdMap.entrySet()) { + String oldSubProcessId = "\"processDefinitionId\":" + entry.getKey(); + String newSubProcessId = "\"processDefinitionId\":" + entry.getValue(); + subProcessJson = subProcessJson.replaceAll(oldSubProcessId, newSubProcessId); + } + + subProcessIdMap.clear(); + } + } + + //if sub-process recursion + Date now = new Date(); + //create sub process in target project + ProcessDefinition processDefine = new ProcessDefinition(); + processDefine.setName(subProcess.getName()); + processDefine.setVersion(subProcess.getVersion()); + processDefine.setReleaseState(subProcess.getReleaseState()); + processDefine.setProjectId(targetProject.getId()); + processDefine.setUserId(loginUser.getId()); + processDefine.setProcessDefinitionJson(subProcessJson); + processDefine.setDescription(subProcess.getDescription()); + processDefine.setLocations(subProcess.getLocations()); + processDefine.setConnects(subProcess.getConnects()); + processDefine.setTimeout(subProcess.getTimeout()); + processDefine.setTenantId(subProcess.getTenantId()); + processDefine.setGlobalParams(subProcess.getGlobalParams()); + processDefine.setCreateTime(now); + processDefine.setUpdateTime(now); + processDefine.setFlag(subProcess.getFlag()); + processDefine.setReceivers(subProcess.getReceivers()); + processDefine.setReceiversCc(subProcess.getReceiversCc()); + processDefineMapper.insert(processDefine); + + logger.info("create sub process, project: {}, process name: {}", targetProject.getName(), processDefine.getName()); + + //modify task node + ProcessDefinition newSubProcessDefine = processDefineMapper.queryByDefineName(processDefine.getProjectId(), processDefine.getName()); + + if (null != newSubProcessDefine) { + subProcessIdMap.put(subProcessId, newSubProcessDefine.getId()); + subParams.put(PROCESSDEFINITIONID, newSubProcessDefine.getId()); + taskNode.set("params", subParams); + } + } + } + } + + /** + * check the process definition node meets the specifications + * + * @param processData process data + * @param processDefinitionJson process definition json + * @return check result code + */ + public Map checkProcessNodeList(ProcessData processData, String processDefinitionJson) { + + Map result = new HashMap<>(); + try { + if (processData == null) { + logger.error("process data is null"); + putMsg(result, Status.DATA_IS_NOT_VALID, processDefinitionJson); + return result; + } + + // Check whether the task node is normal + List taskNodes = processData.getTasks(); + + if (taskNodes == null) { + logger.error("process node info is empty"); + putMsg(result, Status.DATA_IS_NULL, processDefinitionJson); + return result; + } + + // check has cycle + if (graphHasCycle(taskNodes)) { + logger.error("process DAG has cycle"); + putMsg(result, Status.PROCESS_NODE_HAS_CYCLE); + return result; + } + + // check whether the process definition json is normal + for (TaskNode taskNode : taskNodes) { + if (!CheckUtils.checkTaskNodeParameters(taskNode.getParams(), taskNode.getType())) { + logger.error("task node {} parameter invalid", taskNode.getName()); + putMsg(result, Status.PROCESS_NODE_S_PARAMETER_INVALID, taskNode.getName()); + return result; + } + + // check extra params + CheckUtils.checkOtherParams(taskNode.getExtras()); + } + putMsg(result, Status.SUCCESS); + } catch (Exception e) { + result.put(Constants.STATUS, Status.REQUEST_PARAMS_NOT_VALID_ERROR); + result.put(Constants.MSG, e.getMessage()); + } + return result; + } + + /** + * get task node details based on process definition + * + * @param defineId define id + * @return task node list + */ + public Map getTaskNodeListByDefinitionId(Integer defineId) { + Map result = new HashMap<>(); + + ProcessDefinition processDefinition = processDefineMapper.selectById(defineId); + if (processDefinition == null) { + logger.info("process define not exists"); + putMsg(result, Status.PROCESS_DEFINE_NOT_EXIST, defineId); + return result; + } + + String processDefinitionJson = processDefinition.getProcessDefinitionJson(); + + ProcessData processData = JSONUtils.parseObject(processDefinitionJson, ProcessData.class); + + //process data check + if (null == processData) { + logger.error("process data is null"); + putMsg(result, Status.DATA_IS_NOT_VALID, processDefinitionJson); + return result; + } + + List taskNodeList = (processData.getTasks() == null) ? new ArrayList<>() : processData.getTasks(); + + result.put(Constants.DATA_LIST, taskNodeList); + putMsg(result, Status.SUCCESS); + + return result; + + } + + /** + * get task node details based on process definition + * + * @param defineIdList define id list + * @return task node list + */ + public Map getTaskNodeListByDefinitionIdList(String defineIdList) { + Map result = new HashMap<>(); + + Map> taskNodeMap = new HashMap<>(); + String[] idList = defineIdList.split(","); + List idIntList = new ArrayList<>(); + for (String definitionId : idList) { + idIntList.add(Integer.parseInt(definitionId)); + } + Integer[] idArray = idIntList.toArray(new Integer[0]); + List processDefinitionList = processDefineMapper.queryDefinitionListByIdList(idArray); + if (CollectionUtils.isEmpty(processDefinitionList)) { + logger.info("process definition not exists"); + putMsg(result, Status.PROCESS_DEFINE_NOT_EXIST, defineIdList); + return result; + } + + for (ProcessDefinition processDefinition : processDefinitionList) { + String processDefinitionJson = processDefinition.getProcessDefinitionJson(); + ProcessData processData = JSONUtils.parseObject(processDefinitionJson, ProcessData.class); + List taskNodeList = (processData.getTasks() == null) ? new ArrayList<>() : processData.getTasks(); + taskNodeMap.put(processDefinition.getId(), taskNodeList); + } + + result.put(Constants.DATA_LIST, taskNodeMap); + putMsg(result, Status.SUCCESS); + + return result; + + } + + /** + * query process definition all by project id + * + * @param projectId project id + * @return process definitions in the project + */ + public Map queryProcessDefinitionAllByProjectId(Integer projectId) { + + HashMap result = new HashMap<>(); + + List resourceList = processDefineMapper.queryAllDefinitionList(projectId); + result.put(Constants.DATA_LIST, resourceList); + putMsg(result, Status.SUCCESS); + + return result; + } + + /** + * Encapsulates the TreeView structure + * + * @param processId process definition id + * @param limit limit + * @return tree view json data + * @throws Exception exception + */ + public Map viewTree(Integer processId, Integer limit) throws Exception { + Map result = new HashMap<>(); + + ProcessDefinition processDefinition = processDefineMapper.selectById(processId); + if (null == processDefinition) { + logger.info("process define not exists"); + putMsg(result, Status.PROCESS_DEFINE_NOT_EXIST, processDefinition); + return result; + } + DAG dag = genDagGraph(processDefinition); + /** + * nodes that is running + */ + Map> runningNodeMap = new ConcurrentHashMap<>(); + + /** + * nodes that is waiting torun + */ + Map> waitingRunningNodeMap = new ConcurrentHashMap<>(); + + /** + * List of process instances + */ + List processInstanceList = processInstanceService.queryByProcessDefineId(processId, limit); + + for (ProcessInstance processInstance : processInstanceList) { + processInstance.setDuration(DateUtils.differSec(processInstance.getStartTime(), processInstance.getEndTime())); + } + + if (limit > processInstanceList.size()) { + limit = processInstanceList.size(); + } + + TreeViewDto parentTreeViewDto = new TreeViewDto(); + parentTreeViewDto.setName("DAG"); + parentTreeViewDto.setType(""); + // Specify the process definition, because it is a TreeView for a process definition + + for (int i = limit - 1; i >= 0; i--) { + ProcessInstance processInstance = processInstanceList.get(i); + + Date endTime = processInstance.getEndTime() == null ? new Date() : processInstance.getEndTime(); + parentTreeViewDto.getInstances().add(new Instance(processInstance.getId(), processInstance.getName(), "", processInstance.getState().toString() + , processInstance.getStartTime(), endTime, processInstance.getHost(), DateUtils.format2Readable(endTime.getTime() - processInstance.getStartTime().getTime()))); + } + + List parentTreeViewDtoList = new ArrayList<>(); + parentTreeViewDtoList.add(parentTreeViewDto); + // Here is the encapsulation task instance + for (String startNode : dag.getBeginNode()) { + runningNodeMap.put(startNode, parentTreeViewDtoList); + } + + while (Stopper.isRunning()) { + Set postNodeList = null; + Iterator>> iter = runningNodeMap.entrySet().iterator(); + while (iter.hasNext()) { + Map.Entry> en = iter.next(); + String nodeName = en.getKey(); + parentTreeViewDtoList = en.getValue(); + + TreeViewDto treeViewDto = new TreeViewDto(); + treeViewDto.setName(nodeName); + TaskNode taskNode = dag.getNode(nodeName); + treeViewDto.setType(taskNode.getType()); + + //set treeViewDto instances + for (int i = limit - 1; i >= 0; i--) { + ProcessInstance processInstance = processInstanceList.get(i); + TaskInstance taskInstance = taskInstanceMapper.queryByInstanceIdAndName(processInstance.getId(), nodeName); + if (taskInstance == null) { + treeViewDto.getInstances().add(new Instance(-1, "not running", "null")); + } else { + Date startTime = taskInstance.getStartTime() == null ? new Date() : taskInstance.getStartTime(); + Date endTime = taskInstance.getEndTime() == null ? new Date() : taskInstance.getEndTime(); + + int subProcessId = 0; + /** + * if process is sub process, the return sub id, or sub id=0 + */ + if (taskInstance.getTaskType().equals(TaskType.SUB_PROCESS.name())) { + String taskJson = taskInstance.getTaskJson(); + taskNode = JSONUtils.parseObject(taskJson, TaskNode.class); + subProcessId = Integer.parseInt(JSONUtils.parseObject( + taskNode.getParams()).path(CMDPARAM_SUB_PROCESS_DEFINE_ID).asText()); + } + treeViewDto.getInstances().add(new Instance(taskInstance.getId(), taskInstance.getName(), taskInstance.getTaskType(), taskInstance.getState().toString() + , taskInstance.getStartTime(), taskInstance.getEndTime(), taskInstance.getHost(), DateUtils.format2Readable(endTime.getTime() - startTime.getTime()), subProcessId)); + } + } + for (TreeViewDto pTreeViewDto : parentTreeViewDtoList) { + pTreeViewDto.getChildren().add(treeViewDto); + } + postNodeList = dag.getSubsequentNodes(nodeName); + if (CollectionUtils.isNotEmpty(postNodeList)) { + for (String nextNodeName : postNodeList) { + List treeViewDtoList = waitingRunningNodeMap.get(nextNodeName); + if (CollectionUtils.isEmpty(treeViewDtoList)) { + treeViewDtoList = new ArrayList<>(); + } + treeViewDtoList.add(treeViewDto); + waitingRunningNodeMap.put(nextNodeName, treeViewDtoList); + } + } + runningNodeMap.remove(nodeName); + } + if (waitingRunningNodeMap == null || waitingRunningNodeMap.size() == 0) { + break; + } else { + runningNodeMap.putAll(waitingRunningNodeMap); + waitingRunningNodeMap.clear(); + } + } + result.put(Constants.DATA_LIST, parentTreeViewDto); + result.put(Constants.STATUS, Status.SUCCESS); + result.put(Constants.MSG, Status.SUCCESS.getMsg()); + return result; + } + + /** + * Generate the DAG Graph based on the process definition id + * + * @param processDefinition process definition + * @return dag graph + */ + private DAG genDagGraph(ProcessDefinition processDefinition) { + + String processDefinitionJson = processDefinition.getProcessDefinitionJson(); + + ProcessData processData = JSONUtils.parseObject(processDefinitionJson, ProcessData.class); + + //check process data + if (null != processData) { + List taskNodeList = processData.getTasks(); + processDefinition.setGlobalParamList(processData.getGlobalParams()); + ProcessDag processDag = DagHelper.getProcessDag(taskNodeList); + + // Generate concrete Dag to be executed + return DagHelper.buildDagGraph(processDag); + } + + return new DAG<>(); + } + + /** + * whether the graph has a ring + * + * @param taskNodeResponseList task node response list + * @return if graph has cycle flag + */ + private boolean graphHasCycle(List taskNodeResponseList) { + DAG graph = new DAG<>(); + + // Fill the vertices + for (TaskNode taskNodeResponse : taskNodeResponseList) { + graph.addNode(taskNodeResponse.getName(), taskNodeResponse); + } + + // Fill edge relations + for (TaskNode taskNodeResponse : taskNodeResponseList) { + taskNodeResponse.getPreTasks(); + List preTasks = JSONUtils.toList(taskNodeResponse.getPreTasks(), String.class); + if (CollectionUtils.isNotEmpty(preTasks)) { + for (String preTask : preTasks) { + if (!graph.addEdge(preTask, taskNodeResponse.getName())) { + return true; + } + } + } + } + + return graph.hasCycle(); + } + + private String recursionProcessDefinitionName(Integer projectId, String processDefinitionName, int num) { + ProcessDefinition processDefinition = processDefineMapper.queryByDefineName(projectId, processDefinitionName); + if (processDefinition != null) { + if (num > 1) { + String str = processDefinitionName.substring(0, processDefinitionName.length() - 3); + processDefinitionName = str + "(" + num + ")"; + } else { + processDefinitionName = processDefinition.getName() + "(" + num + ")"; + } + } else { + return processDefinitionName; + } + return recursionProcessDefinitionName(projectId, processDefinitionName, num + 1); + } + + private Map copyProcessDefinition(User loginUser, + Integer processId, + Project targetProject) throws JsonProcessingException { + + Map result = new HashMap<>(); + + ProcessDefinition processDefinition = processDefineMapper.selectById(processId); + if (processDefinition == null) { + putMsg(result, Status.PROCESS_DEFINE_NOT_EXIST, processId); + return result; + } else { + return createProcessDefinition( + loginUser, + targetProject.getName(), + processDefinition.getName() + "_copy_" + System.currentTimeMillis(), + processDefinition.getProcessDefinitionJson(), + processDefinition.getDescription(), + processDefinition.getLocations(), + processDefinition.getConnects()); + + } + } + + /** + * batch copy process definition + * + * @param loginUser loginUser + * @param projectName projectName + * @param processDefinitionIds processDefinitionIds + * @param targetProjectId targetProjectId + */ + @Override + public Map batchCopyProcessDefinition(User loginUser, + String projectName, + String processDefinitionIds, + int targetProjectId) { + Map result = new HashMap<>(); + List failedProcessList = new ArrayList<>(); + + if (StringUtils.isEmpty(processDefinitionIds)) { + putMsg(result, Status.PROCESS_DEFINITION_IDS_IS_EMPTY, processDefinitionIds); + return result; + } + + //check src project auth + Map checkResult = checkProjectAndAuth(loginUser, projectName); + if (checkResult != null) { + return checkResult; + } + + Project targetProject = projectMapper.queryDetailById(targetProjectId); + if (targetProject == null) { + putMsg(result, Status.PROJECT_NOT_FOUNT, targetProjectId); + return result; + } + + if (!(targetProject.getName()).equals(projectName)) { + Map checkTargetProjectResult = checkProjectAndAuth(loginUser, targetProject.getName()); + if (checkTargetProjectResult != null) { + return checkTargetProjectResult; + } + } + + String[] processDefinitionIdList = processDefinitionIds.split(Constants.COMMA); + doBatchCopyProcessDefinition(loginUser, targetProject, failedProcessList, processDefinitionIdList); + + checkBatchOperateResult(projectName, targetProject.getName(), result, failedProcessList, true); + + return result; + } + + /** + * batch move process definition + * + * @param loginUser loginUser + * @param projectName projectName + * @param processDefinitionIds processDefinitionIds + * @param targetProjectId targetProjectId + */ + @Override + public Map batchMoveProcessDefinition(User loginUser, + String projectName, + String processDefinitionIds, + int targetProjectId) { + Map result = new HashMap<>(); + List failedProcessList = new ArrayList<>(); + + //check src project auth + Map checkResult = checkProjectAndAuth(loginUser, projectName); + if (checkResult != null) { + return checkResult; + } + + if (StringUtils.isEmpty(processDefinitionIds)) { + putMsg(result, Status.PROCESS_DEFINITION_IDS_IS_EMPTY, processDefinitionIds); + return result; + } + + Project targetProject = projectMapper.queryDetailById(targetProjectId); + if (targetProject == null) { + putMsg(result, Status.PROJECT_NOT_FOUNT, targetProjectId); + return result; + } + + if (!(targetProject.getName()).equals(projectName)) { + Map checkTargetProjectResult = checkProjectAndAuth(loginUser, targetProject.getName()); + if (checkTargetProjectResult != null) { + return checkTargetProjectResult; + } + } + + String[] processDefinitionIdList = processDefinitionIds.split(Constants.COMMA); + doBatchMoveProcessDefinition(targetProject, failedProcessList, processDefinitionIdList); + + checkBatchOperateResult(projectName, targetProject.getName(), result, failedProcessList, false); + + return result; + } + + /** + * switch the defined process definition verison + * + * @param loginUser login user + * @param projectName project name + * @param processDefinitionId process definition id + * @param version the version user want to switch + * @return switch process definition version result code + */ + @Override + public Map switchProcessDefinitionVersion(User loginUser, String projectName + , int processDefinitionId, long version) { + + Map result = new HashMap<>(); + Project project = projectMapper.queryByName(projectName); + // check project auth + Map checkResult = projectService.checkProjectAndAuth(loginUser, project, projectName); + Status resultStatus = (Status) checkResult.get(Constants.STATUS); + if (resultStatus != Status.SUCCESS) { + return checkResult; + } + + ProcessDefinition processDefinition = processDefineMapper.queryByDefineId(processDefinitionId); + if (Objects.isNull(processDefinition)) { + putMsg(result + , Status.SWITCH_PROCESS_DEFINITION_VERSION_NOT_EXIST_PROCESS_DEFINITION_ERROR + , processDefinitionId); + return result; + } + + ProcessDefinitionVersion processDefinitionVersion = processDefinitionVersionService + .queryByProcessDefinitionIdAndVersion(processDefinitionId, version); + if (Objects.isNull(processDefinitionVersion)) { + putMsg(result + , Status.SWITCH_PROCESS_DEFINITION_VERSION_NOT_EXIST_PROCESS_DEFINITION_VERSION_ERROR + , processDefinitionId + , version); + return result; + } + + processDefinition.setVersion(processDefinitionVersion.getVersion()); + processDefinition.setProcessDefinitionJson(processDefinitionVersion.getProcessDefinitionJson()); + processDefinition.setDescription(processDefinitionVersion.getDescription()); + processDefinition.setLocations(processDefinitionVersion.getLocations()); + processDefinition.setConnects(processDefinitionVersion.getConnects()); + processDefinition.setTimeout(processDefinitionVersion.getTimeout()); + processDefinition.setGlobalParams(processDefinitionVersion.getGlobalParams()); + processDefinition.setUpdateTime(new Date()); + processDefinition.setReceivers(processDefinitionVersion.getReceivers()); + processDefinition.setReceiversCc(processDefinitionVersion.getReceiversCc()); + processDefinition.setResourceIds(processDefinitionVersion.getResourceIds()); + + if (processDefineMapper.updateById(processDefinition) > 0) { + putMsg(result, Status.SUCCESS); + } else { + putMsg(result, Status.SWITCH_PROCESS_DEFINITION_VERSION_ERROR); + } + return result; + } + + /** + * do batch move process definition + * + * @param targetProject targetProject + * @param failedProcessList failedProcessList + * @param processDefinitionIdList processDefinitionIdList + */ + private void doBatchMoveProcessDefinition(Project targetProject, List failedProcessList, String[] processDefinitionIdList) { + for (String processDefinitionId : processDefinitionIdList) { + try { + Map moveProcessDefinitionResult = + moveProcessDefinition(Integer.valueOf(processDefinitionId), targetProject); + if (!Status.SUCCESS.equals(moveProcessDefinitionResult.get(Constants.STATUS))) { + setFailedProcessList(failedProcessList, processDefinitionId); + logger.error((String) moveProcessDefinitionResult.get(Constants.MSG)); + } + } catch (Exception e) { + setFailedProcessList(failedProcessList, processDefinitionId); + } + } + } + + /** + * batch copy process definition + * + * @param loginUser loginUser + * @param targetProject targetProject + * @param failedProcessList failedProcessList + * @param processDefinitionIdList processDefinitionIdList + */ + private void doBatchCopyProcessDefinition(User loginUser, Project targetProject, List failedProcessList, String[] processDefinitionIdList) { + for (String processDefinitionId : processDefinitionIdList) { + try { + Map copyProcessDefinitionResult = + copyProcessDefinition(loginUser, Integer.valueOf(processDefinitionId), targetProject); + if (!Status.SUCCESS.equals(copyProcessDefinitionResult.get(Constants.STATUS))) { + setFailedProcessList(failedProcessList, processDefinitionId); + logger.error((String) copyProcessDefinitionResult.get(Constants.MSG)); + } + } catch (Exception e) { + setFailedProcessList(failedProcessList, processDefinitionId); + } + } + } + + /** + * set failed processList + * + * @param failedProcessList failedProcessList + * @param processDefinitionId processDefinitionId + */ + private void setFailedProcessList(List failedProcessList, String processDefinitionId) { + ProcessDefinition processDefinition = processDefineMapper.queryByDefineId(Integer.valueOf(processDefinitionId)); + if (processDefinition != null) { + failedProcessList.add(processDefinitionId + "[" + processDefinition.getName() + "]"); + } else { + failedProcessList.add(processDefinitionId + "[null]"); + } + } + + /** + * check project and auth + * + * @param loginUser loginUser + * @param projectName projectName + */ + private Map checkProjectAndAuth(User loginUser, String projectName) { + Project project = projectMapper.queryByName(projectName); + + //check user access for project + Map checkResult = projectService.checkProjectAndAuth(loginUser, project, projectName); + Status resultStatus = (Status) checkResult.get(Constants.STATUS); + + if (resultStatus != Status.SUCCESS) { + return checkResult; + } + return null; + } + + /** + * move process definition + * + * @param processId processId + * @param targetProject targetProject + * @return move result code + */ + private Map moveProcessDefinition(Integer processId, + Project targetProject) { + + Map result = new HashMap<>(); + + ProcessDefinition processDefinition = processDefineMapper.selectById(processId); + if (processDefinition == null) { + putMsg(result, Status.PROCESS_DEFINE_NOT_EXIST, processId); + return result; + } + + processDefinition.setProjectId(targetProject.getId()); + processDefinition.setUpdateTime(new Date()); + if (processDefineMapper.updateById(processDefinition) > 0) { + putMsg(result, Status.SUCCESS); + } else { + putMsg(result, Status.UPDATE_PROCESS_DEFINITION_ERROR); + } + return result; + } + + /** + * check batch operate result + * + * @param srcProjectName srcProjectName + * @param targetProjectName targetProjectName + * @param result result + * @param failedProcessList failedProcessList + * @param isCopy isCopy + */ + private void checkBatchOperateResult(String srcProjectName, String targetProjectName, + Map result, List failedProcessList, boolean isCopy) { + if (!failedProcessList.isEmpty()) { + if (isCopy) { + putMsg(result, Status.COPY_PROCESS_DEFINITION_ERROR, srcProjectName, targetProjectName, String.join(",", failedProcessList)); + } else { + putMsg(result, Status.MOVE_PROCESS_DEFINITION_ERROR, srcProjectName, targetProjectName, String.join(",", failedProcessList)); + } + } else { + putMsg(result, Status.SUCCESS); + } + } + +} + diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionVersionServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionVersionServiceImpl.java new file mode 100644 index 0000000000..6364242190 --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionVersionServiceImpl.java @@ -0,0 +1,181 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.service.impl; + +import org.apache.dolphinscheduler.api.enums.Status; +import org.apache.dolphinscheduler.api.service.BaseService; +import org.apache.dolphinscheduler.api.service.ProcessDefinitionVersionService; +import org.apache.dolphinscheduler.api.service.ProjectService; +import org.apache.dolphinscheduler.api.utils.PageInfo; +import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; +import org.apache.dolphinscheduler.dao.entity.ProcessDefinitionVersion; +import org.apache.dolphinscheduler.dao.entity.Project; +import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionVersionMapper; +import org.apache.dolphinscheduler.dao.mapper.ProjectMapper; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import com.google.common.collect.ImmutableMap; + +@Service +public class ProcessDefinitionVersionServiceImpl extends BaseService implements + ProcessDefinitionVersionService { + + @Autowired + private ProcessDefinitionVersionMapper processDefinitionVersionMapper; + + @Autowired + private ProjectService projectService; + + @Autowired + private ProjectMapper projectMapper; + + /** + * add the newest version of one process definition + * + * @param processDefinition the process definition that need to record version + * @return the newest version number of this process definition + */ + public long addProcessDefinitionVersion(ProcessDefinition processDefinition) { + + long version = this.queryMaxVersionByProcessDefinitionId(processDefinition.getId()) + 1; + + ProcessDefinitionVersion processDefinitionVersion = ProcessDefinitionVersion + .newBuilder() + .processDefinitionId(processDefinition.getId()) + .version(version) + .processDefinitionJson(processDefinition.getProcessDefinitionJson()) + .description(processDefinition.getDescription()) + .locations(processDefinition.getLocations()) + .connects(processDefinition.getConnects()) + .timeout(processDefinition.getTimeout()) + .globalParams(processDefinition.getGlobalParams()) + .createTime(processDefinition.getUpdateTime()) + .receivers(processDefinition.getReceivers()) + .receiversCc(processDefinition.getReceiversCc()) + .resourceIds(processDefinition.getResourceIds()) + .build(); + + processDefinitionVersionMapper.insert(processDefinitionVersion); + + return version; + } + + /** + * query the max version number by the process definition id + * + * @param processDefinitionId process definition id + * @return the max version number of this id + */ + private long queryMaxVersionByProcessDefinitionId(int processDefinitionId) { + Long maxVersion = processDefinitionVersionMapper.queryMaxVersionByProcessDefinitionId(processDefinitionId); + if (Objects.isNull(maxVersion)) { + return 0L; + } else { + return maxVersion; + } + } + + /** + * query the pagination versions info by one certain process definition id + * + * @param loginUser login user info to check auth + * @param projectName process definition project name + * @param pageNo page number + * @param pageSize page size + * @param processDefinitionId process definition id + * @return the pagination process definition versions info of the certain process definition + */ + public Map queryProcessDefinitionVersions(User loginUser, String projectName, int pageNo, int pageSize, int processDefinitionId) { + + Map result = new HashMap<>(); + + // check the if pageNo or pageSize less than 1 + if (pageNo <= 0 || pageSize <= 0) { + putMsg(result + , Status.QUERY_PROCESS_DEFINITION_VERSIONS_PAGE_NO_OR_PAGE_SIZE_LESS_THAN_1_ERROR + , pageNo + , pageSize); + return result; + } + + Project project = projectMapper.queryByName(projectName); + + // check project auth + Map checkResult = projectService.checkProjectAndAuth(loginUser, project, projectName); + Status resultStatus = (Status) checkResult.get(Constants.STATUS); + if (resultStatus != Status.SUCCESS) { + return checkResult; + } + + PageInfo pageInfo = new PageInfo<>(pageNo, pageSize); + Page page = new Page<>(pageNo, pageSize); + IPage processDefinitionVersionsPaging = processDefinitionVersionMapper.queryProcessDefinitionVersionsPaging(page, processDefinitionId); + List processDefinitionVersions = processDefinitionVersionsPaging.getRecords(); + pageInfo.setLists(processDefinitionVersions); + pageInfo.setTotalCount((int) processDefinitionVersionsPaging.getTotal()); + return ImmutableMap.of( + Constants.MSG, Status.SUCCESS.getMsg() + , Constants.STATUS, Status.SUCCESS + , Constants.DATA_LIST, pageInfo); + } + + /** + * query one certain process definition version by version number and process definition id + * + * @param processDefinitionId process definition id + * @param version version number + * @return the process definition version info + */ + public ProcessDefinitionVersion queryByProcessDefinitionIdAndVersion(int processDefinitionId, long version) { + return processDefinitionVersionMapper.queryByProcessDefinitionIdAndVersion(processDefinitionId, version); + } + + /** + * delete one certain process definition by version number and process definition id + * + * @param loginUser login user info to check auth + * @param projectName process definition project name + * @param processDefinitionId process definition id + * @param version version number + * @return delele result code + */ + public Map deleteByProcessDefinitionIdAndVersion(User loginUser, String projectName, int processDefinitionId, long version) { + Map result = new HashMap<>(); + Project project = projectMapper.queryByName(projectName); + // check project auth + Map checkResult = projectService.checkProjectAndAuth(loginUser, project, projectName); + Status resultStatus = (Status) checkResult.get(Constants.STATUS); + if (resultStatus != Status.SUCCESS) { + return checkResult; + } + processDefinitionVersionMapper.deleteByProcessDefinitionIdAndVersion(processDefinitionId, version); + putMsg(result, Status.SUCCESS); + return result; + } +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectServiceImpl.java new file mode 100644 index 0000000000..395da6027f --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectServiceImpl.java @@ -0,0 +1,443 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.api.service.impl; + +import static org.apache.dolphinscheduler.api.utils.CheckUtils.checkDesc; + +import org.apache.dolphinscheduler.api.enums.Status; +import org.apache.dolphinscheduler.api.service.BaseService; +import org.apache.dolphinscheduler.api.service.ProjectService; +import org.apache.dolphinscheduler.api.utils.PageInfo; +import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.enums.UserType; +import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; +import org.apache.dolphinscheduler.dao.entity.Project; +import org.apache.dolphinscheduler.dao.entity.ProjectUser; +import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper; +import org.apache.dolphinscheduler.dao.mapper.ProjectMapper; +import org.apache.dolphinscheduler.dao.mapper.ProjectUserMapper; + +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; + +/** + * project service implement + **/ +@Service +public class ProjectServiceImpl extends BaseService implements ProjectService { + + @Autowired + private ProjectMapper projectMapper; + + @Autowired + private ProjectUserMapper projectUserMapper; + + @Autowired + private ProcessDefinitionMapper processDefinitionMapper; + + /** + * create project + * + * @param loginUser login user + * @param name project name + * @param desc description + * @return returns an error if it exists + */ + public Map createProject(User loginUser, String name, String desc) { + + Map result = new HashMap<>(); + Map descCheck = checkDesc(desc); + if (descCheck.get(Constants.STATUS) != Status.SUCCESS) { + return descCheck; + } + + Project project = projectMapper.queryByName(name); + if (project != null) { + putMsg(result, Status.PROJECT_ALREADY_EXISTS, name); + return result; + } + + Date now = new Date(); + + project = Project + .newBuilder() + .name(name) + .description(desc) + .userId(loginUser.getId()) + .userName(loginUser.getUserName()) + .createTime(now) + .updateTime(now) + .build(); + + if (projectMapper.insert(project) > 0) { + Project insertedProject = projectMapper.queryByName(name); + result.put(Constants.DATA_LIST, insertedProject); + putMsg(result, Status.SUCCESS); + } else { + putMsg(result, Status.CREATE_PROJECT_ERROR); + } + return result; + } + + /** + * query project details by id + * + * @param projectId project id + * @return project detail information + */ + public Map queryById(Integer projectId) { + + Map result = new HashMap<>(); + Project project = projectMapper.selectById(projectId); + + if (project != null) { + result.put(Constants.DATA_LIST, project); + putMsg(result, Status.SUCCESS); + } else { + putMsg(result, Status.PROJECT_NOT_FOUNT, projectId); + } + return result; + } + + /** + * check project and authorization + * + * @param loginUser login user + * @param project project + * @param projectName project name + * @return true if the login user have permission to see the project + */ + public Map checkProjectAndAuth(User loginUser, Project project, String projectName) { + Map result = new HashMap<>(); + if (project == null) { + putMsg(result, Status.PROJECT_NOT_FOUNT, projectName); + } else if (!checkReadPermission(loginUser, project)) { + // check read permission + putMsg(result, Status.USER_NO_OPERATION_PROJECT_PERM, loginUser.getUserName(), projectName); + } else { + putMsg(result, Status.SUCCESS); + } + return result; + } + + public boolean hasProjectAndPerm(User loginUser, Project project, Map result) { + boolean checkResult = false; + if (project == null) { + putMsg(result, Status.PROJECT_NOT_FOUNT, ""); + } else if (!checkReadPermission(loginUser, project)) { + putMsg(result, Status.USER_NO_OPERATION_PROJECT_PERM, loginUser.getUserName(), project.getName()); + } else { + checkResult = true; + } + return checkResult; + } + + /** + * admin can view all projects + * + * @param loginUser login user + * @param searchVal search value + * @param pageSize page size + * @param pageNo page number + * @return project list which the login user have permission to see + */ + public Map queryProjectListPaging(User loginUser, Integer pageSize, Integer pageNo, String searchVal) { + Map result = new HashMap<>(); + PageInfo pageInfo = new PageInfo<>(pageNo, pageSize); + + Page page = new Page<>(pageNo, pageSize); + + int userId = loginUser.getUserType() == UserType.ADMIN_USER ? 0 : loginUser.getId(); + IPage projectIPage = projectMapper.queryProjectListPaging(page, userId, searchVal); + + List projectList = projectIPage.getRecords(); + if (userId != 0) { + for (Project project : projectList) { + project.setPerm(Constants.DEFAULT_ADMIN_PERMISSION); + } + } + pageInfo.setTotalCount((int) projectIPage.getTotal()); + pageInfo.setLists(projectList); + result.put(Constants.COUNT, (int) projectIPage.getTotal()); + result.put(Constants.DATA_LIST, pageInfo); + putMsg(result, Status.SUCCESS); + + return result; + } + + /** + * delete project by id + * + * @param loginUser login user + * @param projectId project id + * @return delete result code + */ + public Map deleteProject(User loginUser, Integer projectId) { + Map result = new HashMap<>(); + Project project = projectMapper.selectById(projectId); + Map checkResult = getCheckResult(loginUser, project); + if (checkResult != null) { + return checkResult; + } + + if (!hasPerm(loginUser, project.getUserId())) { + putMsg(result, Status.USER_NO_OPERATION_PERM); + return result; + } + + List processDefinitionList = processDefinitionMapper.queryAllDefinitionList(projectId); + + if (!processDefinitionList.isEmpty()) { + putMsg(result, Status.DELETE_PROJECT_ERROR_DEFINES_NOT_NULL); + return result; + } + int delete = projectMapper.deleteById(projectId); + if (delete > 0) { + putMsg(result, Status.SUCCESS); + } else { + putMsg(result, Status.DELETE_PROJECT_ERROR); + } + return result; + } + + /** + * get check result + * + * @param loginUser login user + * @param project project + * @return check result + */ + private Map getCheckResult(User loginUser, Project project) { + String projectName = project == null ? null : project.getName(); + Map checkResult = checkProjectAndAuth(loginUser, project, projectName); + Status status = (Status) checkResult.get(Constants.STATUS); + if (status != Status.SUCCESS) { + return checkResult; + } + return null; + } + + /** + * updateProcessInstance project + * + * @param loginUser login user + * @param projectId project id + * @param projectName project name + * @param desc description + * @return update result code + */ + public Map update(User loginUser, Integer projectId, String projectName, String desc) { + Map result = new HashMap<>(); + + Map descCheck = checkDesc(desc); + if (descCheck.get(Constants.STATUS) != Status.SUCCESS) { + return descCheck; + } + + Project project = projectMapper.selectById(projectId); + boolean hasProjectAndPerm = hasProjectAndPerm(loginUser, project, result); + if (!hasProjectAndPerm) { + return result; + } + Project tempProject = projectMapper.queryByName(projectName); + if (tempProject != null && tempProject.getId() != projectId) { + putMsg(result, Status.PROJECT_ALREADY_EXISTS, projectName); + return result; + } + project.setName(projectName); + project.setDescription(desc); + project.setUpdateTime(new Date()); + + int update = projectMapper.updateById(project); + if (update > 0) { + putMsg(result, Status.SUCCESS); + } else { + putMsg(result, Status.UPDATE_PROJECT_ERROR); + } + return result; + } + + + /** + * query unauthorized project + * + * @param loginUser login user + * @param userId user id + * @return the projects which user have not permission to see + */ + public Map queryUnauthorizedProject(User loginUser, Integer userId) { + Map result = new HashMap<>(); + if (checkAdmin(loginUser, result)) { + return result; + } + /** + * query all project list except specified userId + */ + List projectList = projectMapper.queryProjectExceptUserId(userId); + List resultList = new ArrayList<>(); + Set projectSet = null; + if (projectList != null && !projectList.isEmpty()) { + projectSet = new HashSet<>(projectList); + + List authedProjectList = projectMapper.queryAuthedProjectListByUserId(userId); + + resultList = getUnauthorizedProjects(projectSet, authedProjectList); + } + result.put(Constants.DATA_LIST, resultList); + putMsg(result, Status.SUCCESS); + return result; + } + + /** + * get unauthorized project + * + * @param projectSet project set + * @param authedProjectList authed project list + * @return project list that authorization + */ + private List getUnauthorizedProjects(Set projectSet, List authedProjectList) { + List resultList; + Set authedProjectSet = null; + if (authedProjectList != null && !authedProjectList.isEmpty()) { + authedProjectSet = new HashSet<>(authedProjectList); + projectSet.removeAll(authedProjectSet); + + } + resultList = new ArrayList<>(projectSet); + return resultList; + } + + + /** + * query authorized project + * + * @param loginUser login user + * @param userId user id + * @return projects which the user have permission to see, Except for items created by this user + */ + public Map queryAuthorizedProject(User loginUser, Integer userId) { + Map result = new HashMap<>(); + + if (checkAdmin(loginUser, result)) { + return result; + } + + List projects = projectMapper.queryAuthedProjectListByUserId(userId); + result.put(Constants.DATA_LIST, projects); + putMsg(result, Status.SUCCESS); + + return result; + } + + /** + * query authorized project + * + * @param loginUser login user + * @return projects which the user have permission to see, Except for items created by this user + */ + public Map queryProjectCreatedByUser(User loginUser) { + Map result = new HashMap<>(); + + if (checkAdmin(loginUser, result)) { + return result; + } + + List projects = projectMapper.queryProjectCreatedByUser(loginUser.getId()); + result.put(Constants.DATA_LIST, projects); + putMsg(result, Status.SUCCESS); + + return result; + } + + /** + * check whether have read permission + * + * @param user user + * @param project project + * @return true if the user have permission to see the project, otherwise return false + */ + private boolean checkReadPermission(User user, Project project) { + int permissionId = queryPermission(user, project); + return (permissionId & Constants.READ_PERMISSION) != 0; + } + + /** + * query permission id + * + * @param user user + * @param project project + * @return permission + */ + private int queryPermission(User user, Project project) { + if (user.getUserType() == UserType.ADMIN_USER) { + return Constants.READ_PERMISSION; + } + + if (project.getUserId() == user.getId()) { + return Constants.ALL_PERMISSIONS; + } + + ProjectUser projectUser = projectUserMapper.queryProjectRelation(project.getId(), user.getId()); + + if (projectUser == null) { + return 0; + } + + return projectUser.getPerm(); + + } + + /** + * query all project list that have one or more process definitions. + * + * @return project list + */ + public Map queryAllProjectList() { + Map result = new HashMap<>(); + List projects = projectMapper.selectList(null); + List processDefinitions = processDefinitionMapper.selectList(null); + if (projects != null) { + Set set = new HashSet<>(); + for (ProcessDefinition processDefinition : processDefinitions) { + set.add(processDefinition.getProjectId()); + } + List tempDeletelist = new ArrayList<>(); + for (Project project : projects) { + if (!set.contains(project.getId())) { + tempDeletelist.add(project); + } + } + projects.removeAll(tempDeletelist); + } + result.put(Constants.DATA_LIST, projects); + putMsg(result, Status.SUCCESS); + return result; + } + +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/SessionServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/SessionServiceImpl.java new file mode 100644 index 0000000000..8aaefdadff --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/SessionServiceImpl.java @@ -0,0 +1,158 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.service.impl; + +import java.util.Date; +import java.util.List; +import java.util.UUID; + +import javax.servlet.http.Cookie; +import javax.servlet.http.HttpServletRequest; + +import org.apache.commons.lang.StringUtils; +import org.apache.dolphinscheduler.api.controller.BaseController; +import org.apache.dolphinscheduler.api.service.BaseService; +import org.apache.dolphinscheduler.api.service.SessionService; +import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.utils.CollectionUtils; +import org.apache.dolphinscheduler.dao.entity.Session; +import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.dao.mapper.SessionMapper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +/** + * session service implement + */ +@Service +public class SessionServiceImpl extends BaseService implements SessionService { + + private static final Logger logger = LoggerFactory.getLogger(SessionService.class); + + @Autowired + private SessionMapper sessionMapper; + + /** + * get user session from request + * + * @param request request + * @return session + */ + public Session getSession(HttpServletRequest request) { + String sessionId = request.getHeader(Constants.SESSION_ID); + + if (StringUtils.isBlank(sessionId)) { + Cookie cookie = getCookie(request, Constants.SESSION_ID); + + if (cookie != null) { + sessionId = cookie.getValue(); + } + } + + if (StringUtils.isBlank(sessionId)) { + return null; + } + + String ip = BaseController.getClientIpAddress(request); + logger.debug("get session: {}, ip: {}", sessionId, ip); + + return sessionMapper.selectById(sessionId); + } + + /** + * create session + * + * @param user user + * @param ip ip + * @return session string + */ + @Transactional(rollbackFor = RuntimeException.class) + public String createSession(User user, String ip) { + Session session = null; + + // logined + List sessionList = sessionMapper.queryByUserId(user.getId()); + + Date now = new Date(); + + /** + * if you have logged in and are still valid, return directly + */ + if (CollectionUtils.isNotEmpty(sessionList)) { + // is session list greater 1 ， delete other ，get one + if (sessionList.size() > 1) { + for (int i = 1; i < sessionList.size(); i++) { + sessionMapper.deleteById(sessionList.get(i).getId()); + } + } + session = sessionList.get(0); + if (now.getTime() - session.getLastLoginTime().getTime() <= Constants.SESSION_TIME_OUT * 1000) { + /** + * updateProcessInstance the latest login time + */ + session.setLastLoginTime(now); + sessionMapper.updateById(session); + + return session.getId(); + + } else { + /** + * session expired, then delete this session first + */ + sessionMapper.deleteById(session.getId()); + } + } + + // assign new session + session = new Session(); + + session.setId(UUID.randomUUID().toString()); + session.setIp(ip); + session.setUserId(user.getId()); + session.setLastLoginTime(now); + + sessionMapper.insert(session); + + return session.getId(); + } + + /** + * sign out + * remove ip restrictions + * + * @param ip no use + * @param loginUser login user + */ + public void signOut(String ip, User loginUser) { + try { + /** + * query session by user id and ip + */ + Session session = sessionMapper.queryByUserIdAndIp(loginUser.getId(), ip); + + //delete session + sessionMapper.deleteById(session.getId()); + } catch (Exception e) { + logger.warn("userId : {} , ip : {} , find more one session", loginUser.getId(), ip); + } + } + +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TenantServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TenantServiceImpl.java new file mode 100644 index 0000000000..3a267bcc8c --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TenantServiceImpl.java @@ -0,0 +1,331 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.service.impl; + +import org.apache.dolphinscheduler.api.enums.Status; +import org.apache.dolphinscheduler.api.service.BaseService; +import org.apache.dolphinscheduler.api.service.TenantService; +import org.apache.dolphinscheduler.api.utils.PageInfo; +import org.apache.dolphinscheduler.api.utils.Result; +import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.utils.CollectionUtils; +import org.apache.dolphinscheduler.common.utils.HadoopUtils; +import org.apache.dolphinscheduler.common.utils.PropertyUtils; +import org.apache.dolphinscheduler.common.utils.StringUtils; +import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; +import org.apache.dolphinscheduler.dao.entity.ProcessInstance; +import org.apache.dolphinscheduler.dao.entity.Tenant; +import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper; +import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper; +import org.apache.dolphinscheduler.dao.mapper.TenantMapper; +import org.apache.dolphinscheduler.dao.mapper.UserMapper; + +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; + +/** + * tenant service + */ +@Service +public class TenantServiceImpl extends BaseService implements TenantService { + + private static final Logger logger = LoggerFactory.getLogger(TenantServiceImpl.class); + + @Autowired + private TenantMapper tenantMapper; + + @Autowired + private ProcessInstanceMapper processInstanceMapper; + + @Autowired + private ProcessDefinitionMapper processDefinitionMapper; + + @Autowired + private UserMapper userMapper; + + /** + * create tenant + * + * @param loginUser login user + * @param tenantCode tenant code + * @param tenantName tenant name + * @param queueId queue id + * @param desc description + * @return create result code + * @throws Exception exception + */ + @Transactional(rollbackFor = Exception.class) + public Map createTenant(User loginUser, + String tenantCode, + String tenantName, + int queueId, + String desc) throws Exception { + + Map result = new HashMap<>(5); + result.put(Constants.STATUS, false); + if (checkAdmin(loginUser, result)) { + return result; + } + + if (checkTenantExists(tenantCode)) { + putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, tenantCode); + return result; + } + + Tenant tenant = new Tenant(); + Date now = new Date(); + + if (!tenantCode.matches("^[0-9a-zA-Z_.-]{1,}$") || tenantCode.startsWith("-") || tenantCode.startsWith(".")) { + putMsg(result, Status.VERIFY_TENANT_CODE_ERROR); + return result; + } + tenant.setTenantCode(tenantCode); + tenant.setTenantName(tenantName); + tenant.setQueueId(queueId); + tenant.setDescription(desc); + tenant.setCreateTime(now); + tenant.setUpdateTime(now); + + // save + tenantMapper.insert(tenant); + + // if hdfs startup + if (PropertyUtils.getResUploadStartupState()) { + createTenantDirIfNotExists(tenantCode); + } + + putMsg(result, Status.SUCCESS); + + return result; + } + + /** + * query tenant list paging + * + * @param loginUser login user + * @param searchVal search value + * @param pageNo page number + * @param pageSize page size + * @return tenant list page + */ + public Map queryTenantList(User loginUser, String searchVal, Integer pageNo, Integer pageSize) { + + Map result = new HashMap<>(5); + if (checkAdmin(loginUser, result)) { + return result; + } + + Page page = new Page<>(pageNo, pageSize); + IPage tenantIPage = tenantMapper.queryTenantPaging(page, searchVal); + PageInfo pageInfo = new PageInfo<>(pageNo, pageSize); + pageInfo.setTotalCount((int) tenantIPage.getTotal()); + pageInfo.setLists(tenantIPage.getRecords()); + result.put(Constants.DATA_LIST, pageInfo); + + putMsg(result, Status.SUCCESS); + + return result; + } + + /** + * updateProcessInstance tenant + * + * @param loginUser login user + * @param id tennat id + * @param tenantCode tennat code + * @param tenantName tennat name + * @param queueId queue id + * @param desc description + * @return update result code + * @throws Exception exception + */ + public Map updateTenant(User loginUser, int id, String tenantCode, String tenantName, int queueId, + String desc) throws Exception { + + Map result = new HashMap<>(5); + result.put(Constants.STATUS, false); + + if (checkAdmin(loginUser, result)) { + return result; + } + + Tenant tenant = tenantMapper.queryById(id); + + if (tenant == null) { + putMsg(result, Status.TENANT_NOT_EXIST); + return result; + } + + // updateProcessInstance tenant + /** + * if the tenant code is modified, the original resource needs to be copied to the new tenant. + */ + if (!tenant.getTenantCode().equals(tenantCode)) { + if (checkTenantExists(tenantCode)) { + // if hdfs startup + if (PropertyUtils.getResUploadStartupState()) { + String resourcePath = HadoopUtils.getHdfsDataBasePath() + "/" + tenantCode + "/resources"; + String udfsPath = HadoopUtils.getHdfsUdfDir(tenantCode); + //init hdfs resource + HadoopUtils.getInstance().mkdir(resourcePath); + HadoopUtils.getInstance().mkdir(udfsPath); + } + } else { + putMsg(result, Status.TENANT_CODE_HAS_ALREADY_EXISTS); + return result; + } + } + + Date now = new Date(); + + if (StringUtils.isNotEmpty(tenantCode)) { + tenant.setTenantCode(tenantCode); + } + + if (StringUtils.isNotEmpty(tenantName)) { + tenant.setTenantName(tenantName); + } + + if (queueId != 0) { + tenant.setQueueId(queueId); + } + tenant.setDescription(desc); + tenant.setUpdateTime(now); + tenantMapper.updateById(tenant); + + result.put(Constants.STATUS, Status.SUCCESS); + result.put(Constants.MSG, Status.SUCCESS.getMsg()); + return result; + } + + /** + * delete tenant + * + * @param loginUser login user + * @param id tenant id + * @return delete result code + * @throws Exception exception + */ + @Transactional(rollbackFor = Exception.class) + public Map deleteTenantById(User loginUser, int id) throws Exception { + Map result = new HashMap<>(5); + + if (checkAdmin(loginUser, result)) { + return result; + } + + Tenant tenant = tenantMapper.queryById(id); + if (tenant == null) { + putMsg(result, Status.TENANT_NOT_EXIST); + return result; + } + + List processInstances = getProcessInstancesByTenant(tenant); + if (CollectionUtils.isNotEmpty(processInstances)) { + putMsg(result, Status.DELETE_TENANT_BY_ID_FAIL, processInstances.size()); + return result; + } + + List processDefinitions = + processDefinitionMapper.queryDefinitionListByTenant(tenant.getId()); + if (CollectionUtils.isNotEmpty(processDefinitions)) { + putMsg(result, Status.DELETE_TENANT_BY_ID_FAIL_DEFINES, processDefinitions.size()); + return result; + } + + List userList = userMapper.queryUserListByTenant(tenant.getId()); + if (CollectionUtils.isNotEmpty(userList)) { + putMsg(result, Status.DELETE_TENANT_BY_ID_FAIL_USERS, userList.size()); + return result; + } + + // if resource upload startup + if (PropertyUtils.getResUploadStartupState()) { + String tenantPath = HadoopUtils.getHdfsDataBasePath() + "/" + tenant.getTenantCode(); + + if (HadoopUtils.getInstance().exists(tenantPath)) { + HadoopUtils.getInstance().delete(tenantPath, true); + } + } + + tenantMapper.deleteById(id); + processInstanceMapper.updateProcessInstanceByTenantId(id, -1); + putMsg(result, Status.SUCCESS); + return result; + } + + private List getProcessInstancesByTenant(Tenant tenant) { + return processInstanceMapper.queryByTenantIdAndStatus(tenant.getId(), Constants.NOT_TERMINATED_STATES); + } + + /** + * query tenant list + * + * @param loginUser login user + * @return tenant list + */ + public Map queryTenantList(User loginUser) { + + Map result = new HashMap<>(5); + + List resourceList = tenantMapper.selectList(null); + result.put(Constants.DATA_LIST, resourceList); + putMsg(result, Status.SUCCESS); + + return result; + } + + /** + * verify tenant code + * + * @param tenantCode tenant code + * @return true if tenant code can user, otherwise return false + */ + public Result verifyTenantCode(String tenantCode) { + Result result = new Result(); + if (checkTenantExists(tenantCode)) { + putMsg(result, Status.TENANT_NAME_EXIST, tenantCode); + } else { + putMsg(result, Status.SUCCESS); + } + return result; + } + + /** + * check tenant exists + * + * @param tenantCode tenant code + * @return ture if the tenant code exists, otherwise return false + */ + private boolean checkTenantExists(String tenantCode) { + List tenants = tenantMapper.queryByTenantCode(tenantCode); + return CollectionUtils.isNotEmpty(tenants); + } +} diff --git a/dolphinscheduler-api/src/main/resources/i18n/messages.properties b/dolphinscheduler-api/src/main/resources/i18n/messages.properties index c8e48ad865..d1da3e94a8 100644 --- a/dolphinscheduler-api/src/main/resources/i18n/messages.properties +++ b/dolphinscheduler-api/src/main/resources/i18n/messages.properties @@ -173,7 +173,6 @@ PROCESS_DEFINITION_ID=process definition id PROCESS_DEFINITION_IDS=process definition ids RELEASE_PROCESS_DEFINITION_NOTES=release process definition QUERY_PROCESS_DEFINITION_BY_ID_NOTES=query process definition by id -COPY_PROCESS_DEFINITION_NOTES=copy process definition QUERY_PROCESS_DEFINITION_LIST_NOTES=query process definition list QUERY_PROCESS_DEFINITION_LIST_PAGING_NOTES=query process definition list paging QUERY_ALL_DEFINITION_LIST_NOTES=query all definition list @@ -253,4 +252,13 @@ AUTHORIZED_DATA_SOURCE_NOTES=authorized data source DELETE_SCHEDULER_BY_ID_NOTES=delete scheduler by id QUERY_ALERT_GROUP_LIST_PAGING_NOTES=query alert group list paging EXPORT_PROCESS_DEFINITION_BY_ID_NOTES=export process definition by id -BATCH_EXPORT_PROCESS_DEFINITION_BY_IDS_NOTES= batch export process definition by ids \ No newline at end of file +BATCH_EXPORT_PROCESS_DEFINITION_BY_IDS_NOTES= batch export process definition by ids +QUERY_USER_CREATED_PROJECT_NOTES= query user created project +COPY_PROCESS_DEFINITION_NOTES= copy process definition notes +MOVE_PROCESS_DEFINITION_NOTES= move process definition notes +TARGET_PROJECT_ID= target project id +IS_COPY = is copy +DELETE_PROCESS_DEFINITION_VERSION_NOTES=delete process definition version +QUERY_PROCESS_DEFINITION_VERSIONS_NOTES=query process definition versions +SWITCH_PROCESS_DEFINITION_VERSION_NOTES=switch process definition version +VERSION=version diff --git a/dolphinscheduler-api/src/main/resources/i18n/messages_en_US.properties b/dolphinscheduler-api/src/main/resources/i18n/messages_en_US.properties index 0669e8d8cf..267f93b14d 100644 --- a/dolphinscheduler-api/src/main/resources/i18n/messages_en_US.properties +++ b/dolphinscheduler-api/src/main/resources/i18n/messages_en_US.properties @@ -173,7 +173,6 @@ PROCESS_DEFINITION_ID=process definition id PROCESS_DEFINITION_IDS=process definition ids RELEASE_PROCESS_DEFINITION_NOTES=release process definition QUERY_PROCESS_DEFINITION_BY_ID_NOTES=query process definition by id -COPY_PROCESS_DEFINITION_NOTES=copy process definition QUERY_PROCESS_DEFINITION_LIST_NOTES=query process definition list QUERY_PROCESS_DEFINITION_LIST_PAGING_NOTES=query process definition list paging QUERY_ALL_DEFINITION_LIST_NOTES=query all definition list @@ -254,3 +253,12 @@ DELETE_SCHEDULER_BY_ID_NOTES=delete scheduler by id QUERY_ALERT_GROUP_LIST_PAGING_NOTES=query alert group list paging EXPORT_PROCESS_DEFINITION_BY_ID_NOTES=export process definition by id BATCH_EXPORT_PROCESS_DEFINITION_BY_IDS_NOTES= batch export process definition by ids +QUERY_USER_CREATED_PROJECT_NOTES= query user created project +COPY_PROCESS_DEFINITION_NOTES= copy process definition notes +MOVE_PROCESS_DEFINITION_NOTES= move process definition notes +TARGET_PROJECT_ID= target project id +IS_COPY = is copy +DELETE_PROCESS_DEFINITION_VERSION_NOTES=delete process definition version +QUERY_PROCESS_DEFINITION_VERSIONS_NOTES=query process definition versions +SWITCH_PROCESS_DEFINITION_VERSION_NOTES=switch process definition version +VERSION=version diff --git a/dolphinscheduler-api/src/main/resources/i18n/messages_zh_CN.properties b/dolphinscheduler-api/src/main/resources/i18n/messages_zh_CN.properties index 9053b0924c..16262e6bbc 100644 --- a/dolphinscheduler-api/src/main/resources/i18n/messages_zh_CN.properties +++ b/dolphinscheduler-api/src/main/resources/i18n/messages_zh_CN.properties @@ -171,7 +171,6 @@ UPDATE_PROCESS_DEFINITION_NOTES=更新流程定义 PROCESS_DEFINITION_ID=流程定义ID RELEASE_PROCESS_DEFINITION_NOTES=发布流程定义 QUERY_PROCESS_DEFINITION_BY_ID_NOTES=查询流程定义通过流程定义ID -COPY_PROCESS_DEFINITION_NOTES=复制流程定义 QUERY_PROCESS_DEFINITION_LIST_NOTES=查询流程定义列表 QUERY_PROCESS_DEFINITION_LIST_PAGING_NOTES=分页查询流程定义列表 QUERY_ALL_DEFINITION_LIST_NOTES=查询所有流程定义 @@ -252,4 +251,12 @@ DELETE_SCHEDULER_BY_ID_NOTES=根据定时id删除定时数据 QUERY_ALERT_GROUP_LIST_PAGING_NOTES=分页查询告警组列表 EXPORT_PROCESS_DEFINITION_BY_ID_NOTES=通过工作流ID导出工作流定义 BATCH_EXPORT_PROCESS_DEFINITION_BY_IDS_NOTES=批量导出工作流定义 - +QUERY_USER_CREATED_PROJECT_NOTES= 查询用户创建的项目 +COPY_PROCESS_DEFINITION_NOTES= 复制工作流定义 +MOVE_PROCESS_DEFINITION_NOTES= 移动工作流定义 +TARGET_PROJECT_ID= 目标项目ID +IS_COPY = 是否复制 +DELETE_PROCESS_DEFINITION_VERSION_NOTES=删除流程历史版本 +QUERY_PROCESS_DEFINITION_VERSIONS_NOTES=查询流程历史版本信息 +SWITCH_PROCESS_DEFINITION_VERSION_NOTES=切换流程版本 +VERSION=版本号 diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProcessDefinitionControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProcessDefinitionControllerTest.java index 8c0d04c6c6..f2a54a1a88 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProcessDefinitionControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProcessDefinitionControllerTest.java @@ -14,20 +14,33 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.api.controller; import org.apache.dolphinscheduler.api.enums.Status; -import org.apache.dolphinscheduler.api.service.ProcessDefinitionService; +import org.apache.dolphinscheduler.api.service.ProcessDefinitionVersionService; +import org.apache.dolphinscheduler.api.service.impl.ProcessDefinitionServiceImpl; import org.apache.dolphinscheduler.api.utils.PageInfo; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ReleaseState; import org.apache.dolphinscheduler.common.enums.UserType; -import org.apache.dolphinscheduler.common.model.TaskNode; import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; +import org.apache.dolphinscheduler.dao.entity.ProcessDefinitionVersion; import org.apache.dolphinscheduler.dao.entity.Resource; import org.apache.dolphinscheduler.dao.entity.User; -import org.junit.*; + +import java.text.MessageFormat; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import javax.servlet.http.HttpServletResponse; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.InjectMocks; import org.mockito.Mock; @@ -36,18 +49,12 @@ import org.mockito.junit.MockitoJUnitRunner; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.mock.web.MockHttpServletResponse; -import javax.servlet.http.HttpServletResponse; -import java.text.MessageFormat; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; /** * process definition controller test */ @RunWith(MockitoJUnitRunner.Silent.class) -public class ProcessDefinitionControllerTest{ +public class ProcessDefinitionControllerTest { private static Logger logger = LoggerFactory.getLogger(ProcessDefinitionControllerTest.class); @@ -55,12 +62,15 @@ public class ProcessDefinitionControllerTest{ private ProcessDefinitionController processDefinitionController; @Mock - private ProcessDefinitionService processDefinitionService; + private ProcessDefinitionServiceImpl processDefinitionService; + + @Mock + private ProcessDefinitionVersionService processDefinitionVersionService; protected User user; @Before - public void before(){ + public void before() { User loginUser = new User(); loginUser.setId(1); loginUser.setUserType(UserType.GENERAL_USER); @@ -71,23 +81,27 @@ public class ProcessDefinitionControllerTest{ @Test public void testCreateProcessDefinition() throws Exception { - String json = "{\"globalParams\":[],\"tasks\":[{\"type\":\"SHELL\",\"id\":\"tasks-36196\",\"name\":\"ssh_test1\",\"params\":{\"resourceList\":[],\"localParams\":[],\"rawScript\":\"aa=\\\"1234\\\"\\necho ${aa}\"},\"desc\":\"\",\"runFlag\":\"NORMAL\",\"dependence\":{},\"maxRetryTimes\":\"0\",\"retryInterval\":\"1\",\"timeout\":{\"strategy\":\"\",\"interval\":null,\"enable\":false},\"taskInstancePriority\":\"MEDIUM\",\"workerGroupId\":-1,\"preTasks\":[]}],\"tenantId\":-1,\"timeout\":0}"; + String json = "{\"globalParams\":[],\"tasks\":[{\"type\":\"SHELL\",\"id\":\"tasks-36196\",\"name\"" + + ":\"ssh_test1\",\"params\":{\"resourceList\":[],\"localParams\":[],\"rawScript\":\"aa=\\\"1234\\\"\\" + + "necho ${aa}\"},\"desc\":\"\",\"runFlag\":\"NORMAL\",\"dependence\":{},\"maxRetryTimes\":\"0\"" + + ",\"retryInterval\":\"1\",\"timeout\":{\"strategy\":\"\",\"interval\":null,\"enable\":false}," + + "\"taskInstancePriority\":\"MEDIUM\",\"workerGroupId\":-1,\"preTasks\":[]}],\"tenantId\":-1,\"timeout\":0}"; String locations = "{\"tasks-36196\":{\"name\":\"ssh_test1\",\"targetarr\":\"\",\"x\":141,\"y\":70}}"; String projectName = "test"; String name = "dag_test"; String description = "desc test"; String connects = "[]"; - Map result = new HashMap<>(5); + Map result = new HashMap<>(); putMsg(result, Status.SUCCESS); - result.put("processDefinitionId",1); + result.put("processDefinitionId", 1); Mockito.when(processDefinitionService.createProcessDefinition(user, projectName, name, json, description, locations, connects)).thenReturn(result); Result response = processDefinitionController.createProcessDefinition(user, projectName, name, json, locations, connects, description); - Assert.assertEquals(Status.SUCCESS.getCode(),response.getCode().intValue()); + Assert.assertEquals(Status.SUCCESS.getCode(), response.getCode().intValue()); } private void putMsg(Map result, Status status, Object... statusParams) { @@ -102,56 +116,64 @@ public class ProcessDefinitionControllerTest{ @Test public void testVerifyProcessDefinitionName() throws Exception { - Map result = new HashMap<>(5); + Map result = new HashMap<>(); putMsg(result, Status.PROCESS_INSTANCE_EXIST); String projectName = "test"; String name = "dag_test"; - Mockito.when(processDefinitionService.verifyProcessDefinitionName(user,projectName,name)).thenReturn(result); + Mockito.when(processDefinitionService.verifyProcessDefinitionName(user, projectName, name)).thenReturn(result); - Result response = processDefinitionController.verifyProcessDefinitionName(user,projectName,name); - Assert.assertEquals(Status.PROCESS_INSTANCE_EXIST.getCode(),response.getCode().intValue()); + Result response = processDefinitionController.verifyProcessDefinitionName(user, projectName, name); + Assert.assertEquals(Status.PROCESS_INSTANCE_EXIST.getCode(), response.getCode().intValue()); } @Test public void updateProcessDefinition() throws Exception { - String json = "{\"globalParams\":[],\"tasks\":[{\"type\":\"SHELL\",\"id\":\"tasks-36196\",\"name\":\"ssh_test1\",\"params\":{\"resourceList\":[],\"localParams\":[],\"rawScript\":\"aa=\\\"1234\\\"\\necho ${aa}\"},\"desc\":\"\",\"runFlag\":\"NORMAL\",\"dependence\":{},\"maxRetryTimes\":\"0\",\"retryInterval\":\"1\",\"timeout\":{\"strategy\":\"\",\"interval\":null,\"enable\":false},\"taskInstancePriority\":\"MEDIUM\",\"workerGroupId\":-1,\"preTasks\":[]}],\"tenantId\":-1,\"timeout\":0}"; + String json = "{\"globalParams\":[],\"tasks\":[{\"type\":\"SHELL\",\"id\":\"tasks-36196\",\"name\":\"ssh_test1\"" + + ",\"params\":{\"resourceList\":[],\"localParams\":[],\"rawScript\":\"aa=\\\"1234\\\"\\necho ${aa}\"}" + + ",\"desc\":\"\",\"runFlag\":\"NORMAL\",\"dependence\":{},\"maxRetryTimes\":\"0\",\"retryInterval\"" + + ":\"1\",\"timeout\":{\"strategy\":\"\",\"interval\":null,\"enable\":false},\"taskInstancePriority\"" + + ":\"MEDIUM\",\"workerGroupId\":-1,\"preTasks\":[]}],\"tenantId\":-1,\"timeout\":0}"; String locations = "{\"tasks-36196\":{\"name\":\"ssh_test1\",\"targetarr\":\"\",\"x\":141,\"y\":70}}"; String projectName = "test"; String name = "dag_test"; String description = "desc test"; String connects = "[]"; int id = 1; - Map result = new HashMap<>(5); + Map result = new HashMap<>(); putMsg(result, Status.SUCCESS); - result.put("processDefinitionId",1); + result.put("processDefinitionId", 1); - Mockito.when(processDefinitionService.updateProcessDefinition(user, projectName, id,name, json, + Mockito.when(processDefinitionService.updateProcessDefinition(user, projectName, id, name, json, description, locations, connects)).thenReturn(result); - Result response = processDefinitionController.updateProcessDefinition(user, projectName, name,id, json, + Result response = processDefinitionController.updateProcessDefinition(user, projectName, name, id, json, locations, connects, description); - Assert.assertEquals(Status.SUCCESS.getCode(),response.getCode().intValue()); + Assert.assertEquals(Status.SUCCESS.getCode(), response.getCode().intValue()); } @Test public void testReleaseProcessDefinition() throws Exception { String projectName = "test"; int id = 1; - Map result = new HashMap<>(5); + Map result = new HashMap<>(); putMsg(result, Status.SUCCESS); - Mockito.when(processDefinitionService.releaseProcessDefinition(user, projectName,id,ReleaseState.OFFLINE.ordinal())).thenReturn(result); - Result response = processDefinitionController.releaseProcessDefinition(user, projectName,id,ReleaseState.OFFLINE.ordinal()); - Assert.assertEquals(Status.SUCCESS.getCode(),response.getCode().intValue()); + Mockito.when(processDefinitionService.releaseProcessDefinition(user, projectName, id, ReleaseState.OFFLINE.ordinal())).thenReturn(result); + Result response = processDefinitionController.releaseProcessDefinition(user, projectName, id, ReleaseState.OFFLINE.ordinal()); + Assert.assertEquals(Status.SUCCESS.getCode(), response.getCode().intValue()); } @Test public void testQueryProcessDefinitionById() throws Exception { - String json = "{\"globalParams\":[],\"tasks\":[{\"type\":\"SHELL\",\"id\":\"tasks-36196\",\"name\":\"ssh_test1\",\"params\":{\"resourceList\":[],\"localParams\":[],\"rawScript\":\"aa=\\\"1234\\\"\\necho ${aa}\"},\"desc\":\"\",\"runFlag\":\"NORMAL\",\"dependence\":{},\"maxRetryTimes\":\"0\",\"retryInterval\":\"1\",\"timeout\":{\"strategy\":\"\",\"interval\":null,\"enable\":false},\"taskInstancePriority\":\"MEDIUM\",\"workerGroupId\":-1,\"preTasks\":[]}],\"tenantId\":-1,\"timeout\":0}"; + String json = "{\"globalParams\":[],\"tasks\":[{\"type\":\"SHELL\",\"id\":\"tasks-36196\",\"name\":\"ssh_test1" + + "\",\"params\":{\"resourceList\":[],\"localParams\":[],\"rawScript\":\"aa=\\\"1234\\\"\\necho ${aa}" + + "\"},\"desc\":\"\",\"runFlag\":\"NORMAL\",\"dependence\":{},\"maxRetryTimes\":\"0\",\"retryInterval\"" + + ":\"1\",\"timeout\":{\"strategy\":\"\",\"interval\":null,\"enable\":false},\"taskInstancePriority\":" + + "\"MEDIUM\",\"workerGroupId\":-1,\"preTasks\":[]}],\"tenantId\":-1,\"timeout\":0}"; String locations = "{\"tasks-36196\":{\"name\":\"ssh_test1\",\"targetarr\":\"\",\"x\":141,\"y\":70}}"; String projectName = "test"; String name = "dag_test"; @@ -168,54 +190,73 @@ public class ProcessDefinitionControllerTest{ processDefinition.setName(name); processDefinition.setProcessDefinitionJson(json); - Map result = new HashMap<>(5); + Map result = new HashMap<>(); putMsg(result, Status.SUCCESS); result.put(Constants.DATA_LIST, processDefinition); - Mockito.when(processDefinitionService.queryProcessDefinitionById(user, projectName,id)).thenReturn(result); - Result response = processDefinitionController.queryProcessDefinitionById(user, projectName,id); + Mockito.when(processDefinitionService.queryProcessDefinitionById(user, projectName, id)).thenReturn(result); + Result response = processDefinitionController.queryProcessDefinitionById(user, projectName, id); - Assert.assertEquals(Status.SUCCESS.getCode(),response.getCode().intValue()); + Assert.assertEquals(Status.SUCCESS.getCode(), response.getCode().intValue()); } @Test - public void testCopyProcessDefinition() throws Exception { + public void testBatchCopyProcessDefinition() throws Exception { String projectName = "test"; - int id = 1; + int targetProjectId = 2; + String id = "1"; - Map result = new HashMap<>(5); + Map result = new HashMap<>(); putMsg(result, Status.SUCCESS); - Mockito.when(processDefinitionService.copyProcessDefinition(user, projectName,id)).thenReturn(result); - Result response = processDefinitionController.copyProcessDefinition(user, projectName,id); + Mockito.when(processDefinitionService.batchCopyProcessDefinition(user, projectName, id, targetProjectId)).thenReturn(result); + Result response = processDefinitionController.copyProcessDefinition(user, projectName, id, targetProjectId); - Assert.assertEquals(Status.SUCCESS.getCode(),response.getCode().intValue()); + Assert.assertEquals(Status.SUCCESS.getCode(), response.getCode().intValue()); } + @Test + public void testBatchMoveProcessDefinition() throws Exception { + + String projectName = "test"; + int targetProjectId = 2; + String id = "1"; + + Map result = new HashMap<>(); + putMsg(result, Status.SUCCESS); + + Mockito.when(processDefinitionService.batchMoveProcessDefinition(user, projectName, id, targetProjectId)).thenReturn(result); + Result response = processDefinitionController.moveProcessDefinition(user, projectName, id, targetProjectId); + + Assert.assertEquals(Status.SUCCESS.getCode(), response.getCode().intValue()); + } @Test public void testQueryProcessDefinitionList() throws Exception { String projectName = "test"; - List resourceList = getDefinitionList(); + List resourceList = getDefinitionList(); - Map result = new HashMap<>(5); + Map result = new HashMap<>(); putMsg(result, Status.SUCCESS); result.put(Constants.DATA_LIST, resourceList); - Mockito.when(processDefinitionService.queryProcessDefinitionList(user, projectName)).thenReturn(result); Result response = processDefinitionController.queryProcessDefinitionList(user, projectName); - Assert.assertEquals(Status.SUCCESS.getCode(),response.getCode().intValue()); + Assert.assertEquals(Status.SUCCESS.getCode(), response.getCode().intValue()); } - public List getDefinitionList(){ + public List getDefinitionList() { List resourceList = new ArrayList<>(); - String json = "{\"globalParams\":[],\"tasks\":[{\"type\":\"SHELL\",\"id\":\"tasks-36196\",\"name\":\"ssh_test1\",\"params\":{\"resourceList\":[],\"localParams\":[],\"rawScript\":\"aa=\\\"1234\\\"\\necho ${aa}\"},\"desc\":\"\",\"runFlag\":\"NORMAL\",\"dependence\":{},\"maxRetryTimes\":\"0\",\"retryInterval\":\"1\",\"timeout\":{\"strategy\":\"\",\"interval\":null,\"enable\":false},\"taskInstancePriority\":\"MEDIUM\",\"workerGroupId\":-1,\"preTasks\":[]}],\"tenantId\":-1,\"timeout\":0}"; + String json = "{\"globalParams\":[],\"tasks\":[{\"type\":\"SHELL\",\"id\":\"tasks-36196\",\"name\":\"ssh_test1" + + "\",\"params\":{\"resourceList\":[],\"localParams\":[],\"rawScript\":\"aa=\\\"1234\\\"\\necho ${aa}" + + "\"},\"desc\":\"\",\"runFlag\":\"NORMAL\",\"dependence\":{},\"maxRetryTimes\":\"0\",\"retryInterval" + + "\":\"1\",\"timeout\":{\"strategy\":\"\",\"interval\":null,\"enable\":false},\"taskInstancePriority\"" + + ":\"MEDIUM\",\"workerGroupId\":-1,\"preTasks\":[]}],\"tenantId\":-1,\"timeout\":0}"; String locations = "{\"tasks-36196\":{\"name\":\"ssh_test1\",\"targetarr\":\"\",\"x\":141,\"y\":70}}"; String projectName = "test"; String name = "dag_test"; @@ -247,7 +288,7 @@ public class ProcessDefinitionControllerTest{ resourceList.add(processDefinition); resourceList.add(processDefinition2); - return resourceList; + return resourceList; } @Test @@ -255,27 +296,27 @@ public class ProcessDefinitionControllerTest{ String projectName = "test"; int id = 1; - Map result = new HashMap<>(5); + Map result = new HashMap<>(); putMsg(result, Status.SUCCESS); - Mockito.when(processDefinitionService.deleteProcessDefinitionById(user, projectName,id)).thenReturn(result); - Result response = processDefinitionController.deleteProcessDefinitionById(user, projectName,id); + Mockito.when(processDefinitionService.deleteProcessDefinitionById(user, projectName, id)).thenReturn(result); + Result response = processDefinitionController.deleteProcessDefinitionById(user, projectName, id); - Assert.assertEquals(Status.SUCCESS.getCode(),response.getCode().intValue()); + Assert.assertEquals(Status.SUCCESS.getCode(), response.getCode().intValue()); } - @Test + @Test public void testGetNodeListByDefinitionId() throws Exception { String projectName = "test"; int id = 1; - Map result = new HashMap<>(5); + Map result = new HashMap<>(); putMsg(result, Status.SUCCESS); Mockito.when(processDefinitionService.getTaskNodeListByDefinitionId(id)).thenReturn(result); - Result response = processDefinitionController.getNodeListByDefinitionId(user,projectName,id); + Result response = processDefinitionController.getNodeListByDefinitionId(user, projectName, id); - Assert.assertEquals(Status.SUCCESS.getCode(),response.getCode().intValue()); + Assert.assertEquals(Status.SUCCESS.getCode(), response.getCode().intValue()); } @Test @@ -283,69 +324,130 @@ public class ProcessDefinitionControllerTest{ String projectName = "test"; String idList = "1,2,3"; - Map result = new HashMap<>(5); + Map result = new HashMap<>(); putMsg(result, Status.SUCCESS); Mockito.when(processDefinitionService.getTaskNodeListByDefinitionIdList(idList)).thenReturn(result); - Result response = processDefinitionController.getNodeListByDefinitionIdList(user,projectName,idList); + Result response = processDefinitionController.getNodeListByDefinitionIdList(user, projectName, idList); - Assert.assertEquals(Status.SUCCESS.getCode(),response.getCode().intValue()); + Assert.assertEquals(Status.SUCCESS.getCode(), response.getCode().intValue()); } @Test - public void testQueryProcessDefinitionAllByProjectId() throws Exception{ + public void testQueryProcessDefinitionAllByProjectId() throws Exception { int projectId = 1; - Map result = new HashMap<>(); - putMsg(result,Status.SUCCESS); + Map result = new HashMap<>(); + putMsg(result, Status.SUCCESS); Mockito.when(processDefinitionService.queryProcessDefinitionAllByProjectId(projectId)).thenReturn(result); - Result response = processDefinitionController.queryProcessDefinitionAllByProjectId(user,projectId); + Result response = processDefinitionController.queryProcessDefinitionAllByProjectId(user, projectId); - Assert.assertEquals(Status.SUCCESS.getCode(),response.getCode().intValue()); + Assert.assertEquals(Status.SUCCESS.getCode(), response.getCode().intValue()); } @Test - public void testViewTree() throws Exception{ + public void testViewTree() throws Exception { String projectName = "test"; int processId = 1; int limit = 2; - Map result = new HashMap<>(); - putMsg(result,Status.SUCCESS); + Map result = new HashMap<>(); + putMsg(result, Status.SUCCESS); - Mockito.when(processDefinitionService.viewTree(processId,limit)).thenReturn(result); - Result response = processDefinitionController.viewTree(user,projectName,processId,limit); + Mockito.when(processDefinitionService.viewTree(processId, limit)).thenReturn(result); + Result response = processDefinitionController.viewTree(user, projectName, processId, limit); - Assert.assertEquals(Status.SUCCESS.getCode(),response.getCode().intValue()); + Assert.assertEquals(Status.SUCCESS.getCode(), response.getCode().intValue()); } @Test - public void testQueryProcessDefinitionListPaging() throws Exception{ + public void testQueryProcessDefinitionListPaging() throws Exception { String projectName = "test"; int pageNo = 1; int pageSize = 10; String searchVal = ""; int userId = 1; - Map result = new HashMap<>(); - putMsg(result,Status.SUCCESS); - result.put(Constants.DATA_LIST,new PageInfo(1,10)); + Map result = new HashMap<>(); + putMsg(result, Status.SUCCESS); + result.put(Constants.DATA_LIST, new PageInfo(1, 10)); - Mockito.when(processDefinitionService.queryProcessDefinitionListPaging(user,projectName, searchVal, pageNo, pageSize, userId)).thenReturn(result); - Result response = processDefinitionController.queryProcessDefinitionListPaging(user,projectName,pageNo,searchVal,userId,pageSize); + Mockito.when(processDefinitionService.queryProcessDefinitionListPaging(user, projectName, searchVal, pageNo, pageSize, userId)).thenReturn(result); + Result response = processDefinitionController.queryProcessDefinitionListPaging(user, projectName, pageNo, searchVal, userId, pageSize); - Assert.assertEquals(Status.SUCCESS.getCode(),response.getCode().intValue()); + Assert.assertEquals(Status.SUCCESS.getCode(), response.getCode().intValue()); } @Test - public void testBatchExportProcessDefinitionByIds() throws Exception{ + public void testBatchExportProcessDefinitionByIds() throws Exception { String processDefinitionIds = "1,2"; String projectName = "test"; HttpServletResponse response = new MockHttpServletResponse(); - ProcessDefinitionService service = new ProcessDefinitionService(); - ProcessDefinitionService spy = Mockito.spy(service); - Mockito.doNothing().when(spy).batchExportProcessDefinitionByIds(user, projectName, processDefinitionIds, response); + Mockito.doNothing().when(this.processDefinitionService).batchExportProcessDefinitionByIds(user, projectName, processDefinitionIds, response); processDefinitionController.batchExportProcessDefinitionByIds(user, projectName, processDefinitionIds, response); } + @Test + public void testQueryProcessDefinitionVersions() { + String projectName = "test"; + Map resultMap = new HashMap<>(); + putMsg(resultMap, Status.SUCCESS); + resultMap.put(Constants.DATA_LIST, new PageInfo(1, 10)); + Mockito.when(processDefinitionVersionService.queryProcessDefinitionVersions( + user + , projectName + , 1 + , 10 + , 1)) + .thenReturn(resultMap); + Result result = processDefinitionController.queryProcessDefinitionVersions( + user + , projectName + , 1 + , 10 + , 1); + + Assert.assertEquals(Status.SUCCESS.getCode(), (int) result.getCode()); + } + + @Test + public void testSwitchProcessDefinitionVersion() { + String projectName = "test"; + Map resultMap = new HashMap<>(); + putMsg(resultMap, Status.SUCCESS); + Mockito.when(processDefinitionService.switchProcessDefinitionVersion( + user + , projectName + , 1 + , 10)) + .thenReturn(resultMap); + Result result = processDefinitionController.switchProcessDefinitionVersion( + user + , projectName + , 1 + , 10); + + Assert.assertEquals(Status.SUCCESS.getCode(), (int) result.getCode()); + } + + @Test + public void testDeleteProcessDefinitionVersion() { + String projectName = "test"; + Map resultMap = new HashMap<>(); + putMsg(resultMap, Status.SUCCESS); + Mockito.when(processDefinitionVersionService.deleteByProcessDefinitionIdAndVersion( + user + , projectName + , 1 + , 10)) + .thenReturn(resultMap); + Result result = processDefinitionController.deleteProcessDefinitionVersion( + user + , projectName + , 1 + , 10); + + Assert.assertEquals(Status.SUCCESS.getCode(), (int) result.getCode()); + } + } diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProcessInstanceControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProcessInstanceControllerTest.java index 5189097e68..bdd762afa8 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProcessInstanceControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProcessInstanceControllerTest.java @@ -16,29 +16,27 @@ */ package org.apache.dolphinscheduler.api.controller; -import org.apache.dolphinscheduler.api.enums.Status; -import org.apache.dolphinscheduler.api.utils.Result; -import org.apache.dolphinscheduler.common.enums.ExecutionStatus; -import org.apache.dolphinscheduler.common.utils.*; -import org.junit.Assert; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.http.MediaType; -import org.springframework.test.web.servlet.MvcResult; -import org.springframework.util.LinkedMultiValueMap; -import org.springframework.util.MultiValueMap; - import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; +import org.apache.dolphinscheduler.api.enums.Status; +import org.apache.dolphinscheduler.api.utils.Result; +import org.apache.dolphinscheduler.common.enums.ExecutionStatus; +import org.apache.dolphinscheduler.common.utils.JSONUtils; + +import org.junit.Assert; +import org.junit.Test; +import org.springframework.http.MediaType; +import org.springframework.test.web.servlet.MvcResult; +import org.springframework.util.LinkedMultiValueMap; +import org.springframework.util.MultiValueMap; + /** * process instance controller test */ public class ProcessInstanceControllerTest extends AbstractControllerTest { - private static Logger logger = LoggerFactory.getLogger(ProcessInstanceControllerTest.class); @Test public void testQueryProcessInstanceList() throws Exception { @@ -52,31 +50,30 @@ public class ProcessInstanceControllerTest extends AbstractControllerTest { paramsMap.add("pageNo", "2"); paramsMap.add("pageSize", "2"); - MvcResult mvcResult = mockMvc.perform(get("/projects/{projectName}/instance/list-paging","cxc_1113") + MvcResult mvcResult = mockMvc.perform(get("/projects/{projectName}/instance/list-paging", "cxc_1113") .header("sessionId", sessionId) .params(paramsMap)) .andExpect(status().isOk()) .andExpect(content().contentType(MediaType.APPLICATION_JSON_UTF8)) .andReturn(); Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); + Assert.assertNotNull(result); Assert.assertEquals(Status.SUCCESS.getCode(), result.getCode().intValue()); - logger.info(mvcResult.getResponse().getContentAsString()); } @Test public void testQueryTaskListByProcessId() throws Exception { - MvcResult mvcResult = mockMvc.perform(get("/projects/{projectName}/instance/task-list-by-process-id","cxc_1113") + MvcResult mvcResult = mockMvc.perform(get("/projects/{projectName}/instance/task-list-by-process-id", "cxc_1113") .header(SESSION_ID, sessionId) - .param("processInstanceId","1203")) + .param("processInstanceId", "1203")) .andExpect(status().isOk()) .andExpect(content().contentType(MediaType.APPLICATION_JSON_UTF8)) .andReturn(); Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); - assert result != null; - Assert.assertEquals(Status.PROJECT_NOT_FOUNT.getCode(),result.getCode().intValue()); - logger.info(mvcResult.getResponse().getContentAsString()); + Assert.assertNotNull(result); + Assert.assertEquals(Status.PROJECT_NOT_FOUNT.getCode(), result.getCode().intValue()); } @Test @@ -91,110 +88,108 @@ public class ProcessInstanceControllerTest extends AbstractControllerTest { paramsMap.add("syncDefine", "false"); paramsMap.add("locations", locations); paramsMap.add("connects", "[]"); -// paramsMap.add("flag", "2"); - MvcResult mvcResult = mockMvc.perform(post("/projects/{projectName}/instance/update","cxc_1113") + MvcResult mvcResult = mockMvc.perform(post("/projects/{projectName}/instance/update", "cxc_1113") .header("sessionId", sessionId) .params(paramsMap)) .andExpect(status().isOk()) .andExpect(content().contentType(MediaType.APPLICATION_JSON_UTF8)) .andReturn(); Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); + Assert.assertNotNull(result); Assert.assertEquals(Status.SUCCESS.getCode(), result.getCode().intValue()); - logger.info(mvcResult.getResponse().getContentAsString()); } @Test public void testQueryProcessInstanceById() throws Exception { - MvcResult mvcResult = mockMvc.perform(get("/projects/{projectName}/instance/select-by-id","cxc_1113") + MvcResult mvcResult = mockMvc.perform(get("/projects/{projectName}/instance/select-by-id", "cxc_1113") .header(SESSION_ID, sessionId) - .param("processInstanceId","1203")) + .param("processInstanceId", "1203")) .andExpect(status().isOk()) .andExpect(content().contentType(MediaType.APPLICATION_JSON_UTF8)) .andReturn(); Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); - Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); - logger.info(mvcResult.getResponse().getContentAsString()); + Assert.assertNotNull(result); + Assert.assertEquals(Status.SUCCESS.getCode(), result.getCode().intValue()); } - @Test public void testQuerySubProcessInstanceByTaskId() throws Exception { - MvcResult mvcResult = mockMvc.perform(get("/projects/{projectName}/instance/select-sub-process","cxc_1113") + MvcResult mvcResult = mockMvc.perform(get("/projects/{projectName}/instance/select-sub-process", "cxc_1113") .header(SESSION_ID, sessionId) - .param("taskId","1203")) + .param("taskId", "1203")) .andExpect(status().isOk()) .andExpect(content().contentType(MediaType.APPLICATION_JSON_UTF8)) .andReturn(); Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); - Assert.assertEquals(Status.TASK_INSTANCE_NOT_EXISTS.getCode(),result.getCode().intValue()); - logger.info(mvcResult.getResponse().getContentAsString()); + Assert.assertNotNull(result); + Assert.assertEquals(Status.TASK_INSTANCE_NOT_EXISTS.getCode(), result.getCode().intValue()); } @Test public void testQueryParentInstanceBySubId() throws Exception { - MvcResult mvcResult = mockMvc.perform(get("/projects/{projectName}/instance/select-parent-process","cxc_1113") + MvcResult mvcResult = mockMvc.perform(get("/projects/{projectName}/instance/select-parent-process", "cxc_1113") .header(SESSION_ID, sessionId) - .param("subId","1204")) + .param("subId", "1204")) .andExpect(status().isOk()) .andExpect(content().contentType(MediaType.APPLICATION_JSON_UTF8)) .andReturn(); Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); - Assert.assertEquals(Status.PROCESS_INSTANCE_NOT_SUB_PROCESS_INSTANCE.getCode(),result.getCode().intValue()); - logger.info(mvcResult.getResponse().getContentAsString()); + Assert.assertNotNull(result); + Assert.assertEquals(Status.PROCESS_INSTANCE_NOT_SUB_PROCESS_INSTANCE.getCode(), result.getCode().intValue()); } @Test public void testViewVariables() throws Exception { - MvcResult mvcResult = mockMvc.perform(get("/projects/{projectName}/instance/view-variables","cxc_1113") + MvcResult mvcResult = mockMvc.perform(get("/projects/{projectName}/instance/view-variables", "cxc_1113") .header(SESSION_ID, sessionId) - .param("processInstanceId","1204")) + .param("processInstanceId", "1204")) .andExpect(status().isOk()) .andExpect(content().contentType(MediaType.APPLICATION_JSON_UTF8)) .andReturn(); Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); - Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); - logger.info(mvcResult.getResponse().getContentAsString()); + Assert.assertNotNull(result); + Assert.assertEquals(Status.SUCCESS.getCode(), result.getCode().intValue()); } @Test public void testDeleteProcessInstanceById() throws Exception { - MvcResult mvcResult = mockMvc.perform(get("/projects/{projectName}/instance/delete","cxc_1113") + MvcResult mvcResult = mockMvc.perform(get("/projects/{projectName}/instance/delete", "cxc_1113") .header(SESSION_ID, sessionId) - .param("processInstanceId","1204")) + .param("processInstanceId", "1204")) .andExpect(status().isOk()) .andExpect(content().contentType(MediaType.APPLICATION_JSON_UTF8)) .andReturn(); Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); - Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); - logger.info(mvcResult.getResponse().getContentAsString()); + Assert.assertNotNull(result); + Assert.assertEquals(Status.SUCCESS.getCode(), result.getCode().intValue()); } @Test public void testBatchDeleteProcessInstanceByIds() throws Exception { - MvcResult mvcResult = mockMvc.perform(get("/projects/{projectName}/instance/batch-delete","cxc_1113") + MvcResult mvcResult = mockMvc.perform(get("/projects/{projectName}/instance/batch-delete", "cxc_1113") .header(SESSION_ID, sessionId) - .param("processInstanceIds","1205，1206")) + .param("processInstanceIds", "1205，1206")) .andExpect(status().isOk()) .andExpect(content().contentType(MediaType.APPLICATION_JSON_UTF8)) .andReturn(); Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); - Assert.assertEquals(Status.DELETE_PROCESS_INSTANCE_BY_ID_ERROR.getCode(),result.getCode().intValue()); - logger.info(mvcResult.getResponse().getContentAsString()); + Assert.assertNotNull(result); + Assert.assertEquals(Status.DELETE_PROCESS_INSTANCE_BY_ID_ERROR.getCode(), result.getCode().intValue()); } } diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ResourcesControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ResourcesControllerTest.java index 2843f49f1c..1ca7421e9d 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ResourcesControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ResourcesControllerTest.java @@ -16,7 +16,6 @@ */ package org.apache.dolphinscheduler.api.controller; -import com.fasterxml.jackson.databind.node.ObjectNode; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.enums.ResourceType; @@ -53,8 +52,6 @@ public class ResourcesControllerTest extends AbstractControllerTest{ .andReturn(); Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); - result.getCode().equals(Status.SUCCESS.getCode()); - ObjectNode object = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString()); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); logger.info(mvcResult.getResponse().getContentAsString()); @@ -77,8 +74,6 @@ public class ResourcesControllerTest extends AbstractControllerTest{ .andReturn(); Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); - result.getCode().equals(Status.SUCCESS.getCode()); - ObjectNode object = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString()); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); logger.info(mvcResult.getResponse().getContentAsString()); @@ -280,8 +275,6 @@ public class ResourcesControllerTest extends AbstractControllerTest{ .andReturn(); Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); - result.getCode().equals(Status.SUCCESS.getCode()); - ObjectNode object = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString()); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); logger.info(mvcResult.getResponse().getContentAsString()); @@ -302,8 +295,6 @@ public class ResourcesControllerTest extends AbstractControllerTest{ .andReturn(); Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); - result.getCode().equals(Status.SUCCESS.getCode()); - ObjectNode object = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString()); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); logger.info(mvcResult.getResponse().getContentAsString()); @@ -323,8 +314,6 @@ public class ResourcesControllerTest extends AbstractControllerTest{ .andReturn(); Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); - result.getCode().equals(Status.SUCCESS.getCode()); - ObjectNode object = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString()); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); logger.info(mvcResult.getResponse().getContentAsString()); @@ -343,8 +332,6 @@ public class ResourcesControllerTest extends AbstractControllerTest{ .andReturn(); Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); - result.getCode().equals(Status.SUCCESS.getCode()); - ObjectNode object = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString()); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); logger.info(mvcResult.getResponse().getContentAsString()); @@ -364,8 +351,6 @@ public class ResourcesControllerTest extends AbstractControllerTest{ .andReturn(); Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); - result.getCode().equals(Status.SUCCESS.getCode()); - ObjectNode object = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString()); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); logger.info(mvcResult.getResponse().getContentAsString()); @@ -385,8 +370,6 @@ public class ResourcesControllerTest extends AbstractControllerTest{ .andReturn(); Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); - result.getCode().equals(Status.SUCCESS.getCode()); - ObjectNode object = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString()); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); logger.info(mvcResult.getResponse().getContentAsString()); @@ -405,8 +388,6 @@ public class ResourcesControllerTest extends AbstractControllerTest{ .andReturn(); Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); - result.getCode().equals(Status.SUCCESS.getCode()); - ObjectNode object = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString()); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); logger.info(mvcResult.getResponse().getContentAsString()); @@ -426,8 +407,6 @@ public class ResourcesControllerTest extends AbstractControllerTest{ .andReturn(); Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); - result.getCode().equals(Status.SUCCESS.getCode()); - ObjectNode object = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString()); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); logger.info(mvcResult.getResponse().getContentAsString()); @@ -445,8 +424,6 @@ public class ResourcesControllerTest extends AbstractControllerTest{ .andReturn(); Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); - result.getCode().equals(Status.SUCCESS.getCode()); - ObjectNode object = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString()); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); logger.info(mvcResult.getResponse().getContentAsString()); diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/UsersControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/UsersControllerTest.java index fc86632ed7..6537288067 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/UsersControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/UsersControllerTest.java @@ -18,7 +18,7 @@ package org.apache.dolphinscheduler.api.controller; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.utils.Result; -import org.apache.dolphinscheduler.common.utils.*; +import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.junit.Assert; import org.junit.Test; import org.slf4j.Logger; @@ -33,6 +33,9 @@ import static org.springframework.test.web.servlet.request.MockMvcRequestBuilder import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; +import java.util.ArrayList; +import java.util.List; + /** * users controller test */ @@ -285,6 +288,39 @@ public class UsersControllerTest extends AbstractControllerTest{ Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); - logger.info(mvcResult.getResponse().getContentAsString()); + } + + @Test + public void testActivateUser() throws Exception { + MultiValueMap paramsMap = new LinkedMultiValueMap<>(); + paramsMap.add("userName","user_test"); + + MvcResult mvcResult = mockMvc.perform(post("/users/activate") + .header(SESSION_ID, sessionId) + .params(paramsMap)) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.APPLICATION_JSON_UTF8)) + .andReturn(); + + Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); + Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); + } + + @Test + public void testBatchActivateUser() throws Exception { + List userNames = new ArrayList<>(); + userNames.add("user_sky_cxl"); + userNames.add("19990323"); + userNames.add("test_sky_post_11"); + String jsonUserNames = JSONUtils.toJsonString(userNames); + MvcResult mvcResult = mockMvc.perform(post("/users/batch/activate") + .header(SESSION_ID, sessionId) + .contentType(MediaType.APPLICATION_JSON) + .content(jsonUserNames)) + .andExpect(status().isOk()) + .andReturn(); + + Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); + Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); } } diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/dto/resources/filter/ResourceFilterTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/dto/resources/filter/ResourceFilterTest.java index 8a4a16c4f0..2ddce9e0d0 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/dto/resources/filter/ResourceFilterTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/dto/resources/filter/ResourceFilterTest.java @@ -49,10 +49,9 @@ public class ResourceFilterTest { allList.add(resource6); allList.add(resource7); - ResourceFilter resourceFilter = new ResourceFilter(".jar",allList); List resourceList = resourceFilter.filter(); Assert.assertNotNull(resourceList); - resourceList.stream().forEach(t-> logger.info(t.toString())); + resourceList.forEach(t -> logger.info(t.toString())); } } \ No newline at end of file diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/AccessTokenServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/AccessTokenServiceTest.java index f388445f0c..f5543487ea 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/AccessTokenServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/AccessTokenServiceTest.java @@ -16,10 +16,12 @@ */ package org.apache.dolphinscheduler.api.service; -import com.baomidou.mybatisplus.core.metadata.IPage; -import com.baomidou.mybatisplus.extension.plugins.pagination.Page; -import java.util.Calendar; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.when; + import org.apache.dolphinscheduler.api.enums.Status; +import org.apache.dolphinscheduler.api.service.impl.AccessTokenServiceImpl; import org.apache.dolphinscheduler.api.utils.PageInfo; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.UserType; @@ -27,9 +29,14 @@ import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.dao.entity.AccessToken; import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.mapper.AccessTokenMapper; -import org.junit.After; + +import java.util.ArrayList; +import java.util.Calendar; +import java.util.Date; +import java.util.List; +import java.util.Map; + import org.junit.Assert; -import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.InjectMocks; @@ -38,131 +45,109 @@ import org.mockito.junit.MockitoJUnitRunner; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.ArrayList; -import java.util.Date; -import java.util.List; -import java.util.Map; - -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.when; +import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; @RunWith(MockitoJUnitRunner.class) public class AccessTokenServiceTest { - private static final Logger logger = LoggerFactory.getLogger(AccessTokenServiceTest.class); - @InjectMocks - private AccessTokenService accessTokenService ; + private AccessTokenServiceImpl accessTokenService; @Mock private AccessTokenMapper accessTokenMapper; - @Before - public void setUp() { - - } - - - @After - public void after(){ - - } - - @Test - public void testQueryAccessTokenList(){ + @SuppressWarnings("unchecked") + public void testQueryAccessTokenList() { IPage tokenPage = new Page<>(); tokenPage.setRecords(getList()); tokenPage.setTotal(1L); - when(accessTokenMapper.selectAccessTokenPage(any(Page.class),eq("zhangsan"),eq(0))).thenReturn(tokenPage); + when(accessTokenMapper.selectAccessTokenPage(any(Page.class), eq("zhangsan"), eq(0))).thenReturn(tokenPage); - User user =new User(); - Map result = accessTokenService.queryAccessTokenList(user,"zhangsan",1,10); + User user = new User(); + Map result = accessTokenService.queryAccessTokenList(user, "zhangsan", 1, 10); logger.info(result.toString()); - Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); PageInfo pageInfo = (PageInfo) result.get(Constants.DATA_LIST); - Assert.assertTrue(pageInfo.getTotalCount()>0); + Assert.assertTrue(pageInfo.getTotalCount() > 0); } @Test - public void testCreateToken(){ + public void testCreateToken() { - - when(accessTokenMapper.insert(any(AccessToken.class))).thenReturn(2); - Map result = accessTokenService.createToken(1,getDate(),"AccessTokenServiceTest"); + when(accessTokenMapper.insert(any(AccessToken.class))).thenReturn(2); + Map result = accessTokenService.createToken(1, getDate(), "AccessTokenServiceTest"); logger.info(result.toString()); - Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); } @Test - public void testGenerateToken(){ + public void testGenerateToken() { - Map result = accessTokenService.generateToken(Integer.MAX_VALUE,getDate()); + Map result = accessTokenService.generateToken(Integer.MAX_VALUE, getDate()); logger.info(result.toString()); - Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); String token = (String) result.get(Constants.DATA_LIST); Assert.assertNotNull(token); } @Test - public void testDelAccessTokenById(){ + public void testDelAccessTokenById() { when(accessTokenMapper.selectById(1)).thenReturn(getEntity()); User userLogin = new User(); // not exist - Map result = accessTokenService.delAccessTokenById(userLogin,0); + Map result = accessTokenService.delAccessTokenById(userLogin, 0); logger.info(result.toString()); - Assert.assertEquals(Status.ACCESS_TOKEN_NOT_EXIST,result.get(Constants.STATUS)); + Assert.assertEquals(Status.ACCESS_TOKEN_NOT_EXIST, result.get(Constants.STATUS)); // no operate - result = accessTokenService.delAccessTokenById(userLogin,1); + result = accessTokenService.delAccessTokenById(userLogin, 1); logger.info(result.toString()); - Assert.assertEquals(Status.USER_NO_OPERATION_PERM,result.get(Constants.STATUS)); + Assert.assertEquals(Status.USER_NO_OPERATION_PERM, result.get(Constants.STATUS)); //success userLogin.setId(1); userLogin.setUserType(UserType.ADMIN_USER); - result = accessTokenService.delAccessTokenById(userLogin,1); + result = accessTokenService.delAccessTokenById(userLogin, 1); logger.info(result.toString()); - Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); } @Test - public void testUpdateToken(){ + public void testUpdateToken() { when(accessTokenMapper.selectById(1)).thenReturn(getEntity()); - Map result = accessTokenService.updateToken(1,Integer.MAX_VALUE,getDate(),"token"); + Map result = accessTokenService.updateToken(1, Integer.MAX_VALUE, getDate(), "token"); logger.info(result.toString()); - Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); // not exist - result = accessTokenService.updateToken(2,Integer.MAX_VALUE,getDate(),"token"); + result = accessTokenService.updateToken(2, Integer.MAX_VALUE, getDate(), "token"); logger.info(result.toString()); - Assert.assertEquals(Status.ACCESS_TOKEN_NOT_EXIST,result.get(Constants.STATUS)); + Assert.assertEquals(Status.ACCESS_TOKEN_NOT_EXIST, result.get(Constants.STATUS)); } /** * create entity - * @return */ - private AccessToken getEntity(){ + private AccessToken getEntity() { AccessToken accessToken = new AccessToken(); accessToken.setId(1); accessToken.setUserId(1); accessToken.setToken("AccessTokenServiceTest"); - Date date = DateUtils.add(new Date(),Calendar.DAY_OF_MONTH, 30); + Date date = DateUtils.add(new Date(), Calendar.DAY_OF_MONTH, 30); accessToken.setExpireTime(date); return accessToken; } /** * entity list - * @return */ - private List getList(){ + private List getList() { List list = new ArrayList<>(); list.add(getEntity()); @@ -170,13 +155,11 @@ public class AccessTokenServiceTest { } - /** * get dateStr - * @return */ - private String getDate(){ + private String getDate() { Date date = DateUtils.add(new Date(), Calendar.DAY_OF_MONTH, 30); - return DateUtils.dateToString(date); + return DateUtils.dateToString(date); } } diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/AlertGroupServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/AlertGroupServiceTest.java index ab7dac4d60..1b93e86773 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/AlertGroupServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/AlertGroupServiceTest.java @@ -76,7 +76,7 @@ public class AlertGroupServiceTest { @Test - public void testQueryAlertgroup(){ + public void testQueryAlertGroup(){ Mockito.when(alertGroupMapper.queryAllGroupList()).thenReturn(getList()); HashMap result= alertGroupService.queryAlertgroup(); diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/BaseDAGServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/BaseDAGServiceTest.java deleted file mode 100644 index bb6e3882fe..0000000000 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/BaseDAGServiceTest.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.dolphinscheduler.api.service; - -import org.apache.dolphinscheduler.common.graph.DAG; -import org.apache.dolphinscheduler.common.model.TaskNode; -import org.apache.dolphinscheduler.common.model.TaskNodeRelation; -import org.apache.dolphinscheduler.dao.entity.ProcessInstance; -import org.junit.Assert; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.junit.MockitoJUnitRunner; - -@RunWith(MockitoJUnitRunner.class) -public class BaseDAGServiceTest { - - @Test - public void testProcessInstance2DAG(){ - - ProcessInstance processInstance = new ProcessInstance(); - processInstance.setProcessInstanceJson("{\"globalParams\":[],\"tasks\":[{\"type\":\"SHELL\",\"id\":\"tasks-61567\"," + - "\"name\":\"开始\",\"params\":{\"resourceList\":[],\"localParams\":[],\"rawScript\":\"echo '1'\"}," + - "\"description\":\"\",\"runFlag\":\"NORMAL\",\"dependence\":{},\"maxRetryTimes\":\"0\",\"retryInterval\":\"1\"," + - "\"timeout\":{\"strategy\":\"\",\"interval\":null,\"enable\":false},\"taskInstancePriority\":\"MEDIUM\"," + - "\"workerGroupId\":-1,\"preTasks\":[]},{\"type\":\"SHELL\",\"id\":\"tasks-6-3ug5ej\",\"name\":\"结束\"," + - "\"params\":{\"resourceList\":[],\"localParams\":[],\"rawScript\":\"echo '1'\"},\"description\":\"\"," + - "\"runFlag\":\"NORMAL\",\"dependence\":{},\"maxRetryTimes\":\"0\",\"retryInterval\":\"1\"," + - "\"timeout\":{\"strategy\":\"\",\"interval\":null,\"enable\":false},\"taskInstancePriority\":\"MEDIUM\"," + - "\"workerGroupId\":-1,\"preTasks\":[\"开始\"]}],\"tenantId\":-1,\"timeout\":0}"); - - DAG relationDAG = BaseDAGService.processInstance2DAG(processInstance); - - Assert.assertTrue(relationDAG.containsNode("开始")); - - } -} diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataAnalysisServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataAnalysisServiceTest.java index 6a9e78600b..3d8ae91287 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataAnalysisServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataAnalysisServiceTest.java @@ -17,17 +17,29 @@ package org.apache.dolphinscheduler.api.service; import org.apache.dolphinscheduler.api.enums.Status; +import org.apache.dolphinscheduler.api.service.impl.DataAnalysisServiceImpl; +import org.apache.dolphinscheduler.api.service.impl.ProjectServiceImpl; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; -import org.apache.dolphinscheduler.common.enums.UserType; import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.dao.entity.CommandCount; import org.apache.dolphinscheduler.dao.entity.ExecuteStatusCount; import org.apache.dolphinscheduler.dao.entity.Project; import org.apache.dolphinscheduler.dao.entity.User; -import org.apache.dolphinscheduler.dao.mapper.*; +import org.apache.dolphinscheduler.dao.mapper.CommandMapper; +import org.apache.dolphinscheduler.dao.mapper.ErrorCommandMapper; +import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper; +import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper; +import org.apache.dolphinscheduler.dao.mapper.ProjectMapper; +import org.apache.dolphinscheduler.dao.mapper.TaskInstanceMapper; import org.apache.dolphinscheduler.service.process.ProcessService; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -36,25 +48,19 @@ import org.junit.runner.RunWith; import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.Mockito; -import org.powermock.api.mockito.PowerMockito; -import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; @RunWith(PowerMockRunner.class) public class DataAnalysisServiceTest { - + @InjectMocks - private DataAnalysisService dataAnalysisService; + private DataAnalysisServiceImpl dataAnalysisService; @Mock ProjectMapper projectMapper; @Mock - ProjectService projectService; + ProjectServiceImpl projectService; @Mock ProcessInstanceMapper processInstanceMapper; @@ -71,13 +77,9 @@ public class DataAnalysisServiceTest { @Mock TaskInstanceMapper taskInstanceMapper; - - @Mock ProcessService processService; - private Project project; - private Map resultMap; private User user; @@ -86,26 +88,25 @@ public class DataAnalysisServiceTest { public void setUp() { user = new User(); - project = new Project(); + Project project = new Project(); project.setId(1); resultMap = new HashMap<>(); Mockito.when(projectMapper.selectById(1)).thenReturn(project); - Mockito.when(projectService.hasProjectAndPerm(user,project,resultMap)).thenReturn(true); + Mockito.when(projectService.hasProjectAndPerm(user, project, resultMap)).thenReturn(true); } @After - public void after(){ + public void after() { user = null; projectMapper = null; resultMap = null; } - @Test - public void testCountTaskStateByProject(){ + public void testCountTaskStateByProject() { String startDate = "2020-02-11 16:02:18"; String endDate = "2020-02-11 16:03:18"; @@ -120,42 +121,40 @@ public class DataAnalysisServiceTest { DateUtils.getScheduleDate(endDate), new Integer[]{1})).thenReturn(getTaskInstanceStateCounts()); result = dataAnalysisService.countTaskStateByProject(user, 1, startDate, endDate); - Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); } - @Test - public void testCountProcessInstanceStateByProject(){ + public void testCountProcessInstanceStateByProject() { String startDate = "2020-02-11 16:02:18"; String endDate = "2020-02-11 16:03:18"; //checkProject false - Map result = dataAnalysisService.countProcessInstanceStateByProject(user,2,startDate,endDate); + Map result = dataAnalysisService.countProcessInstanceStateByProject(user, 2, startDate, endDate); Assert.assertTrue(result.isEmpty()); //SUCCESS Mockito.when(processInstanceMapper.countInstanceStateByUser(DateUtils.getScheduleDate(startDate), DateUtils.getScheduleDate(endDate), new Integer[]{1})).thenReturn(getTaskInstanceStateCounts()); - result = dataAnalysisService.countProcessInstanceStateByProject(user,1,startDate,endDate); - Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + result = dataAnalysisService.countProcessInstanceStateByProject(user, 1, startDate, endDate); + Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); } @Test - public void testCountDefinitionByUser(){ + public void testCountDefinitionByUser() { - Map result = dataAnalysisService.countDefinitionByUser(user,1); - Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + Map result = dataAnalysisService.countDefinitionByUser(user, 1); + Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); } - @Test - public void testCountCommandState(){ + public void testCountCommandState() { String startDate = "2020-02-11 16:02:18"; String endDate = "2020-02-11 16:03:18"; //checkProject false - Map result = dataAnalysisService.countCommandState(user,2,startDate,endDate); + Map result = dataAnalysisService.countCommandState(user, 2, startDate, endDate); Assert.assertTrue(result.isEmpty()); List commandCounts = new ArrayList<>(1); CommandCount commandCount = new CommandCount(); @@ -164,26 +163,25 @@ public class DataAnalysisServiceTest { Mockito.when(commandMapper.countCommandState(0, DateUtils.getScheduleDate(startDate), DateUtils.getScheduleDate(endDate), new Integer[]{1})).thenReturn(commandCounts); - Mockito.when(errorCommandMapper.countCommandState( DateUtils.getScheduleDate(startDate), + Mockito.when(errorCommandMapper.countCommandState(DateUtils.getScheduleDate(startDate), DateUtils.getScheduleDate(endDate), new Integer[]{1})).thenReturn(commandCounts); - result = dataAnalysisService.countCommandState(user,1,startDate,endDate); - Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + result = dataAnalysisService.countCommandState(user, 1, startDate, endDate); + Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); } /** - * get list - * @return + * get list */ - private List getTaskInstanceStateCounts(){ + private List getTaskInstanceStateCounts() { List taskInstanceStateCounts = new ArrayList<>(1); ExecuteStatusCount executeStatusCount = new ExecuteStatusCount(); executeStatusCount.setExecutionStatus(ExecutionStatus.RUNNING_EXECUTION); taskInstanceStateCounts.add(executeStatusCount); - return taskInstanceStateCounts; + return taskInstanceStateCounts; } } \ No newline at end of file diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataSourceServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataSourceServiceTest.java index c185868cde..789e5f6cc5 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataSourceServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataSourceServiceTest.java @@ -22,10 +22,14 @@ import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.DbConnectType; import org.apache.dolphinscheduler.common.enums.DbType; import org.apache.dolphinscheduler.common.enums.UserType; +import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.PropertyUtils; +import org.apache.dolphinscheduler.dao.datasource.DataSourceFactory; +import org.apache.dolphinscheduler.dao.datasource.MySQLDataSource; import org.apache.dolphinscheduler.dao.entity.DataSource; import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.mapper.DataSourceMapper; +import org.apache.dolphinscheduler.dao.mapper.DataSourceUserMapper; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; @@ -35,8 +39,6 @@ import org.mockito.Mockito; import org.powermock.api.mockito.PowerMockito; import org.powermock.core.classloader.annotations.PowerMockIgnore; import org.powermock.modules.junit4.PowerMockRunner; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.util.ArrayList; import java.util.List; @@ -45,16 +47,172 @@ import java.util.Map; @RunWith(PowerMockRunner.class) @PowerMockIgnore({"sun.security.*", "javax.net.*"}) public class DataSourceServiceTest { - private static final Logger logger = LoggerFactory.getLogger(DataSourceServiceTest.class); @InjectMocks private DataSourceService dataSourceService; @Mock private DataSourceMapper dataSourceMapper; + @Mock + private DataSourceUserMapper datasourceUserMapper; + + public void createDataSourceTest() { + User loginUser = getAdminUser(); + + String dataSourceName = "dataSource01"; + String dataSourceDesc = "test dataSource"; + DbType dataSourceType = DbType.POSTGRESQL; + String parameter = dataSourceService.buildParameter(dataSourceType, "172.16.133.200", "5432", "dolphinscheduler", null, "postgres", "", null, null); + + // data source exits + List dataSourceList = new ArrayList<>(); + DataSource dataSource = new DataSource(); + dataSource.setName(dataSourceName); + dataSourceList.add(dataSource); + PowerMockito.when(dataSourceMapper.queryDataSourceByName(dataSourceName.trim())).thenReturn(dataSourceList); + Map dataSourceExitsResult = dataSourceService.createDataSource(loginUser, dataSourceName, dataSourceDesc, dataSourceType, parameter); + Assert.assertEquals(Status.DATASOURCE_EXIST, dataSourceExitsResult.get(Constants.STATUS)); + + // data source exits + PowerMockito.when(dataSourceMapper.queryDataSourceByName(dataSourceName.trim())).thenReturn(null); + PowerMockito.when(dataSourceService.checkConnection(dataSourceType, parameter)).thenReturn(false); + Map connectFailedResult = dataSourceService.createDataSource(loginUser, dataSourceName, dataSourceDesc, dataSourceType, parameter); + Assert.assertEquals(Status.DATASOURCE_CONNECT_FAILED, connectFailedResult.get(Constants.STATUS)); + + // data source exits + PowerMockito.when(dataSourceMapper.queryDataSourceByName(dataSourceName.trim())).thenReturn(null); + PowerMockito.when(dataSourceService.checkConnection(dataSourceType, parameter)).thenReturn(true); + PowerMockito.when(DataSourceFactory.getDatasource(dataSourceType, parameter)).thenReturn(null); + Map notValidError = dataSourceService.createDataSource(loginUser, dataSourceName, dataSourceDesc, dataSourceType, parameter); + Assert.assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR, notValidError.get(Constants.STATUS)); + + // success + PowerMockito.when(dataSourceMapper.queryDataSourceByName(dataSourceName.trim())).thenReturn(null); + PowerMockito.when(dataSourceService.checkConnection(dataSourceType, parameter)).thenReturn(true); + PowerMockito.when(DataSourceFactory.getDatasource(dataSourceType, parameter)).thenReturn(JSONUtils.parseObject(parameter, MySQLDataSource.class)); + Map success = dataSourceService.createDataSource(loginUser, dataSourceName, dataSourceDesc, dataSourceType, parameter); + Assert.assertEquals(Status.SUCCESS, success.get(Constants.STATUS)); + } + + public void updateDataSourceTest() { + User loginUser = getAdminUser(); + + int dataSourceId = 12; + String dataSourceName = "dataSource01"; + String dataSourceDesc = "test dataSource"; + DbType dataSourceType = DbType.POSTGRESQL; + String parameter = dataSourceService.buildParameter(dataSourceType, "172.16.133.200", "5432", "dolphinscheduler", null, "postgres", "", null, null); + + // data source not exits + PowerMockito.when(dataSourceMapper.selectById(dataSourceId)).thenReturn(null); + Map resourceNotExits = dataSourceService.updateDataSource(dataSourceId, loginUser, dataSourceName, dataSourceDesc, dataSourceType, parameter); + Assert.assertEquals(Status.RESOURCE_NOT_EXIST, resourceNotExits.get(Constants.STATUS)); + // user no operation perm + DataSource dataSource = new DataSource(); + dataSource.setUserId(0); + PowerMockito.when(dataSourceMapper.selectById(dataSourceId)).thenReturn(dataSource); + Map userNoOperationPerm = dataSourceService.updateDataSource(dataSourceId, loginUser, dataSourceName, dataSourceDesc, dataSourceType, parameter); + Assert.assertEquals(Status.USER_NO_OPERATION_PERM, userNoOperationPerm.get(Constants.STATUS)); + + // data source name exits + dataSource.setUserId(-1); + List dataSourceList = new ArrayList<>(); + dataSourceList.add(dataSource); + PowerMockito.when(dataSourceMapper.selectById(dataSourceId)).thenReturn(dataSource); + PowerMockito.when(dataSourceMapper.queryDataSourceByName(dataSourceName)).thenReturn(dataSourceList); + Map dataSourceNameExist = dataSourceService.updateDataSource(dataSourceId, loginUser, dataSourceName, dataSourceDesc, dataSourceType, parameter); + Assert.assertEquals(Status.DATASOURCE_EXIST, dataSourceNameExist.get(Constants.STATUS)); + + // data source connect failed + PowerMockito.when(dataSourceMapper.selectById(dataSourceId)).thenReturn(dataSource); + PowerMockito.when(dataSourceMapper.queryDataSourceByName(dataSourceName)).thenReturn(null); + PowerMockito.when(dataSourceService.checkConnection(dataSourceType, parameter)).thenReturn(true); + Map connectFailed = dataSourceService.updateDataSource(dataSourceId, loginUser, dataSourceName, dataSourceDesc, dataSourceType, parameter); + Assert.assertEquals(Status.DATASOURCE_CONNECT_FAILED, connectFailed.get(Constants.STATUS)); + + //success + PowerMockito.when(dataSourceMapper.selectById(dataSourceId)).thenReturn(dataSource); + PowerMockito.when(dataSourceMapper.queryDataSourceByName(dataSourceName)).thenReturn(null); + PowerMockito.when(dataSourceService.checkConnection(dataSourceType, parameter)).thenReturn(false); + Map success = dataSourceService.updateDataSource(dataSourceId, loginUser, dataSourceName, dataSourceDesc, dataSourceType, parameter); + Assert.assertEquals(Status.SUCCESS, connectFailed.get(Constants.STATUS)); + + } @Test - public void queryDataSourceListTest(){ + public void queryDataSourceListPagingTest() { + User loginUser = getAdminUser(); + String searchVal = ""; + int pageNo = 1; + int pageSize = 10; + Map success = dataSourceService.queryDataSourceListPaging(loginUser, searchVal, pageNo, pageSize); + Assert.assertEquals(Status.SUCCESS, success.get(Constants.STATUS)); + } + @Test + public void connectionTest() { + int dataSourceId = -1; + PowerMockito.when(dataSourceMapper.selectById(dataSourceId)).thenReturn(null); + Assert.assertFalse(dataSourceService.connectionTest(dataSourceId)); + } + + @Test + public void deleteTest() { + User loginUser = getAdminUser(); + int dataSourceId = 1; + Result result = new Result(); + + //resource not exist + dataSourceService.putMsg(result, Status.RESOURCE_NOT_EXIST); + PowerMockito.when(dataSourceMapper.selectById(dataSourceId)).thenReturn(null); + Assert.assertEquals(result.getCode(), dataSourceService.delete(loginUser, dataSourceId).getCode()); + + // user no operation perm + dataSourceService.putMsg(result, Status.USER_NO_OPERATION_PERM); + DataSource dataSource = new DataSource(); + dataSource.setUserId(0); + PowerMockito.when(dataSourceMapper.selectById(dataSourceId)).thenReturn(dataSource); + Assert.assertEquals(result.getCode(), dataSourceService.delete(loginUser, dataSourceId).getCode()); + + // success + dataSourceService.putMsg(result, Status.SUCCESS); + dataSource.setUserId(-1); + PowerMockito.when(dataSourceMapper.selectById(dataSourceId)).thenReturn(dataSource); + Assert.assertEquals(result.getCode(), dataSourceService.delete(loginUser, dataSourceId).getCode()); + + } + + @Test + public void unauthDatasourceTest() { + User loginUser = getAdminUser(); + int userId = -1; + + //user no operation perm + Map noOperationPerm = dataSourceService.unauthDatasource(loginUser, userId); + Assert.assertEquals(Status.USER_NO_OPERATION_PERM, noOperationPerm.get(Constants.STATUS)); + + //success + loginUser.setUserType(UserType.ADMIN_USER); + Map success = dataSourceService.unauthDatasource(loginUser, userId); + Assert.assertEquals(Status.SUCCESS, success.get(Constants.STATUS)); + } + + @Test + public void authedDatasourceTest() { + User loginUser = getAdminUser(); + int userId = -1; + + //user no operation perm + Map noOperationPerm = dataSourceService.authedDatasource(loginUser, userId); + Assert.assertEquals(Status.USER_NO_OPERATION_PERM, noOperationPerm.get(Constants.STATUS)); + + //success + loginUser.setUserType(UserType.ADMIN_USER); + Map success = dataSourceService.authedDatasource(loginUser, userId); + Assert.assertEquals(Status.SUCCESS, success.get(Constants.STATUS)); + } + + @Test + public void queryDataSourceListTest() { User loginUser = new User(); loginUser.setUserType(UserType.GENERAL_USER); Map map = dataSourceService.queryDataSourceList(loginUser, DbType.MYSQL.ordinal()); @@ -62,35 +220,34 @@ public class DataSourceServiceTest { } @Test - public void verifyDataSourceNameTest(){ + public void verifyDataSourceNameTest() { User loginUser = new User(); loginUser.setUserType(UserType.GENERAL_USER); String dataSourceName = "dataSource1"; PowerMockito.when(dataSourceMapper.queryDataSourceByName(dataSourceName)).thenReturn(getDataSourceList()); - Result result = dataSourceService.verifyDataSourceName(loginUser, dataSourceName); - Assert.assertEquals(Status.DATASOURCE_EXIST.getMsg(),result.getMsg()); + Result result = dataSourceService.verifyDataSourceName(dataSourceName); + Assert.assertEquals(Status.DATASOURCE_EXIST.getMsg(), result.getMsg()); } @Test - public void queryDataSourceTest(){ + public void queryDataSourceTest() { PowerMockito.when(dataSourceMapper.selectById(Mockito.anyInt())).thenReturn(null); Map result = dataSourceService.queryDataSource(Mockito.anyInt()); - Assert.assertEquals(((Status)result.get(Constants.STATUS)).getCode(),Status.RESOURCE_NOT_EXIST.getCode()); + Assert.assertEquals(((Status) result.get(Constants.STATUS)).getCode(), Status.RESOURCE_NOT_EXIST.getCode()); PowerMockito.when(dataSourceMapper.selectById(Mockito.anyInt())).thenReturn(getOracleDataSource()); result = dataSourceService.queryDataSource(Mockito.anyInt()); - Assert.assertEquals(((Status)result.get(Constants.STATUS)).getCode(),Status.SUCCESS.getCode()); + Assert.assertEquals(((Status) result.get(Constants.STATUS)).getCode(), Status.SUCCESS.getCode()); } + private List getDataSourceList() { - private List getDataSourceList(){ - - List dataSources = new ArrayList<>(); + List dataSources = new ArrayList<>(); dataSources.add(getOracleDataSource()); return dataSources; } - private DataSource getOracleDataSource(){ + private DataSource getOracleDataSource() { DataSource dataSource = new DataSource(); dataSource.setName("test"); dataSource.setNote("Note"); @@ -101,31 +258,40 @@ public class DataSourceServiceTest { } @Test - public void buildParameter(){ - String param = dataSourceService.buildParameter("","", DbType.ORACLE, "192.168.9.1","1521","im" - ,"","test","test", DbConnectType.ORACLE_SERVICE_NAME,""); + public void buildParameter() { + String param = dataSourceService.buildParameter(DbType.ORACLE, "192.168.9.1", "1521", "im" + , "", "test", "test", DbConnectType.ORACLE_SERVICE_NAME, ""); String expected = "{\"connectType\":\"ORACLE_SERVICE_NAME\",\"type\":\"ORACLE_SERVICE_NAME\",\"address\":\"jdbc:oracle:thin:@//192.168.9.1:1521\",\"database\":\"im\",\"jdbcUrl\":\"jdbc:oracle:thin:@//192.168.9.1:1521/im\",\"user\":\"test\",\"password\":\"test\"}"; Assert.assertEquals(expected, param); } @Test - public void buildParameterWithDecodePassword(){ - PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE,"true"); - String param = dataSourceService.buildParameter("name","desc", DbType.MYSQL, "192.168.9.1","1521","im" - ,"","test","123456", null,""); + public void buildParameterWithDecodePassword() { + PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE, "true"); + String param = dataSourceService.buildParameter(DbType.MYSQL, "192.168.9.1", "1521", "im" + , "", "test", "123456", null, ""); String expected = "{\"type\":null,\"address\":\"jdbc:mysql://192.168.9.1:1521\",\"database\":\"im\",\"jdbcUrl\":\"jdbc:mysql://192.168.9.1:1521/im\",\"user\":\"test\",\"password\":\"IUAjJCVeJipNVEl6TkRVMg==\"}"; Assert.assertEquals(expected, param); - PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE,"false"); - param = dataSourceService.buildParameter("name","desc", DbType.MYSQL, "192.168.9.1","1521","im" - ,"","test","123456", null,""); + PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE, "false"); + param = dataSourceService.buildParameter(DbType.MYSQL, "192.168.9.1", "1521", "im" + , "", "test", "123456", null, ""); expected = "{\"type\":null,\"address\":\"jdbc:mysql://192.168.9.1:1521\",\"database\":\"im\",\"jdbcUrl\":\"jdbc:mysql://192.168.9.1:1521/im\",\"user\":\"test\",\"password\":\"123456\"}"; Assert.assertEquals(expected, param); } - - - + /** + * get Mock Admin User + * + * @return admin user + */ + private User getAdminUser() { + User loginUser = new User(); + loginUser.setId(-1); + loginUser.setUserName("admin"); + loginUser.setUserType(UserType.GENERAL_USER); + return loginUser; + } } \ No newline at end of file diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ExecutorService2Test.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ExecutorService2Test.java index 59523bdd11..a4c0c6bfe7 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ExecutorService2Test.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ExecutorService2Test.java @@ -16,17 +16,36 @@ */ package org.apache.dolphinscheduler.api.service; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + import org.apache.dolphinscheduler.api.enums.Status; +import org.apache.dolphinscheduler.api.service.impl.ProjectServiceImpl; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.common.enums.Priority; import org.apache.dolphinscheduler.common.enums.ReleaseState; import org.apache.dolphinscheduler.common.enums.RunMode; import org.apache.dolphinscheduler.common.model.Server; -import org.apache.dolphinscheduler.dao.entity.*; +import org.apache.dolphinscheduler.dao.entity.Command; +import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; +import org.apache.dolphinscheduler.dao.entity.Project; +import org.apache.dolphinscheduler.dao.entity.Schedule; +import org.apache.dolphinscheduler.dao.entity.Tenant; +import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper; import org.apache.dolphinscheduler.dao.mapper.ProjectMapper; import org.apache.dolphinscheduler.service.process.ProcessService; + +import java.text.ParseException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -36,13 +55,6 @@ import org.mockito.Mock; import org.mockito.Mockito; import org.mockito.junit.MockitoJUnitRunner; -import java.text.ParseException; -import java.util.*; - -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.times; - /** * test for ExecutorService */ @@ -62,7 +74,7 @@ public class ExecutorService2Test { private ProjectMapper projectMapper; @Mock - private ProjectService projectService; + private ProjectServiceImpl projectService; @Mock private MonitorService monitorService; @@ -84,7 +96,7 @@ public class ExecutorService2Test { private String cronTime; @Before - public void init(){ + public void init() { // user loginUser.setId(userId); @@ -111,7 +123,6 @@ public class ExecutorService2Test { /** * not complement - * @throws ParseException */ @Test public void testNoComplement() throws ParseException { @@ -125,13 +136,12 @@ public class ExecutorService2Test { Priority.LOW, Constants.DEFAULT_WORKER_GROUP, 110); Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); verify(processService, times(1)).createCommand(any(Command.class)); - }catch (Exception e){ + } catch (Exception e) { } } /** * date error - * @throws ParseException */ @Test public void testDateError() throws ParseException { @@ -145,13 +155,12 @@ public class ExecutorService2Test { Priority.LOW, Constants.DEFAULT_WORKER_GROUP, 110); Assert.assertEquals(Status.START_PROCESS_INSTANCE_ERROR, result.get(Constants.STATUS)); verify(processService, times(0)).createCommand(any(Command.class)); - }catch (Exception e){ + } catch (Exception e) { } } /** * serial - * @throws ParseException */ @Test public void testSerial() throws ParseException { @@ -165,17 +174,16 @@ public class ExecutorService2Test { Priority.LOW, Constants.DEFAULT_WORKER_GROUP, 110); Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); verify(processService, times(1)).createCommand(any(Command.class)); - }catch (Exception e){ + } catch (Exception e) { } } /** * without schedule - * @throws ParseException */ @Test public void testParallelWithOutSchedule() throws ParseException { - try{ + try { Mockito.when(processService.queryReleaseSchedulerListByProcessDefinitionId(processDefinitionId)).thenReturn(zeroSchedulerList()); Map result = executorService.execProcessInstance(loginUser, projectName, processDefinitionId, cronTime, CommandType.COMPLEMENT_DATA, @@ -185,17 +193,16 @@ public class ExecutorService2Test { Priority.LOW, Constants.DEFAULT_WORKER_GROUP, 110); Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); verify(processService, times(31)).createCommand(any(Command.class)); - }catch (Exception e){ + } catch (Exception e) { } } /** * with schedule - * @throws ParseException */ @Test public void testParallelWithSchedule() throws ParseException { - try{ + try { Mockito.when(processService.queryReleaseSchedulerListByProcessDefinitionId(processDefinitionId)).thenReturn(oneSchedulerList()); Map result = executorService.execProcessInstance(loginUser, projectName, processDefinitionId, cronTime, CommandType.COMPLEMENT_DATA, @@ -205,13 +212,13 @@ public class ExecutorService2Test { Priority.LOW, Constants.DEFAULT_WORKER_GROUP, 110); Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); verify(processService, times(15)).createCommand(any(Command.class)); - }catch (Exception e){ + } catch (Exception e) { } } @Test - public void testNoMsterServers() throws ParseException{ + public void testNoMsterServers() throws ParseException { Mockito.when(monitorService.getServerListFromZK(true)).thenReturn(new ArrayList()); Map result = executorService.execProcessInstance(loginUser, projectName, @@ -220,11 +227,11 @@ public class ExecutorService2Test { null, null, 0, "", "", RunMode.RUN_MODE_PARALLEL, Priority.LOW, Constants.DEFAULT_WORKER_GROUP, 110); - Assert.assertEquals(result.get(Constants.STATUS),Status.MASTER_NOT_EXISTS); + Assert.assertEquals(result.get(Constants.STATUS), Status.MASTER_NOT_EXISTS); } - private List getMasterServersList(){ + private List getMasterServersList() { List masterServerList = new ArrayList<>(); Server masterServer1 = new Server(); masterServer1.setId(1); @@ -242,11 +249,11 @@ public class ExecutorService2Test { } - private List zeroSchedulerList(){ + private List zeroSchedulerList() { return Collections.EMPTY_LIST; } - private List oneSchedulerList(){ + private List oneSchedulerList() { List schedulerList = new LinkedList<>(); Schedule schedule = new Schedule(); schedule.setCrontab("0 0 0 1/2 * ?"); @@ -254,7 +261,7 @@ public class ExecutorService2Test { return schedulerList; } - private Map checkProjectAndAuth(){ + private Map checkProjectAndAuth() { Map result = new HashMap<>(); result.put(Constants.STATUS, Status.SUCCESS); return result; diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ExecutorServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ExecutorServiceTest.java index 6551919e4a..57cd207c4d 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ExecutorServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ExecutorServiceTest.java @@ -54,7 +54,7 @@ public class ExecutorServiceTest { @Test public void putMsgWithParamsTest() { - Map map = new HashMap<>(5); + Map map = new HashMap<>(); putMsgWithParams(map, Status.PROJECT_ALREADY_EXISTS); logger.info(map.toString()); diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/LoggerServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/LoggerServiceTest.java index 4e41ed39b0..3952a25542 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/LoggerServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/LoggerServiceTest.java @@ -17,10 +17,14 @@ package org.apache.dolphinscheduler.api.service; import org.apache.dolphinscheduler.api.enums.Status; +import org.apache.dolphinscheduler.api.service.impl.LoggerServiceImpl; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.service.process.ProcessService; + +import org.junit.After; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.InjectMocks; @@ -32,25 +36,30 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; @RunWith(MockitoJUnitRunner.class) -@PrepareForTest({LoggerService.class}) +@PrepareForTest({LoggerServiceImpl.class}) public class LoggerServiceTest { private static final Logger logger = LoggerFactory.getLogger(LoggerServiceTest.class); @InjectMocks - private LoggerService loggerService; + private LoggerServiceImpl loggerService; @Mock private ProcessService processService; + @Before + public void init() { + this.loggerService.init(); + } + @Test - public void testQueryDataSourceList(){ + public void testQueryDataSourceList() { TaskInstance taskInstance = new TaskInstance(); Mockito.when(processService.findTaskInstanceById(1)).thenReturn(taskInstance); - Result result = loggerService.queryLog(2,1,1); + Result result = loggerService.queryLog(2, 1, 1); //TASK_INSTANCE_NOT_FOUND - Assert.assertEquals(Status.TASK_INSTANCE_NOT_FOUND.getCode(),result.getCode().intValue()); + Assert.assertEquals(Status.TASK_INSTANCE_NOT_FOUND.getCode(), result.getCode().intValue()); try { //HOST NOT FOUND OR ILLEGAL @@ -59,36 +68,36 @@ public class LoggerServiceTest { Assert.assertTrue(true); logger.error("testQueryDataSourceList error {}", e.getMessage()); } - Assert.assertEquals(Status.TASK_INSTANCE_NOT_FOUND.getCode(),result.getCode().intValue()); + Assert.assertEquals(Status.TASK_INSTANCE_NOT_FOUND.getCode(), result.getCode().intValue()); //SUCCESS taskInstance.setHost("127.0.0.1:8080"); taskInstance.setLogPath("/temp/log"); Mockito.when(processService.findTaskInstanceById(1)).thenReturn(taskInstance); - result = loggerService.queryLog(1,1,1); - Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); + result = loggerService.queryLog(1, 1, 1); + Assert.assertEquals(Status.SUCCESS.getCode(), result.getCode().intValue()); } @Test - public void testGetLogBytes(){ + public void testGetLogBytes() { TaskInstance taskInstance = new TaskInstance(); Mockito.when(processService.findTaskInstanceById(1)).thenReturn(taskInstance); //task instance is null - try{ + try { loggerService.getLogBytes(2); - }catch (RuntimeException e){ + } catch (RuntimeException e) { Assert.assertTrue(true); - logger.error("testGetLogBytes error: {}","task instance is null"); + logger.error("testGetLogBytes error: {}", "task instance is null"); } //task instance host is null - try{ + try { loggerService.getLogBytes(1); - }catch (RuntimeException e){ + } catch (RuntimeException e) { Assert.assertTrue(true); - logger.error("testGetLogBytes error: {}","task instance host is null"); + logger.error("testGetLogBytes error: {}", "task instance host is null"); } //success @@ -100,4 +109,9 @@ public class LoggerServiceTest { } + @After + public void close() { + this.loggerService.close(); + } + } \ No newline at end of file diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java index 8db667e28b..33032f54e4 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java @@ -14,23 +14,50 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.api.service; -import com.fasterxml.jackson.databind.node.ArrayNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import org.apache.dolphinscheduler.api.ApiApplicationServer; import org.apache.dolphinscheduler.api.dto.ProcessMeta; import org.apache.dolphinscheduler.api.enums.Status; +import org.apache.dolphinscheduler.api.service.impl.ProcessDefinitionServiceImpl; +import org.apache.dolphinscheduler.api.service.impl.ProjectServiceImpl; import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.common.enums.*; +import org.apache.dolphinscheduler.common.enums.ExecutionStatus; +import org.apache.dolphinscheduler.common.enums.FailureStrategy; +import org.apache.dolphinscheduler.common.enums.Priority; +import org.apache.dolphinscheduler.common.enums.ReleaseState; +import org.apache.dolphinscheduler.common.enums.UserType; +import org.apache.dolphinscheduler.common.enums.WarningType; import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.common.utils.FileUtils; -import org.apache.dolphinscheduler.common.utils.*; -import org.apache.dolphinscheduler.dao.entity.*; -import org.apache.dolphinscheduler.dao.mapper.*; +import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.StringUtils; +import org.apache.dolphinscheduler.dao.entity.DataSource; +import org.apache.dolphinscheduler.dao.entity.ProcessData; +import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; +import org.apache.dolphinscheduler.dao.entity.ProcessInstance; +import org.apache.dolphinscheduler.dao.entity.Project; +import org.apache.dolphinscheduler.dao.entity.Schedule; +import org.apache.dolphinscheduler.dao.entity.TaskInstance; +import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper; +import org.apache.dolphinscheduler.dao.mapper.ProjectMapper; +import org.apache.dolphinscheduler.dao.mapper.ScheduleMapper; +import org.apache.dolphinscheduler.dao.mapper.TaskInstanceMapper; import org.apache.dolphinscheduler.service.process.ProcessService; + import org.apache.http.entity.ContentType; -import org.json.JSONException; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.text.MessageFormat; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; @@ -38,26 +65,17 @@ import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.Mockito; import org.mockito.junit.MockitoJUnitRunner; -import org.skyscreamer.jsonassert.JSONAssert; -import org.springframework.boot.test.context.SpringBootTest; import org.springframework.mock.web.MockMultipartFile; import org.springframework.web.multipart.MultipartFile; -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.text.MessageFormat; -import java.util.*; +import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; -@RunWith(MockitoJUnitRunner.Silent.class) -@SpringBootTest(classes = ApiApplicationServer.class) +@RunWith(MockitoJUnitRunner.class) public class ProcessDefinitionServiceTest { @InjectMocks - ProcessDefinitionService processDefinitionService; - - @Mock - private DataSourceMapper dataSourceMapper; + private ProcessDefinitionServiceImpl processDefinitionService; @Mock private ProcessDefinitionMapper processDefineMapper; @@ -66,44 +84,166 @@ public class ProcessDefinitionServiceTest { private ProjectMapper projectMapper; @Mock - private ProjectService projectService; + private ProjectServiceImpl projectService; @Mock private ScheduleMapper scheduleMapper; - - @Mock private ProcessService processService; @Mock - private ProcessInstanceMapper processInstanceMapper; + private ProcessInstanceService processInstanceService; @Mock private TaskInstanceMapper taskInstanceMapper; - private String sqlDependentJson = "{\"globalParams\":[]," + - "\"tasks\":[{\"type\":\"SQL\",\"id\":\"tasks-27297\",\"name\":\"sql\"," + - "\"params\":{\"type\":\"MYSQL\",\"datasource\":1,\"sql\":\"select * from test\"," + - "\"udfs\":\"\",\"sqlType\":\"1\",\"title\":\"\",\"receivers\":\"\",\"receiversCc\":\"\",\"showType\":\"TABLE\"" + - ",\"localParams\":[],\"connParams\":\"\"," + - "\"preStatements\":[],\"postStatements\":[]}," + - "\"description\":\"\",\"runFlag\":\"NORMAL\",\"dependence\":{},\"maxRetryTimes\":\"0\"," + - "\"retryInterval\":\"1\",\"timeout\":{\"strategy\":\"\"," + - "\"enable\":false},\"taskInstancePriority\":\"MEDIUM\",\"workerGroupId\":-1," + - "\"preTasks\":[\"dependent\"]},{\"type\":\"DEPENDENT\",\"id\":\"tasks-33787\"," + - "\"name\":\"dependent\",\"params\":{},\"description\":\"\",\"runFlag\":\"NORMAL\"," + - "\"dependence\":{\"relation\":\"AND\",\"dependTaskList\":[{\"relation\":\"AND\"," + - "\"dependItemList\":[{\"projectId\":2,\"definitionId\":46,\"depTasks\":\"ALL\"," + - "\"cycle\":\"day\",\"dateValue\":\"today\"}]}]},\"maxRetryTimes\":\"0\",\"retryInterval\":\"1\"," + - "\"timeout\":{\"strategy\":\"\",\"enable\":false},\"taskInstancePriority\":\"MEDIUM\"," + - "\"workerGroupId\":-1,\"preTasks\":[]}],\"tenantId\":1,\"timeout\":0}"; + @Mock + private ProcessDefinitionVersionService processDefinitionVersionService; - private String shellJson = "{\"globalParams\":[],\"tasks\":[{\"type\":\"SHELL\",\"id\":\"tasks-9527\",\"name\":\"shell-1\"," + - "\"params\":{\"resourceList\":[],\"localParams\":[],\"rawScript\":\"#!/bin/bash\\necho \\\"shell-1\\\"\"}," + - "\"description\":\"\",\"runFlag\":\"NORMAL\",\"dependence\":{},\"maxRetryTimes\":\"0\",\"retryInterval\":\"1\"," + - "\"timeout\":{\"strategy\":\"\",\"interval\":1,\"enable\":false},\"taskInstancePriority\":\"MEDIUM\"," + - "\"workerGroupId\":-1,\"preTasks\":[]}],\"tenantId\":1,\"timeout\":0}"; + private static final String SHELL_JSON = "{\n" + + " \"globalParams\": [\n" + + " \n" + + " ],\n" + + " \"tasks\": [\n" + + " {\n" + + " \"type\": \"SHELL\",\n" + + " \"id\": \"tasks-9527\",\n" + + " \"name\": \"shell-1\",\n" + + " \"params\": {\n" + + " \"resourceList\": [\n" + + " \n" + + " ],\n" + + " \"localParams\": [\n" + + " \n" + + " ],\n" + + " \"rawScript\": \"#!/bin/bash\\necho \\\"shell-1\\\"\"\n" + + " },\n" + + " \"description\": \"\",\n" + + " \"runFlag\": \"NORMAL\",\n" + + " \"dependence\": {\n" + + " \n" + + " },\n" + + " \"maxRetryTimes\": \"0\",\n" + + " \"retryInterval\": \"1\",\n" + + " \"timeout\": {\n" + + " \"strategy\": \"\",\n" + + " \"interval\": 1,\n" + + " \"enable\": false\n" + + " },\n" + + " \"taskInstancePriority\": \"MEDIUM\",\n" + + " \"workerGroupId\": -1,\n" + + " \"preTasks\": [\n" + + " \n" + + " ]\n" + + " }\n" + + " ],\n" + + " \"tenantId\": 1,\n" + + " \"timeout\": 0\n" + + "}"; + + private static final String CYCLE_SHELL_JSON = "{\n" + + " \"globalParams\": [\n" + + " \n" + + " ],\n" + + " \"tasks\": [\n" + + " {\n" + + " \"type\": \"SHELL\",\n" + + " \"id\": \"tasks-9527\",\n" + + " \"name\": \"shell-1\",\n" + + " \"params\": {\n" + + " \"resourceList\": [\n" + + " \n" + + " ],\n" + + " \"localParams\": [\n" + + " \n" + + " ],\n" + + " \"rawScript\": \"#!/bin/bash\\necho \\\"shell-1\\\"\"\n" + + " },\n" + + " \"description\": \"\",\n" + + " \"runFlag\": \"NORMAL\",\n" + + " \"dependence\": {\n" + + " \n" + + " },\n" + + " \"maxRetryTimes\": \"0\",\n" + + " \"retryInterval\": \"1\",\n" + + " \"timeout\": {\n" + + " \"strategy\": \"\",\n" + + " \"interval\": 1,\n" + + " \"enable\": false\n" + + " },\n" + + " \"taskInstancePriority\": \"MEDIUM\",\n" + + " \"workerGroupId\": -1,\n" + + " \"preTasks\": [\n" + + " \"tasks-9529\"\n" + + " ]\n" + + " },\n" + + " {\n" + + " \"type\": \"SHELL\",\n" + + " \"id\": \"tasks-9528\",\n" + + " \"name\": \"shell-1\",\n" + + " \"params\": {\n" + + " \"resourceList\": [\n" + + " \n" + + " ],\n" + + " \"localParams\": [\n" + + " \n" + + " ],\n" + + " \"rawScript\": \"#!/bin/bash\\necho \\\"shell-1\\\"\"\n" + + " },\n" + + " \"description\": \"\",\n" + + " \"runFlag\": \"NORMAL\",\n" + + " \"dependence\": {\n" + + " \n" + + " },\n" + + " \"maxRetryTimes\": \"0\",\n" + + " \"retryInterval\": \"1\",\n" + + " \"timeout\": {\n" + + " \"strategy\": \"\",\n" + + " \"interval\": 1,\n" + + " \"enable\": false\n" + + " },\n" + + " \"taskInstancePriority\": \"MEDIUM\",\n" + + " \"workerGroupId\": -1,\n" + + " \"preTasks\": [\n" + + " \"tasks-9527\"\n" + + " ]\n" + + " },\n" + + " {\n" + + " \"type\": \"SHELL\",\n" + + " \"id\": \"tasks-9529\",\n" + + " \"name\": \"shell-1\",\n" + + " \"params\": {\n" + + " \"resourceList\": [\n" + + " \n" + + " ],\n" + + " \"localParams\": [\n" + + " \n" + + " ],\n" + + " \"rawScript\": \"#!/bin/bash\\necho \\\"shell-1\\\"\"\n" + + " },\n" + + " \"description\": \"\",\n" + + " \"runFlag\": \"NORMAL\",\n" + + " \"dependence\": {\n" + + " \n" + + " },\n" + + " \"maxRetryTimes\": \"0\",\n" + + " \"retryInterval\": \"1\",\n" + + " \"timeout\": {\n" + + " \"strategy\": \"\",\n" + + " \"interval\": 1,\n" + + " \"enable\": false\n" + + " },\n" + + " \"taskInstancePriority\": \"MEDIUM\",\n" + + " \"workerGroupId\": -1,\n" + + " \"preTasks\": [\n" + + " \"tasks-9528\"\n" + + " ]\n" + + " }\n" + + " ],\n" + + " \"tenantId\": 1,\n" + + " \"timeout\": 0\n" + + "}"; @Test public void testQueryProcessDefinitionList() { @@ -115,25 +255,26 @@ public class ProcessDefinitionServiceTest { loginUser.setId(-1); loginUser.setUserType(UserType.GENERAL_USER); - Map result = new HashMap<>(5); + Map result = new HashMap<>(); putMsg(result, Status.PROJECT_NOT_FOUNT, projectName); //project not found - Mockito.when(projectService.checkProjectAndAuth(loginUser,project,projectName)).thenReturn(result); - Map map = processDefinitionService.queryProcessDefinitionList(loginUser,"project_test1"); + Mockito.when(projectService.checkProjectAndAuth(loginUser, project, projectName)).thenReturn(result); + Map map = processDefinitionService.queryProcessDefinitionList(loginUser, "project_test1"); Assert.assertEquals(Status.PROJECT_NOT_FOUNT, map.get(Constants.STATUS)); //project check auth success putMsg(result, Status.SUCCESS, projectName); - Mockito.when(projectService.checkProjectAndAuth(loginUser,project,projectName)).thenReturn(result); + Mockito.when(projectService.checkProjectAndAuth(loginUser, project, projectName)).thenReturn(result); List resourceList = new ArrayList<>(); resourceList.add(getProcessDefinition()); Mockito.when(processDefineMapper.queryAllDefinitionList(project.getId())).thenReturn(resourceList); - Map checkSuccessRes = processDefinitionService.queryProcessDefinitionList(loginUser,"project_test1"); + Map checkSuccessRes = processDefinitionService.queryProcessDefinitionList(loginUser, "project_test1"); Assert.assertEquals(Status.SUCCESS, checkSuccessRes.get(Constants.STATUS)); } @Test + @SuppressWarnings("unchecked") public void testQueryProcessDefinitionListPaging() { String projectName = "project_test1"; Mockito.when(projectMapper.queryByName(projectName)).thenReturn(getProject(projectName)); @@ -144,14 +285,30 @@ public class ProcessDefinitionServiceTest { loginUser.setId(-1); loginUser.setUserType(UserType.GENERAL_USER); - Map result = new HashMap<>(5); + Map result = new HashMap<>(); putMsg(result, Status.PROJECT_NOT_FOUNT, projectName); //project not found - Mockito.when(projectService.checkProjectAndAuth(loginUser,project,projectName)).thenReturn(result); - Map map = processDefinitionService.queryProcessDefinitionListPaging(loginUser, "project_test1", "",1, 5,0); + Mockito.when(projectService.checkProjectAndAuth(loginUser, project, projectName)).thenReturn(result); + Map map = processDefinitionService.queryProcessDefinitionListPaging(loginUser, "project_test1", "", 1, 5, 0); Assert.assertEquals(Status.PROJECT_NOT_FOUNT, map.get(Constants.STATUS)); + putMsg(result, Status.SUCCESS, projectName); + loginUser.setId(1); + Mockito.when(projectService.checkProjectAndAuth(loginUser, project, projectName)).thenReturn(result); + Page page = new Page<>(1, 10); + page.setTotal(30); + Mockito.when(processDefineMapper.queryDefineListPaging( + Mockito.any(IPage.class) + , Mockito.eq("") + , Mockito.eq(loginUser.getId()) + , Mockito.eq(project.getId()) + , Mockito.anyBoolean())).thenReturn(page); + + Map map1 = processDefinitionService.queryProcessDefinitionListPaging( + loginUser, projectName, "", 1, 10, loginUser.getId()); + + Assert.assertEquals(Status.SUCCESS, map1.get(Constants.STATUS)); } @Test @@ -165,18 +322,18 @@ public class ProcessDefinitionServiceTest { loginUser.setId(-1); loginUser.setUserType(UserType.GENERAL_USER); - Map result = new HashMap<>(5); + Map result = new HashMap<>(); putMsg(result, Status.PROJECT_NOT_FOUNT, projectName); //project check auth fail - Mockito.when(projectService.checkProjectAndAuth(loginUser,project,projectName)).thenReturn(result); + Mockito.when(projectService.checkProjectAndAuth(loginUser, project, projectName)).thenReturn(result); Map map = processDefinitionService.queryProcessDefinitionById(loginUser, "project_test1", 1); Assert.assertEquals(Status.PROJECT_NOT_FOUNT, map.get(Constants.STATUS)); //project check auth success, instance not exist putMsg(result, Status.SUCCESS, projectName); - Mockito.when(projectService.checkProjectAndAuth(loginUser,project,projectName)).thenReturn(result); + Mockito.when(projectService.checkProjectAndAuth(loginUser, project, projectName)).thenReturn(result); Mockito.when(processDefineMapper.selectById(1)).thenReturn(null); Map instanceNotexitRes = processDefinitionService.queryProcessDefinitionById(loginUser, "project_test1", 1); @@ -190,48 +347,119 @@ public class ProcessDefinitionServiceTest { } @Test - public void testCopyProcessDefinition() throws Exception{ - String projectName = "project_test1"; - Mockito.when(projectMapper.queryByName(projectName)).thenReturn(getProject(projectName)); + public void testBatchCopyProcessDefinition() { + String projectName = "project_test1"; Project project = getProject(projectName); User loginUser = new User(); loginUser.setId(-1); loginUser.setUserType(UserType.GENERAL_USER); - Map result = new HashMap<>(5); - //project check auth success, instance not exist + // copy project definition ids empty test + Map map = processDefinitionService.batchCopyProcessDefinition(loginUser, projectName, StringUtils.EMPTY, 0); + Assert.assertEquals(Status.PROCESS_DEFINITION_IDS_IS_EMPTY, map.get(Constants.STATUS)); + + Map result = new HashMap<>(); + + // project check auth fail + putMsg(result, Status.PROJECT_NOT_FOUNT, projectName); + Mockito.when(projectMapper.queryByName(projectName)).thenReturn(getProject(projectName)); + Mockito.when(projectService.checkProjectAndAuth(loginUser, project, projectName)).thenReturn(result); + Map map1 = processDefinitionService.batchCopyProcessDefinition( + loginUser, projectName, String.valueOf(project.getId()), 0); + Assert.assertEquals(Status.PROJECT_NOT_FOUNT, map1.get(Constants.STATUS)); + + // project check auth success, target project is null putMsg(result, Status.SUCCESS, projectName); - Mockito.when(projectService.checkProjectAndAuth(loginUser,project,projectName)).thenReturn(result); + Mockito.when(projectMapper.queryByName(projectName)).thenReturn(getProject(projectName)); + Mockito.when(projectService.checkProjectAndAuth(loginUser, project, projectName)).thenReturn(result); + Mockito.when(projectMapper.queryDetailById(0)).thenReturn(null); + Map map2 = processDefinitionService.batchCopyProcessDefinition( + loginUser, projectName, String.valueOf(project.getId()), 0); + Assert.assertEquals(Status.PROJECT_NOT_FOUNT, map2.get(Constants.STATUS)); + + // project check auth success, target project name not equal project name, check auth target project fail + Project project1 = getProject(projectName); + Mockito.when(projectMapper.queryByName(projectName)).thenReturn(project1); + Mockito.when(projectService.checkProjectAndAuth(loginUser, project, projectName)).thenReturn(result); + + putMsg(result, Status.SUCCESS, projectName); + String projectName2 = "project_test2"; + Project project2 = getProject(projectName2); + Mockito.when(projectMapper.queryByName(projectName2)).thenReturn(project2); + Mockito.when(projectService.checkProjectAndAuth(loginUser, project2, projectName2)).thenReturn(result); + Mockito.when(projectMapper.queryDetailById(1)).thenReturn(project2); + // instance exit + ProcessDefinition definition = getProcessDefinition(); + definition.setLocations("{\"tasks-36196\":{\"name\":\"ssh_test1\",\"targetarr\":\"\",\"x\":141,\"y\":70}}"); + definition.setProcessDefinitionJson("{\"globalParams\":[],\"tasks\":[{\"type\":\"SHELL\",\"id\":\"tasks-36196\"," + + "\"name\":\"ssh_test1\",\"params\":{\"resourceList\":[],\"localParams\":[],\"rawScript\":\"aa=\\\"1234" + + "\\\"\\necho ${aa}\"},\"desc\":\"\",\"runFlag\":\"NORMAL\",\"dependence\":{},\"maxRetryTimes\":\"0\"," + + "\"retryInterval\":\"1\",\"timeout\":{\"strategy\":\"\",\"interval\":null,\"enable\":false}," + + "\"taskInstancePriority\":\"MEDIUM\",\"workerGroupId\":-1,\"preTasks\":[]}],\"tenantId\":-1,\"timeout\":0}"); + definition.setConnects("[]"); + + Mockito.when(processDefineMapper.selectById(46)).thenReturn(definition); + + Map map3 = processDefinitionService.batchCopyProcessDefinition( + loginUser, projectName, "46", 1); + Assert.assertEquals(Status.SUCCESS, map3.get(Constants.STATUS)); + + } + + @Test + public void testBatchMoveProcessDefinition() { + String projectName = "project_test1"; + Project project1 = getProject(projectName); + Mockito.when(projectMapper.queryByName(projectName)).thenReturn(project1); + + String projectName2 = "project_test2"; + Project project2 = getProject(projectName2); + Mockito.when(projectMapper.queryByName(projectName2)).thenReturn(project2); + + int targetProjectId = 2; + Mockito.when(projectMapper.queryDetailById(targetProjectId)).thenReturn(getProjectById(targetProjectId)); + + Project project = getProject(projectName); + Project targetProject = getProjectById(targetProjectId); + + User loginUser = new User(); + loginUser.setId(-1); + loginUser.setUserType(UserType.GENERAL_USER); + + Map result = new HashMap<>(); + putMsg(result, Status.SUCCESS, projectName); + + Map result2 = new HashMap<>(); + putMsg(result2, Status.SUCCESS, targetProject.getName()); + + Mockito.when(projectService.checkProjectAndAuth(loginUser, project, projectName)).thenReturn(result); + Mockito.when(projectService.checkProjectAndAuth(loginUser, project2, projectName2)).thenReturn(result); ProcessDefinition definition = getProcessDefinition(); definition.setLocations("{\"tasks-36196\":{\"name\":\"ssh_test1\",\"targetarr\":\"\",\"x\":141,\"y\":70}}"); - definition.setProcessDefinitionJson("{\"globalParams\":[],\"tasks\":[{\"type\":\"SHELL\",\"id\":\"tasks-36196\",\"name\":\"ssh_test1\",\"params\":{\"resourceList\":[],\"localParams\":[],\"rawScript\":\"aa=\\\"1234\\\"\\necho ${aa}\"},\"desc\":\"\",\"runFlag\":\"NORMAL\",\"dependence\":{},\"maxRetryTimes\":\"0\",\"retryInterval\":\"1\",\"timeout\":{\"strategy\":\"\",\"interval\":null,\"enable\":false},\"taskInstancePriority\":\"MEDIUM\",\"workerGroupId\":-1,\"preTasks\":[]}],\"tenantId\":-1,\"timeout\":0}"); + definition.setProcessDefinitionJson("{\"globalParams\":[],\"tasks\":[{\"type\":\"SHELL\",\"id\":\"tasks-36196\"" + + ",\"name\":\"ssh_test1\",\"params\":{\"resourceList\":[],\"localParams\":[],\"rawScript\":\"aa=\\\"1234" + + "\\\"\\necho ${aa}\"},\"desc\":\"\",\"runFlag\":\"NORMAL\",\"dependence\":{},\"maxRetryTimes\":\"0\"," + + "\"retryInterval\":\"1\",\"timeout\":{\"strategy\":\"\",\"interval\":null,\"enable\":false}," + + "\"taskInstancePriority\":\"MEDIUM\",\"workerGroupId\":-1,\"preTasks\":[]}],\"tenantId\":-1,\"timeout\":0}"); definition.setConnects("[]"); - //instance exit + + // check target project result == null + Mockito.when(processDefineMapper.updateById(definition)).thenReturn(46); Mockito.when(processDefineMapper.selectById(46)).thenReturn(definition); - Map createProcessResult = new HashMap<>(5); putMsg(result, Status.SUCCESS); - Mockito.when(processDefinitionService.createProcessDefinition( - loginUser, - definition.getProjectName(), - definition.getName(), - definition.getProcessDefinitionJson(), - definition.getDescription(), - definition.getLocations(), - definition.getConnects())).thenReturn(createProcessResult); - - Map successRes = processDefinitionService.copyProcessDefinition(loginUser, - "project_test1", 46); + Map successRes = processDefinitionService.batchMoveProcessDefinition( + loginUser, "project_test1", "46", 2); Assert.assertEquals(Status.SUCCESS, successRes.get(Constants.STATUS)); } @Test - public void deleteProcessDefinitionByIdTest() throws Exception { + public void deleteProcessDefinitionByIdTest() { String projectName = "project_test1"; Mockito.when(projectMapper.queryByName(projectName)).thenReturn(getProject(projectName)); @@ -241,15 +469,15 @@ public class ProcessDefinitionServiceTest { loginUser.setUserType(UserType.GENERAL_USER); //project check auth fail - Map result = new HashMap<>(5); + Map result = new HashMap<>(); putMsg(result, Status.PROJECT_NOT_FOUNT, projectName); - Mockito.when(projectService.checkProjectAndAuth(loginUser,project,projectName)).thenReturn(result); + Mockito.when(projectService.checkProjectAndAuth(loginUser, project, projectName)).thenReturn(result); Map map = processDefinitionService.deleteProcessDefinitionById(loginUser, "project_test1", 6); Assert.assertEquals(Status.PROJECT_NOT_FOUNT, map.get(Constants.STATUS)); //project check auth success, instance not exist putMsg(result, Status.SUCCESS, projectName); - Mockito.when(projectService.checkProjectAndAuth(loginUser,project,projectName)).thenReturn(result); + Mockito.when(projectService.checkProjectAndAuth(loginUser, project, projectName)).thenReturn(result); Mockito.when(processDefineMapper.selectById(1)).thenReturn(null); Map instanceNotexitRes = processDefinitionService.deleteProcessDefinitionById(loginUser, "project_test1", 1); @@ -317,40 +545,48 @@ public class ProcessDefinitionServiceTest { Project project = getProject(projectName); User loginUser = new User(); - loginUser.setId(-1); + loginUser.setId(1); loginUser.setUserType(UserType.GENERAL_USER); //project check auth fail - Map result = new HashMap<>(5); + Map result = new HashMap<>(); putMsg(result, Status.PROJECT_NOT_FOUNT, projectName); - Mockito.when(projectService.checkProjectAndAuth(loginUser,project,projectName)).thenReturn(result); + Mockito.when(projectService.checkProjectAndAuth(loginUser, project, projectName)).thenReturn(result); Map map = processDefinitionService.releaseProcessDefinition(loginUser, "project_test1", 6, ReleaseState.OFFLINE.getCode()); Assert.assertEquals(Status.PROJECT_NOT_FOUNT, map.get(Constants.STATUS)); - //project check auth success, processs definition online + // project check auth success, processs definition online putMsg(result, Status.SUCCESS, projectName); Mockito.when(processDefineMapper.selectById(46)).thenReturn(getProcessDefinition()); - Mockito.when(processDefineMapper.updateById(getProcessDefinition())).thenReturn(1); - Map onlineRes = processDefinitionService.releaseProcessDefinition(loginUser, "project_test1", - 46, ReleaseState.ONLINE.getCode()); + Map onlineRes = processDefinitionService.releaseProcessDefinition( + loginUser, "project_test1", 46, ReleaseState.ONLINE.getCode()); Assert.assertEquals(Status.SUCCESS, onlineRes.get(Constants.STATUS)); - //release error code - Map failRes = processDefinitionService.releaseProcessDefinition(loginUser, "project_test1", - 46, 2); + // project check auth success, processs definition online + ProcessDefinition processDefinition1 = getProcessDefinition(); + processDefinition1.setResourceIds("1,2"); + Mockito.when(processDefineMapper.selectById(46)).thenReturn(processDefinition1); + Mockito.when(processService.getUserById(1)).thenReturn(loginUser); + Map onlineWithResourceRes = processDefinitionService.releaseProcessDefinition( + loginUser, "project_test1", 46, ReleaseState.ONLINE.getCode()); + Assert.assertEquals(Status.SUCCESS, onlineWithResourceRes.get(Constants.STATUS)); + + // release error code + Map failRes = processDefinitionService.releaseProcessDefinition( + loginUser, "project_test1", 46, 2); Assert.assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR, failRes.get(Constants.STATUS)); //FIXME has function exit code 1 when exception //process definition offline -// List schedules = new ArrayList<>(); -// Schedule schedule = getSchedule(); -// schedules.add(schedule); -// Mockito.when(scheduleMapper.selectAllByProcessDefineArray(new int[]{46})).thenReturn(schedules); -// Mockito.when(scheduleMapper.updateById(schedule)).thenReturn(1); -// Map offlineRes = processDefinitionService.releaseProcessDefinition(loginUser, "project_test1", -// 46, ReleaseState.OFFLINE.getCode()); -// Assert.assertEquals(Status.SUCCESS, offlineRes.get(Constants.STATUS)); + // List schedules = new ArrayList<>(); + // Schedule schedule = getSchedule(); + // schedules.add(schedule); + // Mockito.when(scheduleMapper.selectAllByProcessDefineArray(new int[]{46})).thenReturn(schedules); + // Mockito.when(scheduleMapper.updateById(schedule)).thenReturn(1); + // Map offlineRes = processDefinitionService.releaseProcessDefinition(loginUser, "project_test1", + // 46, ReleaseState.OFFLINE.getCode()); + // Assert.assertEquals(Status.SUCCESS, offlineRes.get(Constants.STATUS)); } @Test @@ -364,22 +600,22 @@ public class ProcessDefinitionServiceTest { loginUser.setUserType(UserType.GENERAL_USER); //project check auth fail - Map result = new HashMap<>(5); + Map result = new HashMap<>(); putMsg(result, Status.PROJECT_NOT_FOUNT, projectName); - Mockito.when(projectService.checkProjectAndAuth(loginUser,project,projectName)).thenReturn(result); + Mockito.when(projectService.checkProjectAndAuth(loginUser, project, projectName)).thenReturn(result); Map map = processDefinitionService.verifyProcessDefinitionName(loginUser, "project_test1", "test_pdf"); Assert.assertEquals(Status.PROJECT_NOT_FOUNT, map.get(Constants.STATUS)); //project check auth success, process not exist putMsg(result, Status.SUCCESS, projectName); - Mockito.when(processDefineMapper.queryByDefineName(project.getId(),"test_pdf")).thenReturn(null); + Mockito.when(processDefineMapper.queryByDefineName(project.getId(), "test_pdf")).thenReturn(null); Map processNotExistRes = processDefinitionService.verifyProcessDefinitionName(loginUser, "project_test1", "test_pdf"); Assert.assertEquals(Status.SUCCESS, processNotExistRes.get(Constants.STATUS)); //process exist - Mockito.when(processDefineMapper.queryByDefineName(project.getId(),"test_pdf")).thenReturn(getProcessDefinition()); + Mockito.when(processDefineMapper.queryByDefineName(project.getId(), "test_pdf")).thenReturn(getProcessDefinition()); Map processExistRes = processDefinitionService.verifyProcessDefinitionName(loginUser, "project_test1", "test_pdf"); Assert.assertEquals(Status.PROCESS_INSTANCE_EXIST, processExistRes.get(Constants.STATUS)); @@ -391,27 +627,33 @@ public class ProcessDefinitionServiceTest { Map dataNotValidRes = processDefinitionService.checkProcessNodeList(null, ""); Assert.assertEquals(Status.DATA_IS_NOT_VALID, dataNotValidRes.get(Constants.STATUS)); - //task not empty - String processDefinitionJson = shellJson; + // task not empty + String processDefinitionJson = SHELL_JSON; ProcessData processData = JSONUtils.parseObject(processDefinitionJson, ProcessData.class); - assert processData != null; + Assert.assertNotNull(processData); Map taskEmptyRes = processDefinitionService.checkProcessNodeList(processData, processDefinitionJson); Assert.assertEquals(Status.SUCCESS, taskEmptyRes.get(Constants.STATUS)); - //task empty + // task empty processData.setTasks(null); Map taskNotEmptyRes = processDefinitionService.checkProcessNodeList(processData, processDefinitionJson); Assert.assertEquals(Status.DATA_IS_NULL, taskNotEmptyRes.get(Constants.STATUS)); + // task cycle + String processDefinitionJsonCycle = CYCLE_SHELL_JSON; + ProcessData processDataCycle = JSONUtils.parseObject(processDefinitionJsonCycle, ProcessData.class); + Map taskCycleRes = processDefinitionService.checkProcessNodeList(processDataCycle, processDefinitionJsonCycle); + Assert.assertEquals(Status.PROCESS_NODE_HAS_CYCLE, taskCycleRes.get(Constants.STATUS)); + //json abnormal - String abnormalJson = processDefinitionJson.replaceAll("SHELL",""); + String abnormalJson = processDefinitionJson.replaceAll("SHELL", ""); processData = JSONUtils.parseObject(abnormalJson, ProcessData.class); Map abnormalTaskRes = processDefinitionService.checkProcessNodeList(processData, abnormalJson); Assert.assertEquals(Status.PROCESS_NODE_S_PARAMETER_INVALID, abnormalTaskRes.get(Constants.STATUS)); } @Test - public void testGetTaskNodeListByDefinitionId() throws Exception { + public void testGetTaskNodeListByDefinitionId() { //process definition not exist Mockito.when(processDefineMapper.selectById(46)).thenReturn(null); Map processDefinitionNullRes = processDefinitionService.getTaskNodeListByDefinitionId(46); @@ -424,14 +666,14 @@ public class ProcessDefinitionServiceTest { Assert.assertEquals(Status.DATA_IS_NOT_VALID, successRes.get(Constants.STATUS)); //success - processDefinition.setProcessDefinitionJson(shellJson); + processDefinition.setProcessDefinitionJson(SHELL_JSON); Mockito.when(processDefineMapper.selectById(46)).thenReturn(processDefinition); Map dataNotValidRes = processDefinitionService.getTaskNodeListByDefinitionId(46); Assert.assertEquals(Status.SUCCESS, dataNotValidRes.get(Constants.STATUS)); } @Test - public void testGetTaskNodeListByDefinitionIdList() throws Exception { + public void testGetTaskNodeListByDefinitionIdList() { //process definition not exist String defineIdList = "46"; Integer[] idArray = {46}; @@ -441,7 +683,7 @@ public class ProcessDefinitionServiceTest { //process definition exist ProcessDefinition processDefinition = getProcessDefinition(); - processDefinition.setProcessDefinitionJson(shellJson); + processDefinition.setProcessDefinitionJson(SHELL_JSON); List processDefinitionList = new ArrayList<>(); processDefinitionList.add(processDefinition); Mockito.when(processDefineMapper.queryDefinitionListByIdList(idArray)).thenReturn(processDefinitionList); @@ -453,7 +695,7 @@ public class ProcessDefinitionServiceTest { public void testQueryProcessDefinitionAllByProjectId() { int projectId = 1; ProcessDefinition processDefinition = getProcessDefinition(); - processDefinition.setProcessDefinitionJson(shellJson); + processDefinition.setProcessDefinitionJson(SHELL_JSON); List processDefinitionList = new ArrayList<>(); processDefinitionList.add(processDefinition); Mockito.when(processDefineMapper.queryAllDefinitionList(projectId)).thenReturn(processDefinitionList); @@ -465,7 +707,7 @@ public class ProcessDefinitionServiceTest { public void testViewTree() throws Exception { //process definition not exist ProcessDefinition processDefinition = getProcessDefinition(); - processDefinition.setProcessDefinitionJson(shellJson); + processDefinition.setProcessDefinitionJson(SHELL_JSON); Mockito.when(processDefineMapper.selectById(46)).thenReturn(null); Map processDefinitionNullRes = processDefinitionService.viewTree(46, 10); Assert.assertEquals(Status.PROCESS_DEFINE_NOT_EXIST, processDefinitionNullRes.get(Constants.STATUS)); @@ -491,7 +733,7 @@ public class ProcessDefinitionServiceTest { //task instance not exist Mockito.when(processDefineMapper.selectById(46)).thenReturn(processDefinition); - Mockito.when(processInstanceMapper.queryByProcessDefineId(46, 10)).thenReturn(processInstanceList); + Mockito.when(processInstanceService.queryByProcessDefineId(46, 10)).thenReturn(processInstanceList); Mockito.when(taskInstanceMapper.queryByInstanceIdAndName(processInstance.getId(), "shell-1")).thenReturn(null); Map taskNullRes = processDefinitionService.viewTree(46, 10); Assert.assertEquals(Status.SUCCESS, taskNullRes.get(Constants.STATUS)); @@ -502,185 +744,76 @@ public class ProcessDefinitionServiceTest { Assert.assertEquals(Status.SUCCESS, taskNotNuLLRes.get(Constants.STATUS)); } - /** - * add datasource param and dependent when export process - * @throws JSONException - */ - @Test - public void testAddTaskNodeSpecialParam() throws JSONException { - - Mockito.when(dataSourceMapper.selectById(1)).thenReturn(getDataSource()); - Mockito.when(processDefineMapper.queryByDefineId(2)).thenReturn(getProcessDefinition()); - - String corSqlDependentJson = processDefinitionService.addExportTaskNodeSpecialParam(sqlDependentJson); - - JSONAssert.assertEquals(sqlDependentJson,corSqlDependentJson,false); - - } - - @Test - public void testExportProcessMetaDataStr() { - Mockito.when(scheduleMapper.queryByProcessDefinitionId(46)).thenReturn(getSchedulerList()); - ProcessDefinition processDefinition = getProcessDefinition(); - processDefinition.setProcessDefinitionJson(sqlDependentJson); - - String exportProcessMetaDataStr = processDefinitionService.exportProcessMetaDataStr(46, processDefinition); - Assert.assertNotEquals(sqlDependentJson,exportProcessMetaDataStr); - } - - @Test - public void testAddExportTaskNodeSpecialParam() throws JSONException { - String shellData = shellJson; - - String resultStr = processDefinitionService.addExportTaskNodeSpecialParam(shellData); - JSONAssert.assertEquals(shellJson, resultStr, false); - } - - @Test - public void testImportProcessSchedule() { - User loginUser = new User(); - loginUser.setId(1); - loginUser.setUserType(UserType.GENERAL_USER); - - String currentProjectName = "test"; - String processDefinitionName = "test_process"; - Integer processDefinitionId = 1; - Schedule schedule = getSchedule(); - - ProcessMeta processMeta = getProcessMeta(); - - int insertFlag = processDefinitionService.importProcessSchedule(loginUser, currentProjectName, processMeta, - processDefinitionName, processDefinitionId); - Assert.assertEquals(0, insertFlag); - - ProcessMeta processMetaCron = new ProcessMeta(); - processMetaCron.setScheduleCrontab(schedule.getCrontab()); - - int insertFlagCron = processDefinitionService.importProcessSchedule(loginUser, currentProjectName, processMetaCron, - processDefinitionName, processDefinitionId); - Assert.assertEquals(0, insertFlagCron); - - WorkerGroup workerGroup = new WorkerGroup(); - workerGroup.setName("ds-test-workergroup"); - List workerGroups = new ArrayList<>(); - workerGroups.add(workerGroup); - - processMetaCron.setScheduleWorkerGroupName("ds-test"); - int insertFlagWorker = processDefinitionService.importProcessSchedule(loginUser, currentProjectName, processMetaCron, - processDefinitionName, processDefinitionId); - Assert.assertEquals(0, insertFlagWorker); - - int workerNullFlag = processDefinitionService.importProcessSchedule(loginUser, currentProjectName, processMetaCron, - processDefinitionName, processDefinitionId); - Assert.assertEquals(0, workerNullFlag); - - - } - - /** - * import sub process test - */ - @Test - public void testImportSubProcess() { - - User loginUser = new User(); - loginUser.setId(1); - loginUser.setUserType(UserType.ADMIN_USER); - - Project testProject = getProject("test"); - - //Recursive subprocess sub2 process in sub1 process and sub1process in top process - String topProcessJson = "{\"globalParams\":[]," + - "\"tasks\":[{\"type\":\"SHELL\",\"id\":\"tasks-38634\",\"name\":\"shell1\"," + - "\"params\":{\"resourceList\":[],\"localParams\":[],\"rawScript\":\"#!/bin/bash\\necho \\\"shell-1\\\"\"}," + - "\"description\":\"\",\"runFlag\":\"NORMAL\",\"dependence\":{},\"maxRetryTimes\":\"0\"," + - "\"retryInterval\":\"1\",\"timeout\":{\"strategy\":\"\",\"interval\":null,\"enable\":false}," + - "\"taskInstancePriority\":\"MEDIUM\",\"workerGroupId\":-1,\"preTasks\":[]}," + - "{\"type\":\"SUB_PROCESS\",\"id\":\"tasks-44207\",\"name\":\"shell-4\"," + - "\"params\":{\"processDefinitionId\":39},\"description\":\"\",\"runFlag\":\"NORMAL\"," + - "\"dependence\":{},\"timeout\":{\"strategy\":\"\",\"interval\":null,\"enable\":false}," + - "\"taskInstancePriority\":\"MEDIUM\",\"workerGroupId\":-1," + - "\"preTasks\":[\"shell1\"]}],\"tenantId\":1,\"timeout\":0}"; - - String sub1ProcessJson = "{\"globalParams\":[],\"tasks\":[{\"type\":\"SHELL\",\"id\":\"tasks-84090\"," + - "\"name\":\"shell-4\",\"params\":{\"resourceList\":[],\"localParams\":[],\"rawScript\":\"#!/bin/bash\\necho \\\"shell-4\\\"\"}," + - "\"description\":\"\",\"runFlag\":\"NORMAL\",\"dependence\":{},\"maxRetryTimes\":\"0\"," + - "\"retryInterval\":\"1\",\"timeout\":{\"strategy\":\"\",\"interval\":null,\"enable\":false}," + - "\"taskInstancePriority\":\"MEDIUM\",\"workerGroupId\":-1,\"preTasks\":[]},{\"type\":\"SUB_PROCESS\"," + - "\"id\":\"tasks-87364\",\"name\":\"shell-5\"," + - "\"params\":{\"processDefinitionId\":46},\"description\":\"\",\"runFlag\":\"NORMAL\",\"dependence\":{}," + - "\"timeout\":{\"strategy\":\"\",\"interval\":null,\"enable\":false},\"taskInstancePriority\":\"MEDIUM\"," + - "\"workerGroupId\":-1,\"preTasks\":[\"shell-4\"]}],\"tenantId\":1,\"timeout\":0}"; - - String sub2ProcessJson = "{\"globalParams\":[]," + - "\"tasks\":[{\"type\":\"SHELL\",\"id\":\"tasks-52423\",\"name\":\"shell-5\"," + - "\"params\":{\"resourceList\":[],\"localParams\":[],\"rawScript\":\"echo \\\"shell-5\\\"\"},\"description\":\"\"," + - "\"runFlag\":\"NORMAL\",\"dependence\":{},\"maxRetryTimes\":\"0\",\"retryInterval\":\"1\"," + - "\"timeout\":{\"strategy\":\"\",\"interval\":null,\"enable\":false},\"taskInstancePriority\":\"MEDIUM\",\"workerGroupId\":-1," + - "\"preTasks\":[]}],\"tenantId\":1,\"timeout\":0}"; - - - ObjectNode jsonObject = JSONUtils.parseObject(topProcessJson); - ArrayNode jsonArray = (ArrayNode) jsonObject.path("tasks"); - - String originSubJson = jsonArray.toString(); - - Map subProcessIdMap = new HashMap<>(20); - - ProcessDefinition shellDefinition1 = new ProcessDefinition(); - shellDefinition1.setId(39); - shellDefinition1.setName("shell-4"); - shellDefinition1.setProjectId(2); - shellDefinition1.setProcessDefinitionJson(sub1ProcessJson); - - ProcessDefinition shellDefinition2 = new ProcessDefinition(); - shellDefinition2.setId(46); - shellDefinition2.setName("shell-5"); - shellDefinition2.setProjectId(2); - shellDefinition2.setProcessDefinitionJson(sub2ProcessJson); - - Mockito.when(processDefineMapper.queryByDefineId(39)).thenReturn(shellDefinition1); - Mockito.when(processDefineMapper.queryByDefineId(46)).thenReturn(shellDefinition2); - Mockito.when(processDefineMapper.queryByDefineName(testProject.getId(), "shell-5")).thenReturn(null); - Mockito.when(processDefineMapper.queryByDefineName(testProject.getId(), "shell-4")).thenReturn(null); - Mockito.when(processDefineMapper.queryByDefineName(testProject.getId(), "testProject")).thenReturn(shellDefinition2); - - processDefinitionService.importSubProcess(loginUser,testProject, jsonArray, subProcessIdMap); - - String correctSubJson = jsonArray.toString(); - - Assert.assertEquals(originSubJson, correctSubJson); - - } - @Test public void testImportProcessDefinitionById() throws IOException { - String processJson = "[{\"projectName\":\"testProject\",\"processDefinitionName\":\"shell-4\"," + - "\"processDefinitionJson\":\"{\\\"tenantId\\\":1,\\\"globalParams\\\":[]," + - "\\\"tasks\\\":[{\\\"workerGroupId\\\":\\\"default\\\",\\\"description\\\":\\\"\\\",\\\"runFlag\\\":\\\"NORMAL\\\"," + - "\\\"type\\\":\\\"SHELL\\\",\\\"params\\\":{\\\"rawScript\\\":\\\"#!/bin/bash\\\\necho \\\\\\\"shell-4\\\\\\\"\\\"," + - "\\\"localParams\\\":[],\\\"resourceList\\\":[]},\\\"timeout\\\":{\\\"enable\\\":false,\\\"strategy\\\":\\\"\\\"}," + - "\\\"maxRetryTimes\\\":\\\"0\\\",\\\"taskInstancePriority\\\":\\\"MEDIUM\\\",\\\"name\\\":\\\"shell-4\\\"," + - "\\\"dependence\\\":{},\\\"retryInterval\\\":\\\"1\\\",\\\"preTasks\\\":[],\\\"id\\\":\\\"tasks-84090\\\"}," + - "{\\\"taskInstancePriority\\\":\\\"MEDIUM\\\",\\\"name\\\":\\\"shell-5\\\",\\\"workerGroupId\\\":\\\"default\\\\," + - "\\\"description\\\":\\\"\\\",\\\"dependence\\\":{},\\\"preTasks\\\":[\\\"shell-4\\\"],\\\"id\\\":\\\"tasks-87364\\\"," + - "\\\"runFlag\\\":\\\"NORMAL\\\",\\\"type\\\":\\\"SUB_PROCESS\\\",\\\"params\\\":{\\\"processDefinitionId\\\":46}," + - "\\\"timeout\\\":{\\\"enable\\\":false,\\\"strategy\\\":\\\"\\\"}}],\\\"timeout\\\":0}\"," + - "\"processDefinitionDescription\":\"\",\"processDefinitionLocations\":\"{\\\"tasks-84090\\\":{\\\"name\\\":\\\"shell-4\\\"," + - "\\\"targetarr\\\":\\\"\\\",\\\"x\\\":128,\\\"y\\\":114},\\\"tasks-87364\\\":{\\\"name\\\":\\\"shell-5\\\"," + - "\\\"targetarr\\\":\\\"tasks-84090\\\",\\\"x\\\":266,\\\"y\\\":115}}\"," + - "\"processDefinitionConnects\":\"[{\\\"endPointSourceId\\\":\\\"tasks-84090\\\"," + - "\\\"endPointTargetId\\\":\\\"tasks-87364\\\"}]\"}]"; + String processJson = "[\n" + + " {\n" + + " \"projectName\": \"testProject\",\n" + + " \"processDefinitionName\": \"shell-4\",\n" + + " \"processDefinitionJson\": \"{\\\"tenantId\\\":1" + + ",\\\"globalParams\\\":[],\\\"tasks\\\":[{\\\"workerGroupId\\\":\\\"3\\\",\\\"description\\\"" + + ":\\\"\\\",\\\"runFlag\\\":\\\"NORMAL\\\",\\\"type\\\":\\\"SHELL\\\",\\\"params\\\":{\\\"rawScript\\\"" + + ":\\\"#!/bin/bash\\\\necho \\\\\\\"shell-4\\\\\\\"\\\",\\\"localParams\\\":[],\\\"resourceList\\\":[]}" + + ",\\\"timeout\\\":{\\\"enable\\\":false,\\\"strategy\\\":\\\"\\\"},\\\"maxRetryTimes\\\":\\\"0\\\"" + + ",\\\"taskInstancePriority\\\":\\\"MEDIUM\\\",\\\"name\\\":\\\"shell-4\\\",\\\"dependence\\\":{}" + + ",\\\"retryInterval\\\":\\\"1\\\",\\\"preTasks\\\":[],\\\"id\\\":\\\"tasks-84090\\\"}" + + ",{\\\"taskInstancePriority\\\":\\\"MEDIUM\\\",\\\"name\\\":\\\"shell-5\\\",\\\"workerGroupId\\\"" + + ":\\\"3\\\",\\\"description\\\":\\\"\\\",\\\"dependence\\\":{},\\\"preTasks\\\":[\\\"shell-4\\\"]" + + ",\\\"id\\\":\\\"tasks-87364\\\",\\\"runFlag\\\":\\\"NORMAL\\\",\\\"type\\\":\\\"SUB_PROCESS\\\"" + + ",\\\"params\\\":{\\\"processDefinitionId\\\":46},\\\"timeout\\\":{\\\"enable\\\":false" + + ",\\\"strategy\\\":\\\"\\\"}}],\\\"timeout\\\":0}\",\n" + + " \"processDefinitionDescription\": \"\",\n" + + " \"processDefinitionLocations\": \"{\\\"tasks-84090\\\":{\\\"name\\\":\\\"shell-4\\\"" + + ",\\\"targetarr\\\":\\\"\\\",\\\"x\\\":128,\\\"y\\\":114},\\\"tasks-87364\\\":{\\\"name\\\"" + + ":\\\"shell-5\\\",\\\"targetarr\\\":\\\"tasks-84090\\\",\\\"x\\\":266,\\\"y\\\":115}}\",\n" + + " \"processDefinitionConnects\": \"[{\\\"endPointSourceId\\\":\\\"tasks-84090\\\"" + + ",\\\"endPointTargetId\\\":\\\"tasks-87364\\\"}]\"\n" + + " }\n" + + "]"; - String subProcessJson = "{\"globalParams\":[]," + - "\"tasks\":[{\"type\":\"SHELL\",\"id\":\"tasks-52423\",\"name\":\"shell-5\"," + - "\"params\":{\"resourceList\":[],\"localParams\":[],\"rawScript\":\"echo \\\"shell-5\\\"\"},\"description\":\"\"," + - "\"runFlag\":\"NORMAL\",\"dependence\":{},\"maxRetryTimes\":\"0\",\"retryInterval\":\"1\"," + - "\"timeout\":{\"strategy\":\"\",\"interval\":null,\"enable\":false},\"taskInstancePriority\":\"MEDIUM\",\"workerGroupId\":\\\"default\\\\," + - "\"preTasks\":[]}],\"tenantId\":1,\"timeout\":0}"; + String subProcessJson = "{\n" + + " \"globalParams\": [\n" + + " \n" + + " ],\n" + + " \"tasks\": [\n" + + " {\n" + + " \"type\": \"SHELL\",\n" + + " \"id\": \"tasks-52423\",\n" + + " \"name\": \"shell-5\",\n" + + " \"params\": {\n" + + " \"resourceList\": [\n" + + " \n" + + " ],\n" + + " \"localParams\": [\n" + + " \n" + + " ],\n" + + " \"rawScript\": \"echo \\\"shell-5\\\"\"\n" + + " },\n" + + " \"description\": \"\",\n" + + " \"runFlag\": \"NORMAL\",\n" + + " \"dependence\": {\n" + + " \n" + + " },\n" + + " \"maxRetryTimes\": \"0\",\n" + + " \"retryInterval\": \"1\",\n" + + " \"timeout\": {\n" + + " \"strategy\": \"\",\n" + + " \"interval\": null,\n" + + " \"enable\": false\n" + + " },\n" + + " \"taskInstancePriority\": \"MEDIUM\",\n" + + " \"workerGroupId\": \"3\",\n" + + " \"preTasks\": [\n" + + " \n" + + " ]\n" + + " }\n" + + " ],\n" + + " \"tenantId\": 1,\n" + + " \"timeout\": 0\n" + + "}"; - FileUtils.writeStringToFile(new File("/tmp/task.json"),processJson); + FileUtils.writeStringToFile(new File("/tmp/task.json"), processJson); File file = new File("/tmp/task.json"); @@ -694,7 +827,7 @@ public class ProcessDefinitionServiceTest { loginUser.setUserType(UserType.ADMIN_USER); String currentProjectName = "testProject"; - Map result = new HashMap<>(5); + Map result = new HashMap<>(); putMsg(result, Status.SUCCESS, currentProjectName); ProcessDefinition shellDefinition2 = new ProcessDefinition(); @@ -707,96 +840,168 @@ public class ProcessDefinitionServiceTest { Mockito.when(projectService.checkProjectAndAuth(loginUser, getProject(currentProjectName), currentProjectName)).thenReturn(result); Mockito.when(processDefineMapper.queryByDefineId(46)).thenReturn(shellDefinition2); - //import process -// Map importProcessResult = processDefinitionService.importProcessDefinition(loginUser, multipartFile, currentProjectName); -// -// Assert.assertEquals(Status.SUCCESS, importProcessResult.get(Constants.STATUS)); -// -// boolean delete = file.delete(); -// -// Assert.assertTrue(delete); + Map importProcessResult = processDefinitionService.importProcessDefinition(loginUser, multipartFile, currentProjectName); -// String processMetaJson = ""; -// improssProcessCheckData(file, loginUser, currentProjectName, processMetaJson); -// -// processMetaJson = "{\"scheduleWorkerGroupId\":-1}"; -// improssProcessCheckData(file, loginUser, currentProjectName, processMetaJson); -// -// processMetaJson = "{\"scheduleWorkerGroupId\":-1,\"projectName\":\"test\"}"; -// improssProcessCheckData(file, loginUser, currentProjectName, processMetaJson); -// -// processMetaJson = "{\"scheduleWorkerGroupId\":-1,\"projectName\":\"test\",\"processDefinitionName\":\"test_definition\"}"; -// improssProcessCheckData(file, loginUser, currentProjectName, processMetaJson); + Assert.assertEquals(Status.SUCCESS, importProcessResult.get(Constants.STATUS)); + boolean delete = file.delete(); - } - - /** - * check import process metadata - * @param file file - * @param loginUser login user - * @param currentProjectName current project name - * @param processMetaJson process meta json - * @throws IOException IO exception - */ - private void improssProcessCheckData(File file, User loginUser, String currentProjectName, String processMetaJson) throws IOException { - //check null - FileUtils.writeStringToFile(new File("/tmp/task.json"),processMetaJson); - - File fileEmpty = new File("/tmp/task.json"); - - FileInputStream fileEmptyInputStream = new FileInputStream("/tmp/task.json"); - - MultipartFile multiFileEmpty = new MockMultipartFile(fileEmpty.getName(), fileEmpty.getName(), - ContentType.APPLICATION_OCTET_STREAM.toString(), fileEmptyInputStream); - - Map resEmptyProcess = processDefinitionService.importProcessDefinition(loginUser, multiFileEmpty, currentProjectName); - - Assert.assertEquals(Status.DATA_IS_NULL, resEmptyProcess.get(Constants.STATUS)); - - boolean deleteFlag = file.delete(); - - Assert.assertTrue(deleteFlag); + Assert.assertTrue(delete); } @Test - public void testUpdateProcessDefinition () { + public void testUpdateProcessDefinition() { User loginUser = new User(); loginUser.setId(1); loginUser.setUserType(UserType.ADMIN_USER); - Map result = new HashMap<>(5); + Map result = new HashMap<>(); putMsg(result, Status.SUCCESS); String projectName = "project_test1"; Project project = getProject(projectName); + ProcessDefinition processDefinition = getProcessDefinition(); + Mockito.when(projectMapper.queryByName(projectName)).thenReturn(getProject(projectName)); Mockito.when(projectService.checkProjectAndAuth(loginUser, project, projectName)).thenReturn(result); - Mockito.when(processService.findProcessDefineById(1)).thenReturn(getProcessDefinition()); + Mockito.when(processService.findProcessDefineById(1)).thenReturn(processDefinition); + Mockito.when(processDefinitionVersionService.addProcessDefinitionVersion(processDefinition)).thenReturn(1L); + String sqlDependentJson = "{\n" + + " \"globalParams\": [\n" + + " \n" + + " ],\n" + + " \"tasks\": [\n" + + " {\n" + + " \"type\": \"SQL\",\n" + + " \"id\": \"tasks-27297\",\n" + + " \"name\": \"sql\",\n" + + " \"params\": {\n" + + " \"type\": \"MYSQL\",\n" + + " \"datasource\": 1,\n" + + " \"sql\": \"select * from test\",\n" + + " \"udfs\": \"\",\n" + + " \"sqlType\": \"1\",\n" + + " \"title\": \"\",\n" + + " \"receivers\": \"\",\n" + + " \"receiversCc\": \"\",\n" + + " \"showType\": \"TABLE\",\n" + + " \"localParams\": [\n" + + " \n" + + " ],\n" + + " \"connParams\": \"\",\n" + + " \"preStatements\": [\n" + + " \n" + + " ],\n" + + " \"postStatements\": [\n" + + " \n" + + " ]\n" + + " },\n" + + " \"description\": \"\",\n" + + " \"runFlag\": \"NORMAL\",\n" + + " \"dependence\": {\n" + + " \n" + + " },\n" + + " \"maxRetryTimes\": \"0\",\n" + + " \"retryInterval\": \"1\",\n" + + " \"timeout\": {\n" + + " \"strategy\": \"\",\n" + + " \"enable\": false\n" + + " },\n" + + " \"taskInstancePriority\": \"MEDIUM\",\n" + + " \"workerGroupId\": -1,\n" + + " \"preTasks\": [\n" + + " \"dependent\"\n" + + " ]\n" + + " },\n" + + " {\n" + + " \"type\": \"DEPENDENT\",\n" + + " \"id\": \"tasks-33787\",\n" + + " \"name\": \"dependent\",\n" + + " \"params\": {\n" + + " \n" + + " },\n" + + " \"description\": \"\",\n" + + " \"runFlag\": \"NORMAL\",\n" + + " \"dependence\": {\n" + + " \"relation\": \"AND\",\n" + + " \"dependTaskList\": [\n" + + " {\n" + + " \"relation\": \"AND\",\n" + + " \"dependItemList\": [\n" + + " {\n" + + " \"projectId\": 2,\n" + + " \"definitionId\": 46,\n" + + " \"depTasks\": \"ALL\",\n" + + " \"cycle\": \"day\",\n" + + " \"dateValue\": \"today\"\n" + + " }\n" + + " ]\n" + + " }\n" + + " ]\n" + + " },\n" + + " \"maxRetryTimes\": \"0\",\n" + + " \"retryInterval\": \"1\",\n" + + " \"timeout\": {\n" + + " \"strategy\": \"\",\n" + + " \"enable\": false\n" + + " },\n" + + " \"taskInstancePriority\": \"MEDIUM\",\n" + + " \"workerGroupId\": -1,\n" + + " \"preTasks\": [\n" + + " \n" + + " ]\n" + + " }\n" + + " ],\n" + + " \"tenantId\": 1,\n" + + " \"timeout\": 0\n" + + "}"; Map updateResult = processDefinitionService.updateProcessDefinition(loginUser, projectName, 1, "test", sqlDependentJson, "", "", ""); Assert.assertEquals(Status.UPDATE_PROCESS_DEFINITION_ERROR, updateResult.get(Constants.STATUS)); } + @Test + public void testBatchExportProcessDefinitionByIds() { + processDefinitionService.batchExportProcessDefinitionByIds( + null, null, null, null); + + User loginUser = new User(); + loginUser.setId(1); + loginUser.setUserType(UserType.ADMIN_USER); + + String projectName = "project_test1"; + Project project = getProject(projectName); + + Map result = new HashMap<>(); + putMsg(result, Status.PROJECT_NOT_FOUNT); + Mockito.when(projectMapper.queryByName(projectName)).thenReturn(getProject(projectName)); + Mockito.when(projectService.checkProjectAndAuth(loginUser, project, projectName)).thenReturn(result); + + processDefinitionService.batchExportProcessDefinitionByIds( + loginUser, projectName, "1", null); + } + /** * get mock datasource + * * @return DataSource */ - private DataSource getDataSource(){ + private DataSource getDataSource() { DataSource dataSource = new DataSource(); dataSource.setId(2); dataSource.setName("test"); - return dataSource; + return dataSource; } /** * get mock processDefinition + * * @return ProcessDefinition */ - private ProcessDefinition getProcessDefinition(){ + private ProcessDefinition getProcessDefinition() { ProcessDefinition processDefinition = new ProcessDefinition(); processDefinition.setId(46); @@ -805,24 +1010,40 @@ public class ProcessDefinitionServiceTest { processDefinition.setTenantId(1); processDefinition.setDescription(""); - return processDefinition; + return processDefinition; } /** * get mock Project + * * @param projectName projectName * @return Project */ - private Project getProject(String projectName){ + private Project getProject(String projectName) { Project project = new Project(); project.setId(1); project.setName(projectName); project.setUserId(1); - return project; + return project; + } + + /** + * get mock Project + * + * @param projectId projectId + * @return Project + */ + private Project getProjectById(int projectId) { + Project project = new Project(); + project.setId(projectId); + project.setName("project_test2"); + project.setUserId(1); + return project; } /** * get mock schedule + * * @return schedule */ private Schedule getSchedule() { @@ -845,6 +1066,7 @@ public class ProcessDefinitionServiceTest { /** * get mock processMeta + * * @return processMeta */ private ProcessMeta getProcessMeta() { @@ -876,4 +1098,4 @@ public class ProcessDefinitionServiceTest { result.put(Constants.MSG, status.getMsg()); } } -} \ No newline at end of file +} diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionVersionServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionVersionServiceTest.java new file mode 100644 index 0000000000..169ef2bcfe --- /dev/null +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionVersionServiceTest.java @@ -0,0 +1,274 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.service; + +import org.apache.dolphinscheduler.api.enums.Status; +import org.apache.dolphinscheduler.api.service.impl.ProcessDefinitionVersionServiceImpl; +import org.apache.dolphinscheduler.api.service.impl.ProjectServiceImpl; +import org.apache.dolphinscheduler.api.utils.PageInfo; +import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.enums.UserType; +import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; +import org.apache.dolphinscheduler.dao.entity.ProcessDefinitionVersion; +import org.apache.dolphinscheduler.dao.entity.Project; +import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionVersionMapper; +import org.apache.dolphinscheduler.dao.mapper.ProjectMapper; + +import java.text.MessageFormat; +import java.util.HashMap; +import java.util.Map; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import com.google.common.collect.Lists; + +@RunWith(MockitoJUnitRunner.class) +public class ProcessDefinitionVersionServiceTest { + + @InjectMocks + private ProcessDefinitionVersionServiceImpl processDefinitionVersionService; + + @Mock + private ProcessDefinitionVersionMapper processDefinitionVersionMapper; + + @Mock + private ProjectMapper projectMapper; + + @Mock + private ProjectServiceImpl projectService; + + @Test + public void testAddProcessDefinitionVersion() { + long expectedVersion = 5L; + ProcessDefinition processDefinition = getProcessDefinition(); + Mockito.when(processDefinitionVersionMapper + .queryMaxVersionByProcessDefinitionId(processDefinition.getId())) + .thenReturn(expectedVersion); + + long version = processDefinitionVersionService.addProcessDefinitionVersion(processDefinition); + + Assert.assertEquals(expectedVersion + 1, version); + } + + @Test + @SuppressWarnings("unchecked") + public void testQueryProcessDefinitionVersions() { + // pageNo <= 0 + int pageNo = -1; + int pageSize = 10; + int processDefinitionId = 66; + + String projectName = "project_test1"; + User loginUser = new User(); + loginUser.setId(-1); + loginUser.setUserType(UserType.GENERAL_USER); + Map resultMap1 = processDefinitionVersionService.queryProcessDefinitionVersions( + loginUser + , projectName + , pageNo + , pageSize + , processDefinitionId); + Assert.assertEquals(Status.QUERY_PROCESS_DEFINITION_VERSIONS_PAGE_NO_OR_PAGE_SIZE_LESS_THAN_1_ERROR + , resultMap1.get(Constants.STATUS)); + + // pageSize <= 0 + pageNo = 1; + pageSize = -1; + Map resultMap2 = processDefinitionVersionService.queryProcessDefinitionVersions( + loginUser + , projectName + , pageNo + , pageSize + , processDefinitionId); + Assert.assertEquals(Status.QUERY_PROCESS_DEFINITION_VERSIONS_PAGE_NO_OR_PAGE_SIZE_LESS_THAN_1_ERROR + , resultMap2.get(Constants.STATUS)); + + Map res = new HashMap<>(); + putMsg(res, Status.PROJECT_NOT_FOUNT); + Project project = getProject(projectName); + Mockito.when(projectMapper.queryByName(projectName)) + .thenReturn(project); + Mockito.when(projectService.checkProjectAndAuth(loginUser, project, projectName)) + .thenReturn(res); + + // project auth fail + pageNo = 1; + pageSize = 10; + Map resultMap3 = processDefinitionVersionService.queryProcessDefinitionVersions( + loginUser + , projectName + , pageNo + , pageSize + , processDefinitionId); + Assert.assertEquals(Status.PROJECT_NOT_FOUNT, resultMap3.get(Constants.STATUS)); + + putMsg(res, Status.SUCCESS); + + Mockito.when(projectService.checkProjectAndAuth(loginUser, project, projectName)) + .thenReturn(res); + + ProcessDefinitionVersion processDefinitionVersion = getProcessDefinitionVersion(getProcessDefinition()); + + Mockito.when(processDefinitionVersionMapper + .queryProcessDefinitionVersionsPaging(Mockito.any(Page.class), Mockito.eq(processDefinitionId))) + .thenReturn(new Page() + .setRecords(Lists.newArrayList(processDefinitionVersion))); + + Map resultMap4 = processDefinitionVersionService.queryProcessDefinitionVersions( + loginUser + , projectName + , pageNo + , pageSize + , processDefinitionId); + Assert.assertEquals(Status.SUCCESS, resultMap4.get(Constants.STATUS)); + Assert.assertEquals(processDefinitionVersion + , ((PageInfo) resultMap4.get(Constants.DATA_LIST)) + .getLists().get(0)); + } + + @Test + public void testQueryByProcessDefinitionIdAndVersion() { + + ProcessDefinitionVersion expectedProcessDefinitionVersion = + getProcessDefinitionVersion(getProcessDefinition()); + + int processDefinitionId = 66; + long version = 10; + Mockito.when(processDefinitionVersionMapper.queryByProcessDefinitionIdAndVersion(processDefinitionId, version)) + .thenReturn(expectedProcessDefinitionVersion); + + ProcessDefinitionVersion processDefinitionVersion = processDefinitionVersionService + .queryByProcessDefinitionIdAndVersion(processDefinitionId, version); + + Assert.assertEquals(expectedProcessDefinitionVersion, processDefinitionVersion); + } + + @Test + public void testDeleteByProcessDefinitionIdAndVersion() { + String projectName = "project_test1"; + int processDefinitionId = 66; + long version = 10; + Project project = getProject(projectName); + Mockito.when(projectMapper.queryByName(projectName)) + .thenReturn(project); + + User loginUser = new User(); + loginUser.setId(-1); + loginUser.setUserType(UserType.GENERAL_USER); + + // project auth fail + Mockito.when(projectService.checkProjectAndAuth(loginUser, project, projectName)) + .thenReturn(new HashMap<>()); + + Map resultMap1 = processDefinitionVersionService.deleteByProcessDefinitionIdAndVersion( + loginUser + , projectName + , processDefinitionId + , version); + + Assert.assertEquals(0, resultMap1.size()); + + Map res = new HashMap<>(); + putMsg(res, Status.SUCCESS); + + Mockito.when(processDefinitionVersionMapper.deleteByProcessDefinitionIdAndVersion(processDefinitionId, version)) + .thenReturn(1); + Mockito.when(projectService.checkProjectAndAuth(loginUser, project, projectName)) + .thenReturn(res); + + Map resultMap2 = processDefinitionVersionService.deleteByProcessDefinitionIdAndVersion( + loginUser + , projectName + , processDefinitionId + , version); + + Assert.assertEquals(Status.SUCCESS, resultMap2.get(Constants.STATUS)); + + } + + /** + * get mock processDefinitionVersion by processDefinition + * + * @return processDefinitionVersion + */ + private ProcessDefinitionVersion getProcessDefinitionVersion(ProcessDefinition processDefinition) { + return ProcessDefinitionVersion + .newBuilder() + .processDefinitionId(processDefinition.getId()) + .version(1) + .processDefinitionJson(processDefinition.getProcessDefinitionJson()) + .description(processDefinition.getDescription()) + .locations(processDefinition.getLocations()) + .connects(processDefinition.getConnects()) + .timeout(processDefinition.getTimeout()) + .globalParams(processDefinition.getGlobalParams()) + .createTime(processDefinition.getUpdateTime()) + .receivers(processDefinition.getReceivers()) + .receiversCc(processDefinition.getReceiversCc()) + .resourceIds(processDefinition.getResourceIds()) + .build(); + } + + /** + * get mock processDefinition + * + * @return ProcessDefinition + */ + private ProcessDefinition getProcessDefinition() { + + ProcessDefinition processDefinition = new ProcessDefinition(); + processDefinition.setId(66); + processDefinition.setName("test_pdf"); + processDefinition.setProjectId(2); + processDefinition.setTenantId(1); + processDefinition.setDescription(""); + + return processDefinition; + } + + /** + * get mock Project + * + * @param projectName projectName + * @return Project + */ + private Project getProject(String projectName) { + Project project = new Project(); + project.setId(1); + project.setName(projectName); + project.setUserId(1); + return project; + } + + private void putMsg(Map result, Status status, Object... statusParams) { + result.put(Constants.STATUS, status); + if (statusParams != null && statusParams.length > 0) { + result.put(Constants.MSG, MessageFormat.format(status.getMsg(), statusParams)); + } else { + result.put(Constants.MSG, status.getMsg()); + } + } +} \ No newline at end of file diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessInstanceServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessInstanceServiceTest.java index 651964bb16..5511f69aeb 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessInstanceServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessInstanceServiceTest.java @@ -14,18 +14,46 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.api.service; -import com.baomidou.mybatisplus.extension.plugins.pagination.Page; -import org.apache.dolphinscheduler.api.ApiApplicationServer; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.when; + import org.apache.dolphinscheduler.api.enums.Status; +import org.apache.dolphinscheduler.api.service.impl.LoggerServiceImpl; +import org.apache.dolphinscheduler.api.service.impl.ProjectServiceImpl; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.common.enums.*; +import org.apache.dolphinscheduler.common.enums.CommandType; +import org.apache.dolphinscheduler.common.enums.DependResult; +import org.apache.dolphinscheduler.common.enums.ExecutionStatus; +import org.apache.dolphinscheduler.common.enums.Flag; +import org.apache.dolphinscheduler.common.enums.TaskType; +import org.apache.dolphinscheduler.common.enums.UserType; import org.apache.dolphinscheduler.common.utils.DateUtils; -import org.apache.dolphinscheduler.dao.entity.*; -import org.apache.dolphinscheduler.dao.mapper.*; +import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; +import org.apache.dolphinscheduler.dao.entity.ProcessInstance; +import org.apache.dolphinscheduler.dao.entity.Project; +import org.apache.dolphinscheduler.dao.entity.TaskInstance; +import org.apache.dolphinscheduler.dao.entity.Tenant; +import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.dao.entity.WorkerGroup; +import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper; +import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper; +import org.apache.dolphinscheduler.dao.mapper.ProjectMapper; +import org.apache.dolphinscheduler.dao.mapper.TaskInstanceMapper; import org.apache.dolphinscheduler.service.process.ProcessService; + +import java.io.IOException; +import java.text.MessageFormat; +import java.text.ParseException; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; @@ -33,22 +61,11 @@ import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.Mockito; import org.mockito.junit.MockitoJUnitRunner; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.boot.test.context.SpringBootTest; -import java.io.IOException; -import java.text.MessageFormat; -import java.text.ParseException; -import java.util.*; - -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.when; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; @RunWith(MockitoJUnitRunner.Silent.class) -@SpringBootTest(classes = ApiApplicationServer.class) public class ProcessInstanceServiceTest { - private static final Logger logger = LoggerFactory.getLogger(ProcessInstanceServiceTest.class); @InjectMocks ProcessInstanceService processInstanceService; @@ -57,7 +74,7 @@ public class ProcessInstanceServiceTest { ProjectMapper projectMapper; @Mock - ProjectService projectService; + ProjectServiceImpl projectService; @Mock ProcessService processService; @@ -71,6 +88,9 @@ public class ProcessInstanceServiceTest { @Mock ProcessDefinitionService processDefinitionService; + @Mock + ProcessDefinitionVersionService processDefinitionVersionService; + @Mock ExecutorService execService; @@ -78,25 +98,22 @@ public class ProcessInstanceServiceTest { TaskInstanceMapper taskInstanceMapper; @Mock - LoggerService loggerService; - - + LoggerServiceImpl loggerService; @Mock UsersService usersService; - private String shellJson = "{\"globalParams\":[],\"tasks\":[{\"type\":\"SHELL\",\"id\":\"tasks-9527\",\"name\":\"shell-1\"," + - "\"params\":{\"resourceList\":[],\"localParams\":[],\"rawScript\":\"#!/bin/bash\\necho \\\"shell-1\\\"\"}," + - "\"description\":\"\",\"runFlag\":\"NORMAL\",\"dependence\":{},\"maxRetryTimes\":\"0\",\"retryInterval\":\"1\"," + - "\"timeout\":{\"strategy\":\"\",\"interval\":1,\"enable\":false},\"taskInstancePriority\":\"MEDIUM\"," + - "\"workerGroupId\":-1,\"preTasks\":[]}],\"tenantId\":1,\"timeout\":0}"; - + private String shellJson = "{\"globalParams\":[],\"tasks\":[{\"type\":\"SHELL\",\"id\":\"tasks-9527\",\"name\":\"shell-1\"," + + "\"params\":{\"resourceList\":[],\"localParams\":[],\"rawScript\":\"#!/bin/bash\\necho \\\"shell-1\\\"\"}," + + "\"description\":\"\",\"runFlag\":\"NORMAL\",\"dependence\":{},\"maxRetryTimes\":\"0\",\"retryInterval\":\"1\"," + + "\"timeout\":{\"strategy\":\"\",\"interval\":1,\"enable\":false},\"taskInstancePriority\":\"MEDIUM\"," + + "\"workerGroupId\":-1,\"preTasks\":[]}],\"tenantId\":1,\"timeout\":0}"; @Test public void testQueryProcessInstanceList() { String projectName = "project_test1"; User loginUser = getAdminUser(); - Map result = new HashMap<>(5); + Map result = new HashMap<>(); putMsg(result, Status.PROJECT_NOT_FOUNT, projectName); //project auth fail @@ -153,30 +170,28 @@ public class ProcessInstanceServiceTest { User loginUser = getAdminUser(); Map result = new HashMap<>(5); putMsg(result, Status.PROJECT_NOT_FOUNT, projectName); - int size=10; - String startTime="2020-01-01 00:00:00"; - String endTime="2020-08-02 00:00:00"; + int size = 10; + String startTime = "2020-01-01 00:00:00"; + String endTime = "2020-08-02 00:00:00"; Date start = DateUtils.getScheduleDate(startTime); Date end = DateUtils.getScheduleDate(endTime); //project auth fail when(projectMapper.queryByName(projectName)).thenReturn(null); when(projectService.checkProjectAndAuth(loginUser, null, projectName)).thenReturn(result); - Map proejctAuthFailRes = processInstanceService.queryTopNLongestRunningProcessInstance(loginUser,projectName,size,startTime,endTime); + Map proejctAuthFailRes = processInstanceService.queryTopNLongestRunningProcessInstance(loginUser, projectName, size, startTime, endTime); Assert.assertEquals(Status.PROJECT_NOT_FOUNT, proejctAuthFailRes.get(Constants.STATUS)); //project auth success putMsg(result, Status.SUCCESS, projectName); Project project = getProject(projectName); ProcessInstance processInstance = getProcessInstance(); - List processInstanceList = new ArrayList<>(); - processInstanceList.add(processInstance); when(projectMapper.queryByName(projectName)).thenReturn(project); when(projectService.checkProjectAndAuth(loginUser, project, projectName)).thenReturn(result); when(usersService.queryUser(loginUser.getId())).thenReturn(loginUser); when(usersService.getUserIdByName(loginUser.getUserName())).thenReturn(loginUser.getId()); when(usersService.queryUser(processInstance.getExecutorId())).thenReturn(loginUser); - Map successRes = processInstanceService.queryTopNLongestRunningProcessInstance(loginUser,projectName,size,startTime,endTime); + Map successRes = processInstanceService.queryTopNLongestRunningProcessInstance(loginUser, projectName, size, startTime, endTime); Assert.assertEquals(Status.SUCCESS, successRes.get(Constants.STATUS)); } @@ -185,7 +200,7 @@ public class ProcessInstanceServiceTest { public void testQueryProcessInstanceById() { String projectName = "project_test1"; User loginUser = getAdminUser(); - Map result = new HashMap<>(5); + Map result = new HashMap<>(); putMsg(result, Status.PROJECT_NOT_FOUNT, projectName); //project auth fail @@ -223,7 +238,7 @@ public class ProcessInstanceServiceTest { public void testQueryTaskListByProcessId() throws IOException { String projectName = "project_test1"; User loginUser = getAdminUser(); - Map result = new HashMap<>(5); + Map result = new HashMap<>(); putMsg(result, Status.PROJECT_NOT_FOUNT, projectName); //project auth fail @@ -253,26 +268,23 @@ public class ProcessInstanceServiceTest { Assert.assertEquals(Status.SUCCESS, successRes.get(Constants.STATUS)); } - @Test - public void testParseLogForDependentResult() { - String logString = "[INFO] 2019-03-19 17:11:08.475 org.apache.dolphinscheduler.server.worker.log.TaskLogger:[172] - [taskAppId=TASK_223_10739_452334] dependent item complete :|| 223-ALL-day-last1Day,SUCCESS\n" + - "[INFO] 2019-03-19 17:11:08.476 org.apache.dolphinscheduler.server.worker.runner.TaskScheduleThread:[172] - task : 223_10739_452334 exit status code : 0\n" + - "[root@node2 current]# "; - try { - Map resultMap = - processInstanceService.parseLogForDependentResult(logString); - Assert.assertEquals(1, resultMap.size()); - } catch (IOException e) { - - } + public void testParseLogForDependentResult() throws IOException { + String logString = "[INFO] 2019-03-19 17:11:08.475 org.apache.dolphinscheduler.server.worker.log.TaskLogger:[172]" + + " - [taskAppId=TASK_223_10739_452334] dependent item complete :|| 223-ALL-day-last1Day,SUCCESS\n" + + "[INFO] 2019-03-19 17:11:08.476 org.apache.dolphinscheduler.server.worker.runner.TaskScheduleThread:[172]" + + " - task : 223_10739_452334 exit status code : 0\n" + + "[root@node2 current]# "; + Map resultMap = + processInstanceService.parseLogForDependentResult(logString); + Assert.assertEquals(1, resultMap.size()); } @Test public void testQuerySubProcessInstanceByTaskId() { String projectName = "project_test1"; User loginUser = getAdminUser(); - Map result = new HashMap<>(5); + Map result = new HashMap<>(); putMsg(result, Status.PROJECT_NOT_FOUNT, projectName); //project auth fail @@ -318,7 +330,7 @@ public class ProcessInstanceServiceTest { public void testUpdateProcessInstance() throws ParseException { String projectName = "project_test1"; User loginUser = getAdminUser(); - Map result = new HashMap<>(5); + Map result = new HashMap<>(); putMsg(result, Status.PROJECT_NOT_FOUNT, projectName); //project auth fail @@ -359,6 +371,7 @@ public class ProcessInstanceServiceTest { when(processService.getTenantForProcess(Mockito.anyInt(), Mockito.anyInt())).thenReturn(tenant); when(processService.updateProcessInstance(processInstance)).thenReturn(1); when(processDefinitionService.checkProcessNodeList(Mockito.any(), eq(shellJson))).thenReturn(result); + when(processDefinitionVersionService.addProcessDefinitionVersion(processDefinition)).thenReturn(1L); Map processInstanceFinishRes = processInstanceService.updateProcessInstance(loginUser, projectName, 1, shellJson, "2020-02-21 00:00:00", true, Flag.YES, "", ""); Assert.assertEquals(Status.UPDATE_PROCESS_INSTANCE_ERROR, processInstanceFinishRes.get(Constants.STATUS)); @@ -374,7 +387,7 @@ public class ProcessInstanceServiceTest { public void testQueryParentInstanceBySubId() { String projectName = "project_test1"; User loginUser = getAdminUser(); - Map result = new HashMap<>(5); + Map result = new HashMap<>(); putMsg(result, Status.PROJECT_NOT_FOUNT, projectName); //project auth fail @@ -389,6 +402,7 @@ public class ProcessInstanceServiceTest { when(projectMapper.queryByName(projectName)).thenReturn(project); when(projectService.checkProjectAndAuth(loginUser, project, projectName)).thenReturn(result); when(processService.findProcessInstanceDetailById(1)).thenReturn(null); + when(projectService.checkProjectAndAuth(loginUser, project, projectName)).thenReturn(result); Map processInstanceNullRes = processInstanceService.queryParentInstanceBySubId(loginUser, projectName, 1); Assert.assertEquals(Status.PROCESS_INSTANCE_NOT_EXIST, processInstanceNullRes.get(Constants.STATUS)); @@ -415,7 +429,7 @@ public class ProcessInstanceServiceTest { public void testDeleteProcessInstanceById() { String projectName = "project_test1"; User loginUser = getAdminUser(); - Map result = new HashMap<>(5); + Map result = new HashMap<>(); putMsg(result, Status.PROJECT_NOT_FOUNT, projectName); //project auth fail @@ -547,5 +561,4 @@ public class ProcessInstanceServiceTest { } } - } \ No newline at end of file diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProjectServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProjectServiceTest.java index 51f9e148d1..85b23b3fcb 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProjectServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProjectServiceTest.java @@ -14,11 +14,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.api.service; -import com.baomidou.mybatisplus.core.metadata.IPage; -import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import org.apache.dolphinscheduler.api.enums.Status; +import org.apache.dolphinscheduler.api.service.impl.ProjectServiceImpl; import org.apache.dolphinscheduler.api.utils.PageInfo; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.UserType; @@ -30,10 +30,13 @@ import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper; import org.apache.dolphinscheduler.dao.mapper.ProjectMapper; import org.apache.dolphinscheduler.dao.mapper.ProjectUserMapper; -import org.apache.dolphinscheduler.dao.mapper.UserMapper; -import org.junit.After; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + import org.junit.Assert; -import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.InjectMocks; @@ -43,141 +46,144 @@ import org.mockito.junit.MockitoJUnitRunner; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; @RunWith(MockitoJUnitRunner.class) public class ProjectServiceTest { - private static final Logger logger = LoggerFactory.getLogger(ProjectServiceTest.class); @InjectMocks - private ProjectService projectService; + private ProjectServiceImpl projectService; + @Mock private ProjectMapper projectMapper; - @Mock - private UserMapper userMapper; + @Mock private ProjectUserMapper projectUserMapper; + @Mock private ProcessDefinitionMapper processDefinitionMapper; - private String projectName = "ProjectServiceTest"; private String userName = "ProjectServiceTest"; - @Before - public void setUp() { - - } - - - @After - public void after(){ - - } - @Test - public void testCreateProject(){ + public void testCreateProject() { - User loginUser = getLoginUser(); + User loginUser = getLoginUser(); loginUser.setId(1); Map result = projectService.createProject(loginUser, projectName, getDesc()); logger.info(result.toString()); - Assert.assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR,result.get(Constants.STATUS)); + Assert.assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR, result.get(Constants.STATUS)); //project name exist Mockito.when(projectMapper.queryByName(projectName)).thenReturn(getProject()); result = projectService.createProject(loginUser, projectName, projectName); logger.info(result.toString()); - Assert.assertEquals(Status.PROJECT_ALREADY_EXISTS,result.get(Constants.STATUS)); + Assert.assertEquals(Status.PROJECT_ALREADY_EXISTS, result.get(Constants.STATUS)); //success Mockito.when(projectMapper.insert(Mockito.any(Project.class))).thenReturn(1); result = projectService.createProject(loginUser, "test", "test"); logger.info(result.toString()); - Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); - + Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); } + @Test - public void testQueryById(){ + public void testQueryById() { //not exist Map result = projectService.queryById(Integer.MAX_VALUE); - Assert.assertEquals(Status.PROJECT_NOT_FOUNT,result.get(Constants.STATUS)); + Assert.assertEquals(Status.PROJECT_NOT_FOUNT, result.get(Constants.STATUS)); logger.info(result.toString()); //success Mockito.when(projectMapper.selectById(1)).thenReturn(getProject()); result = projectService.queryById(1); logger.info(result.toString()); - Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); } + @Test - public void testCheckProjectAndAuth(){ + public void testCheckProjectAndAuth() { Mockito.when(projectUserMapper.queryProjectRelation(1, 1)).thenReturn(getProjectUser()); User loginUser = getLoginUser(); - Map result = projectService.checkProjectAndAuth(loginUser,null,projectName); + Map result = projectService.checkProjectAndAuth(loginUser, null, projectName); logger.info(result.toString()); - Status status = (Status)result.get(Constants.STATUS); - Assert.assertEquals(Status.PROJECT_NOT_FOUNT,result.get(Constants.STATUS)); + Status status = (Status) result.get(Constants.STATUS); + Assert.assertEquals(Status.PROJECT_NOT_FOUNT, result.get(Constants.STATUS)); Project project = getProject(); //USER_NO_OPERATION_PROJECT_PERM project.setUserId(2); - result = projectService.checkProjectAndAuth(loginUser,project,projectName); + result = projectService.checkProjectAndAuth(loginUser, project, projectName); logger.info(result.toString()); - Assert.assertEquals(Status.USER_NO_OPERATION_PROJECT_PERM,result.get(Constants.STATUS)); + Assert.assertEquals(Status.USER_NO_OPERATION_PROJECT_PERM, result.get(Constants.STATUS)); //success project.setUserId(1); - result = projectService.checkProjectAndAuth(loginUser,project,projectName); + result = projectService.checkProjectAndAuth(loginUser, project, projectName); logger.info(result.toString()); - Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); + + Map result2 = new HashMap<>(); + + result2 = projectService.checkProjectAndAuth(loginUser, null, projectName); + Assert.assertEquals(Status.PROJECT_NOT_FOUNT, result2.get(Constants.STATUS)); + + Project project1 = getProject(); + // USER_NO_OPERATION_PROJECT_PERM + project1.setUserId(2); + result2 = projectService.checkProjectAndAuth(loginUser, project1, projectName); + Assert.assertEquals(Status.USER_NO_OPERATION_PROJECT_PERM, result2.get(Constants.STATUS)); + + //success + project1.setUserId(1); + projectService.checkProjectAndAuth(loginUser, project1, projectName); } @Test - public void testHasProjectAndPerm(){ + public void testHasProjectAndPerm() { - // Mockito.when(projectUserMapper.queryProjectRelation(1, 1)).thenReturn(getProjectUser()); + // Mockito.when(projectUserMapper.queryProjectRelation(1, 1)).thenReturn(getProjectUser()); User loginUser = getLoginUser(); Project project = getProject(); Map result = new HashMap<>(); // not exist user User tempUser = new User(); tempUser.setId(Integer.MAX_VALUE); - boolean checkResult = projectService.hasProjectAndPerm(tempUser,project,result); + boolean checkResult = projectService.hasProjectAndPerm(tempUser, project, result); logger.info(result.toString()); Assert.assertFalse(checkResult); //success result = new HashMap<>(); project.setUserId(1); - checkResult = projectService.hasProjectAndPerm(loginUser,project,result); + checkResult = projectService.hasProjectAndPerm(loginUser, project, result); logger.info(result.toString()); Assert.assertTrue(checkResult); } - @Test - public void testQueryProjectListPaging(){ - IPage page = new Page<>(1,10); + @Test + public void testQueryProjectListPaging() { + + IPage page = new Page<>(1, 10); page.setRecords(getList()); page.setTotal(1L); Mockito.when(projectMapper.queryProjectListPaging(Mockito.any(Page.class), Mockito.eq(1), Mockito.eq(projectName))).thenReturn(page); User loginUser = getLoginUser(); // project owner - Map result = projectService.queryProjectListPaging(loginUser,10,1,projectName); + Map result = projectService.queryProjectListPaging(loginUser, 10, 1, projectName); logger.info(result.toString()); PageInfo pageInfo = (PageInfo) result.get(Constants.DATA_LIST); Assert.assertTrue(CollectionUtils.isNotEmpty(pageInfo.getLists())); @@ -185,156 +191,175 @@ public class ProjectServiceTest { //admin Mockito.when(projectMapper.queryProjectListPaging(Mockito.any(Page.class), Mockito.eq(0), Mockito.eq(projectName))).thenReturn(page); loginUser.setUserType(UserType.ADMIN_USER); - result = projectService.queryProjectListPaging(loginUser,10,1,projectName); + result = projectService.queryProjectListPaging(loginUser, 10, 1, projectName); logger.info(result.toString()); pageInfo = (PageInfo) result.get(Constants.DATA_LIST); Assert.assertTrue(CollectionUtils.isNotEmpty(pageInfo.getLists())); } + @Test - public void testDeleteProject(){ + public void testDeleteProject() { Mockito.when(projectMapper.selectById(1)).thenReturn(getProject()); User loginUser = getLoginUser(); //PROJECT_NOT_FOUNT - Map result= projectService.deleteProject(loginUser,12); + Map result = projectService.deleteProject(loginUser, 12); logger.info(result.toString()); - Assert.assertEquals(Status.PROJECT_NOT_FOUNT,result.get(Constants.STATUS)); + Assert.assertEquals(Status.PROJECT_NOT_FOUNT, result.get(Constants.STATUS)); loginUser.setId(2); //USER_NO_OPERATION_PROJECT_PERM - result= projectService.deleteProject(loginUser,1); + result = projectService.deleteProject(loginUser, 1); logger.info(result.toString()); - Assert.assertEquals(Status.USER_NO_OPERATION_PROJECT_PERM,result.get(Constants.STATUS)); + Assert.assertEquals(Status.USER_NO_OPERATION_PROJECT_PERM, result.get(Constants.STATUS)); //DELETE_PROJECT_ERROR_DEFINES_NOT_NULL Mockito.when(processDefinitionMapper.queryAllDefinitionList(1)).thenReturn(getProcessDefinitions()); loginUser.setUserType(UserType.ADMIN_USER); - result= projectService.deleteProject(loginUser,1); + result = projectService.deleteProject(loginUser, 1); logger.info(result.toString()); - Assert.assertEquals(Status.DELETE_PROJECT_ERROR_DEFINES_NOT_NULL,result.get(Constants.STATUS)); + Assert.assertEquals(Status.DELETE_PROJECT_ERROR_DEFINES_NOT_NULL, result.get(Constants.STATUS)); //success Mockito.when(projectMapper.deleteById(1)).thenReturn(1); Mockito.when(processDefinitionMapper.queryAllDefinitionList(1)).thenReturn(new ArrayList<>()); - result= projectService.deleteProject(loginUser,1); + result = projectService.deleteProject(loginUser, 1); logger.info(result.toString()); - Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); - + Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); } @Test - public void testUpdate(){ + public void testUpdate() { User loginUser = getLoginUser(); Project project = getProject(); project.setId(2); Mockito.when(projectMapper.queryByName(projectName)).thenReturn(project); - Mockito.when( projectMapper.selectById(1)).thenReturn(getProject()); + Mockito.when(projectMapper.selectById(1)).thenReturn(getProject()); // PROJECT_NOT_FOUNT - Map result = projectService.update(loginUser,12,projectName,"desc"); + Map result = projectService.update(loginUser, 12, projectName, "desc"); logger.info(result.toString()); - Assert.assertEquals(Status.PROJECT_NOT_FOUNT,result.get(Constants.STATUS)); + Assert.assertEquals(Status.PROJECT_NOT_FOUNT, result.get(Constants.STATUS)); //PROJECT_ALREADY_EXISTS - result = projectService.update(loginUser,1,projectName,"desc"); + result = projectService.update(loginUser, 1, projectName, "desc"); logger.info(result.toString()); - Assert.assertEquals(Status.PROJECT_ALREADY_EXISTS,result.get(Constants.STATUS)); + Assert.assertEquals(Status.PROJECT_ALREADY_EXISTS, result.get(Constants.STATUS)); //success project.setUserId(1); Mockito.when(projectMapper.updateById(Mockito.any(Project.class))).thenReturn(1); - result = projectService.update(loginUser,1,"test","desc"); + result = projectService.update(loginUser, 1, "test", "desc"); logger.info(result.toString()); - Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); } + @Test - public void testQueryAuthorizedProject(){ + public void testQueryAuthorizedProject() { User loginUser = getLoginUser(); Mockito.when(projectMapper.queryAuthedProjectListByUserId(1)).thenReturn(getList()); //USER_NO_OPERATION_PERM - Map result = projectService.queryAuthorizedProject(loginUser,3); + Map result = projectService.queryAuthorizedProject(loginUser, 3); logger.info(result.toString()); - Assert.assertEquals(Status.USER_NO_OPERATION_PERM,result.get(Constants.STATUS)); + Assert.assertEquals(Status.USER_NO_OPERATION_PERM, result.get(Constants.STATUS)); //success loginUser.setUserType(UserType.ADMIN_USER); - result = projectService.queryAuthorizedProject(loginUser,1); + result = projectService.queryAuthorizedProject(loginUser, 1); logger.info(result.toString()); List projects = (List) result.get(Constants.DATA_LIST); Assert.assertTrue(CollectionUtils.isNotEmpty(projects)); } + @Test - public void testQueryAllProjectList(){ + public void testQueryCreatedProject() { + + User loginUser = getLoginUser(); + + Mockito.when(projectMapper.queryProjectCreatedByUser(1)).thenReturn(getList()); + //USER_NO_OPERATION_PERM + Map result = projectService.queryProjectCreatedByUser(loginUser); + logger.info(result.toString()); + Assert.assertEquals(Status.USER_NO_OPERATION_PERM, result.get(Constants.STATUS)); + + //success + loginUser.setUserType(UserType.ADMIN_USER); + result = projectService.queryProjectCreatedByUser(loginUser); + logger.info(result.toString()); + List projects = (List) result.get(Constants.DATA_LIST); + Assert.assertTrue(CollectionUtils.isNotEmpty(projects)); + + } + + @Test + public void testQueryAllProjectList() { Mockito.when(projectMapper.selectList(null)).thenReturn(getList()); Mockito.when(processDefinitionMapper.selectList(null)).thenReturn(getProcessDefinitions()); Map result = projectService.queryAllProjectList(); logger.info(result.toString()); - List projects = (List) result.get(Constants.DATA_LIST); + List projects = (List) result.get(Constants.DATA_LIST); Assert.assertTrue(CollectionUtils.isNotEmpty(projects)); } + @Test - public void testQueryUnauthorizedProject(){ - // Mockito.when(projectMapper.queryAuthedProjectListByUserId(1)).thenReturn(getList()); + public void testQueryUnauthorizedProject() { + // Mockito.when(projectMapper.queryAuthedProjectListByUserId(1)).thenReturn(getList()); Mockito.when(projectMapper.queryProjectExceptUserId(2)).thenReturn(getList()); User loginUser = new User(); loginUser.setUserType(UserType.ADMIN_USER); - Map result = projectService.queryUnauthorizedProject(loginUser,2); + Map result = projectService.queryUnauthorizedProject(loginUser, 2); logger.info(result.toString()); List projects = (List) result.get(Constants.DATA_LIST); Assert.assertTrue(CollectionUtils.isNotEmpty(projects)); } - - private Project getProject(){ + private Project getProject() { Project project = new Project(); project.setId(1); project.setName(projectName); project.setUserId(1); - return project; + return project; } - private List getList(){ + private List getList() { List list = new ArrayList<>(); list.add(getProject()); return list; } - /** * create admin user - * @return */ - private User getLoginUser(){ + private User getLoginUser() { User loginUser = new User(); loginUser.setUserType(UserType.GENERAL_USER); loginUser.setUserName(userName); loginUser.setId(1); - return loginUser; + return loginUser; } /** * get project user - */ - private ProjectUser getProjectUser(){ + private ProjectUser getProjectUser() { ProjectUser projectUser = new ProjectUser(); projectUser.setProjectId(1); projectUser.setUserId(1); - return projectUser; + return projectUser; } - private List getProcessDefinitions(){ + private List getProcessDefinitions() { List list = new ArrayList<>(); ProcessDefinition processDefinition = new ProcessDefinition(); processDefinition.setProjectId(1); @@ -342,15 +367,11 @@ public class ProjectServiceTest { return list; } - - - - private String getDesc(){ - return "projectUserMapper.deleteProjectRelation(projectId,userId)projectUserMappe" + - ".deleteProjectRelation(projectId,userId)projectUserMappe" + - "r.deleteProjectRelation(projectId,userId)projectUserMapper" + - ".deleteProjectRelation(projectId,userId)projectUserMapper.deleteProjectRelation(projectId,userId)"; + private String getDesc() { + return "projectUserMapper.deleteProjectRelation(projectId,userId)projectUserMappe" + + ".deleteProjectRelation(projectId,userId)projectUserMappe" + + "r.deleteProjectRelation(projectId,userId)projectUserMapper" + + ".deleteProjectRelation(projectId,userId)projectUserMapper.deleteProjectRelation(projectId,userId)"; } - } diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/SchedulerServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/SchedulerServiceTest.java index f75d808e56..deadc2129c 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/SchedulerServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/SchedulerServiceTest.java @@ -17,6 +17,7 @@ package org.apache.dolphinscheduler.api.service; import org.apache.dolphinscheduler.api.enums.Status; +import org.apache.dolphinscheduler.api.service.impl.ProjectServiceImpl; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ReleaseState; import org.apache.dolphinscheduler.common.model.Server; @@ -24,12 +25,16 @@ import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; import org.apache.dolphinscheduler.dao.entity.Project; import org.apache.dolphinscheduler.dao.entity.Schedule; import org.apache.dolphinscheduler.dao.entity.User; -import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper; import org.apache.dolphinscheduler.dao.mapper.ProjectMapper; -import org.apache.dolphinscheduler.dao.mapper.ProjectUserMapper; import org.apache.dolphinscheduler.dao.mapper.ScheduleMapper; import org.apache.dolphinscheduler.service.process.ProcessService; import org.apache.dolphinscheduler.service.quartz.QuartzExecutors; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -40,13 +45,6 @@ import org.mockito.Mockito; import org.powermock.api.mockito.PowerMockito; import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; -import org.quartz.Scheduler; -import org.springframework.beans.factory.annotation.Autowired; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; @RunWith(PowerMockRunner.class) @PrepareForTest(QuartzExecutors.class) @@ -57,10 +55,6 @@ public class SchedulerServiceTest { @InjectMocks private SchedulerService schedulerService; - - @Autowired - private ExecutorService executorService; - @Mock private MonitorService monitorService; @@ -72,21 +66,13 @@ public class SchedulerServiceTest { @Mock private ProjectMapper projectMapper; - @Mock - private ProjectUserMapper projectUserMapper; - @Mock - private ProjectService projectService; @Mock - private ProcessDefinitionMapper processDefinitionMapper; + private ProjectServiceImpl projectService; @Mock private QuartzExecutors quartzExecutors; - @Mock - private Scheduler scheduler; - - @Before public void setUp() { @@ -176,10 +162,10 @@ public class SchedulerServiceTest { Mockito.when(quartzExecutors.deleteJob("1", "1")).thenReturn(true); Mockito.when(quartzExecutors.buildJobGroupName(1)).thenReturn("1"); Mockito.when(quartzExecutors.buildJobName(1)).thenReturn("1"); - boolean flag = true; + boolean flag = true; try { schedulerService.deleteSchedule(1, 1); - }catch (Exception e){ + } catch (Exception e) { flag = false; } Assert.assertTrue(flag); diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/SessionServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/SessionServiceTest.java index 7e98721207..b51f85f456 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/SessionServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/SessionServiceTest.java @@ -16,7 +16,12 @@ */ package org.apache.dolphinscheduler.api.service; +import java.util.ArrayList; import java.util.Calendar; +import java.util.Date; +import java.util.List; + +import org.apache.dolphinscheduler.api.service.impl.SessionServiceImpl; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.UserType; import org.apache.dolphinscheduler.common.utils.DateUtils; @@ -38,10 +43,6 @@ import org.slf4j.LoggerFactory; import org.springframework.mock.web.MockCookie; import org.springframework.mock.web.MockHttpServletRequest; -import java.util.ArrayList; -import java.util.Date; -import java.util.List; - @RunWith(MockitoJUnitRunner.class) public class SessionServiceTest { @@ -49,7 +50,7 @@ public class SessionServiceTest { private static final Logger logger = LoggerFactory.getLogger(SessionServiceTest.class); @InjectMocks - private SessionService sessionService; + private SessionServiceImpl sessionService; @Mock private SessionMapper sessionMapper; diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TaskInstanceServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TaskInstanceServiceTest.java index ebb6139577..16547b3fd7 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TaskInstanceServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TaskInstanceServiceTest.java @@ -16,9 +16,13 @@ */ package org.apache.dolphinscheduler.api.service; -import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.when; + import org.apache.dolphinscheduler.api.ApiApplicationServer; import org.apache.dolphinscheduler.api.enums.Status; +import org.apache.dolphinscheduler.api.service.impl.ProjectServiceImpl; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.enums.UserType; @@ -30,6 +34,14 @@ import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.mapper.ProjectMapper; import org.apache.dolphinscheduler.dao.mapper.TaskInstanceMapper; import org.apache.dolphinscheduler.service.process.ProcessService; + +import java.text.MessageFormat; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; @@ -41,11 +53,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.boot.test.context.SpringBootTest; -import java.text.MessageFormat; -import java.util.*; - -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.when; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; @RunWith(MockitoJUnitRunner.Silent.class) @SpringBootTest(classes = ApiApplicationServer.class) @@ -59,7 +67,7 @@ public class TaskInstanceServiceTest { ProjectMapper projectMapper; @Mock - ProjectService projectService; + ProjectServiceImpl projectService; @Mock ProcessService processService; @@ -67,28 +75,23 @@ public class TaskInstanceServiceTest { @Mock TaskInstanceMapper taskInstanceMapper; - @Mock - ProcessInstanceService processInstanceService; - @Mock UsersService usersService; @Test - public void queryTaskListPaging(){ - + public void queryTaskListPaging() { String projectName = "project_test1"; User loginUser = getAdminUser(); - Map result = new HashMap<>(5); + Map result = new HashMap<>(); putMsg(result, Status.PROJECT_NOT_FOUNT, projectName); //project auth fail when(projectMapper.queryByName(projectName)).thenReturn(null); - when(projectService.checkProjectAndAuth(loginUser,null,projectName)).thenReturn(result); + when(projectService.checkProjectAndAuth(loginUser, null, projectName)).thenReturn(result); Map proejctAuthFailRes = taskInstanceService.queryTaskListPaging(loginUser, "project_test1", 0, "", "test_user", "2019-02-26 19:48:00", "2019-02-26 19:48:22", "", null, "", 1, 20); Assert.assertEquals(Status.PROJECT_NOT_FOUNT, proejctAuthFailRes.get(Constants.STATUS)); - //project putMsg(result, Status.SUCCESS, projectName); Project project = getProject(projectName); @@ -101,7 +104,7 @@ public class TaskInstanceServiceTest { taskInstanceList.add(taskInstance); pageReturn.setRecords(taskInstanceList); when(projectMapper.queryByName(Mockito.anyString())).thenReturn(project); - when(projectService.checkProjectAndAuth(loginUser,project,projectName)).thenReturn(result); + when(projectService.checkProjectAndAuth(loginUser, project, projectName)).thenReturn(result); when(usersService.queryUser(loginUser.getId())).thenReturn(loginUser); when(usersService.getUserIdByName(loginUser.getUserName())).thenReturn(loginUser.getId()); when(taskInstanceMapper.queryTaskInstanceListPaging(Mockito.any(Page.class), eq(project.getId()), eq(1), eq(""), eq(""), @@ -126,10 +129,28 @@ public class TaskInstanceServiceTest { Map executorNullRes = taskInstanceService.queryTaskListPaging(loginUser, projectName, 1, "", "test_user", "2020-01-01 00:00:00", "2020-01-02 00:00:00", "", ExecutionStatus.SUCCESS, "192.168.xx.xx", 1, 20); Assert.assertEquals(Status.SUCCESS, executorNullRes.get(Constants.STATUS)); + + //start/end date null + when(taskInstanceMapper.queryTaskInstanceListPaging(Mockito.any(Page.class), eq(project.getId()), eq(1), eq(""), eq(""), + eq(0), Mockito.any(), eq("192.168.xx.xx"), any(), any())).thenReturn(pageReturn); + Map executorNullDateRes = taskInstanceService.queryTaskListPaging(loginUser, projectName, 1, "", + "", null, null, "", ExecutionStatus.SUCCESS, "192.168.xx.xx", 1, 20); + Assert.assertEquals(Status.SUCCESS, executorNullDateRes.get(Constants.STATUS)); + + //start date error format + when(taskInstanceMapper.queryTaskInstanceListPaging(Mockito.any(Page.class), eq(project.getId()), eq(1), eq(""), eq(""), + eq(0), Mockito.any(), eq("192.168.xx.xx"), any(), any())).thenReturn(pageReturn); + Map executorErrorStartDateRes = taskInstanceService.queryTaskListPaging(loginUser, projectName, 1, "", + "", "error date", null, "", ExecutionStatus.SUCCESS, "192.168.xx.xx", 1, 20); + Assert.assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR, executorErrorStartDateRes.get(Constants.STATUS)); + Map executorErrorEndDateRes = taskInstanceService.queryTaskListPaging(loginUser, projectName, 1, "", + "", null, "error date", "", ExecutionStatus.SUCCESS, "192.168.xx.xx", 1, 20); + Assert.assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR, executorErrorEndDateRes.get(Constants.STATUS)); } /** * get Mock Admin User + * * @return admin user */ private User getAdminUser() { @@ -142,19 +163,21 @@ public class TaskInstanceServiceTest { /** * get mock Project + * * @param projectName projectName * @return Project */ - private Project getProject(String projectName){ + private Project getProject(String projectName) { Project project = new Project(); project.setId(1); project.setName(projectName); project.setUserId(1); - return project; + return project; } /** * get Mock process instance + * * @return process instance */ private ProcessInstance getProcessInstance() { @@ -169,6 +192,7 @@ public class TaskInstanceServiceTest { /** * get Mock task instance + * * @return task instance */ private TaskInstance getTaskInstance() { diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TenantServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TenantServiceTest.java index f7f506b69b..5dcf59cf74 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TenantServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TenantServiceTest.java @@ -14,14 +14,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.api.service; -import java.util.ArrayList; -import java.util.List; -import java.util.Locale; -import java.util.Map; - import org.apache.dolphinscheduler.api.enums.Status; +import org.apache.dolphinscheduler.api.service.impl.TenantServiceImpl; import org.apache.dolphinscheduler.api.utils.PageInfo; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; @@ -35,6 +32,12 @@ import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper; import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper; import org.apache.dolphinscheduler.dao.mapper.TenantMapper; import org.apache.dolphinscheduler.dao.mapper.UserMapper; + +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.Map; + import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; @@ -54,54 +57,61 @@ public class TenantServiceTest { private static final Logger logger = LoggerFactory.getLogger(TenantServiceTest.class); @InjectMocks - private TenantService tenantService; + private TenantServiceImpl tenantService; + @Mock private TenantMapper tenantMapper; + @Mock private ProcessDefinitionMapper processDefinitionMapper; + @Mock private ProcessInstanceMapper processInstanceMapper; + @Mock private UserMapper userMapper; - private String tenantCode = "TenantServiceTest"; - private String tenantName = "TenantServiceTest"; + private static final String tenantCode = "TenantServiceTest"; + private static final String tenantName = "TenantServiceTest"; @Test - public void testCreateTenant(){ + public void testCreateTenant() { User loginUser = getLoginUser(); Mockito.when(tenantMapper.queryByTenantCode(tenantCode)).thenReturn(getList()); try { //check tenantCode - Map result = tenantService.createTenant(getLoginUser(), "%!1111", tenantName, 1, "TenantServiceTest"); + Map result = + tenantService.createTenant(getLoginUser(), "%!1111", tenantName, 1, "TenantServiceTest"); logger.info(result.toString()); - Assert.assertEquals(Status.VERIFY_TENANT_CODE_ERROR,result.get(Constants.STATUS)); + Assert.assertEquals(Status.VERIFY_TENANT_CODE_ERROR, result.get(Constants.STATUS)); //check exist result = tenantService.createTenant(loginUser, tenantCode, tenantName, 1, "TenantServiceTest"); logger.info(result.toString()); - Assert.assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR,result.get(Constants.STATUS)); + Assert.assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR, result.get(Constants.STATUS)); // success result = tenantService.createTenant(loginUser, "test", "test", 1, "TenantServiceTest"); logger.info(result.toString()); - Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); - + Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); + } catch (Exception e) { - logger.error("create tenant error",e); - Assert.assertTrue(false); + logger.error("create tenant error", e); + Assert.fail(); } } @Test - public void testQueryTenantListPage(){ + @SuppressWarnings("unchecked") + public void testQueryTenantListPage() { - IPage page = new Page<>(1,10); + IPage page = new Page<>(1, 10); page.setRecords(getList()); page.setTotal(1L); - Mockito.when(tenantMapper.queryTenantPaging(Mockito.any(Page.class), Mockito.eq("TenantServiceTest"))).thenReturn(page); + Mockito.when(tenantMapper.queryTenantPaging(Mockito.any(Page.class), Mockito.eq("TenantServiceTest"))) + .thenReturn(page); Map result = tenantService.queryTenantList(getLoginUser(), "TenantServiceTest", 1, 10); logger.info(result.toString()); PageInfo pageInfo = (PageInfo) result.get(Constants.DATA_LIST); @@ -110,87 +120,71 @@ public class TenantServiceTest { } @Test - public void testUpdateTenant(){ + public void testUpdateTenant() { Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant()); try { // id not exist - Map result = tenantService.updateTenant(getLoginUser(), 912222, tenantCode, tenantName, 1, "desc"); + Map result = + tenantService.updateTenant(getLoginUser(), 912222, tenantCode, tenantName, 1, "desc"); logger.info(result.toString()); // success - Assert.assertEquals(Status.TENANT_NOT_EXIST,result.get(Constants.STATUS)); + Assert.assertEquals(Status.TENANT_NOT_EXIST, result.get(Constants.STATUS)); result = tenantService.updateTenant(getLoginUser(), 1, tenantCode, "TenantServiceTest001", 1, "desc"); logger.info(result.toString()); - Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); } catch (Exception e) { - logger.error("update tenant error",e); - Assert.assertTrue(false); + logger.error("update tenant error", e); + Assert.fail(); } } @Test - public void testDeleteTenantById(){ + public void testDeleteTenantById() { Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant()); - Mockito.when(processInstanceMapper.queryByTenantIdAndStatus(1, Constants.NOT_TERMINATED_STATES)).thenReturn(getInstanceList()); + Mockito.when(processInstanceMapper.queryByTenantIdAndStatus(1, Constants.NOT_TERMINATED_STATES)) + .thenReturn(getInstanceList()); Mockito.when(processDefinitionMapper.queryDefinitionListByTenant(2)).thenReturn(getDefinitionsList()); - Mockito.when( userMapper.queryUserListByTenant(3)).thenReturn(getUserList()); + Mockito.when(userMapper.queryUserListByTenant(3)).thenReturn(getUserList()); try { //TENANT_NOT_EXIST - Map result = tenantService.deleteTenantById(getLoginUser(),12); + Map result = tenantService.deleteTenantById(getLoginUser(), 12); logger.info(result.toString()); - Assert.assertEquals(Status.TENANT_NOT_EXIST,result.get(Constants.STATUS)); + Assert.assertEquals(Status.TENANT_NOT_EXIST, result.get(Constants.STATUS)); //DELETE_TENANT_BY_ID_FAIL - result = tenantService.deleteTenantById(getLoginUser(),1); + result = tenantService.deleteTenantById(getLoginUser(), 1); logger.info(result.toString()); - Assert.assertEquals(Status.DELETE_TENANT_BY_ID_FAIL,result.get(Constants.STATUS)); + Assert.assertEquals(Status.DELETE_TENANT_BY_ID_FAIL, result.get(Constants.STATUS)); //DELETE_TENANT_BY_ID_FAIL_DEFINES Mockito.when(tenantMapper.queryById(2)).thenReturn(getTenant(2)); - result = tenantService.deleteTenantById(getLoginUser(),2); + result = tenantService.deleteTenantById(getLoginUser(), 2); logger.info(result.toString()); - Assert.assertEquals(Status.DELETE_TENANT_BY_ID_FAIL_DEFINES,result.get(Constants.STATUS)); + Assert.assertEquals(Status.DELETE_TENANT_BY_ID_FAIL_DEFINES, result.get(Constants.STATUS)); //DELETE_TENANT_BY_ID_FAIL_USERS Mockito.when(tenantMapper.queryById(3)).thenReturn(getTenant(3)); - result = tenantService.deleteTenantById(getLoginUser(),3); + result = tenantService.deleteTenantById(getLoginUser(), 3); logger.info(result.toString()); - Assert.assertEquals(Status.DELETE_TENANT_BY_ID_FAIL_USERS,result.get(Constants.STATUS)); + Assert.assertEquals(Status.DELETE_TENANT_BY_ID_FAIL_USERS, result.get(Constants.STATUS)); // success Mockito.when(tenantMapper.queryById(4)).thenReturn(getTenant(4)); - result = tenantService.deleteTenantById(getLoginUser(),4); + result = tenantService.deleteTenantById(getLoginUser(), 4); logger.info(result.toString()); - Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); } catch (Exception e) { - logger.error("delete tenant error",e); - Assert.assertTrue(false); + logger.error("delete tenant error", e); + Assert.fail(); } } @Test - public void testQueryTenantList(){ - - Mockito.when( tenantMapper.selectList(null)).thenReturn(getList()); - Map result = tenantService.queryTenantList(getLoginUser()); - logger.info(result.toString()); - List tenantList = (List) result.get(Constants.DATA_LIST); - Assert.assertTrue(CollectionUtils.isNotEmpty(tenantList)); - - Mockito.when( tenantMapper.queryByTenantCode("1")).thenReturn(getList()); - Map successRes = tenantService.queryTenantList("1"); - Assert.assertEquals(Status.SUCCESS,successRes.get(Constants.STATUS)); - - Mockito.when( tenantMapper.queryByTenantCode("1")).thenReturn(null); - Map tenantNotExistRes = tenantService.queryTenantList("1"); - Assert.assertEquals(Status.TENANT_NOT_EXIST,tenantNotExistRes.get(Constants.STATUS)); - } - - @Test - public void testVerifyTenantCode(){ + public void testVerifyTenantCode() { Mockito.when(tenantMapper.queryByTenantCode(tenantCode)).thenReturn(getList()); // tenantCode not exist @@ -209,12 +203,10 @@ public class TenantServiceTest { Assert.assertEquals(resultString, result.getMsg()); } - /** * get user - * @return */ - private User getLoginUser(){ + private User getLoginUser() { User loginUser = new User(); loginUser.setUserType(UserType.ADMIN_USER); @@ -223,9 +215,8 @@ public class TenantServiceTest { /** * get list - * @return */ - private List getList(){ + private List getList() { List tenantList = new ArrayList<>(); tenantList.add(getTenant()); return tenantList; @@ -233,16 +224,15 @@ public class TenantServiceTest { /** * get tenant - * @return */ - private Tenant getTenant(){ + private Tenant getTenant() { return getTenant(1); } + /** * get tenant - * @return */ - private Tenant getTenant(int id){ + private Tenant getTenant(int id) { Tenant tenant = new Tenant(); tenant.setId(id); tenant.setTenantCode(tenantCode); @@ -250,25 +240,24 @@ public class TenantServiceTest { return tenant; } - private List getUserList(){ + private List getUserList() { List userList = new ArrayList<>(); userList.add(getLoginUser()); return userList; } - private List getInstanceList(){ + private List getInstanceList() { List processInstances = new ArrayList<>(); ProcessInstance processInstance = new ProcessInstance(); processInstances.add(processInstance); return processInstances; } - private List getDefinitionsList(){ + private List getDefinitionsList() { List processDefinitions = new ArrayList<>(); ProcessDefinition processDefinition = new ProcessDefinition(); processDefinitions.add(processDefinition); return processDefinitions; } - } diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UsersServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UsersServiceTest.java index 6939e6a280..c4d3d6e126 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UsersServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UsersServiceTest.java @@ -42,14 +42,14 @@ import org.mockito.junit.MockitoJUnitRunner; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.when; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + @RunWith(MockitoJUnitRunner.class) public class UsersServiceTest { private static final Logger logger = LoggerFactory.getLogger(UsersServiceTest.class); @@ -462,42 +462,122 @@ public class UsersServiceTest { try { //userName error Map result = usersService.registerUser(userName, userPassword, repeatPassword, email); - logger.info(result.toString()); Assert.assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR, result.get(Constants.STATUS)); userName = "userTest0002"; userPassword = "userTest000111111111111111"; //password error result = usersService.registerUser(userName, userPassword, repeatPassword, email); - logger.info(result.toString()); Assert.assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR, result.get(Constants.STATUS)); userPassword = "userTest0002"; email = "1q.com"; //email error result = usersService.registerUser(userName, userPassword, repeatPassword, email); - logger.info(result.toString()); Assert.assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR, result.get(Constants.STATUS)); //repeatPassword error email = "7400@qq.com"; repeatPassword = "userPassword"; result = usersService.registerUser(userName, userPassword, repeatPassword, email); - logger.info(result.toString()); Assert.assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR, result.get(Constants.STATUS)); //success repeatPassword = "userTest0002"; result = usersService.registerUser(userName, userPassword, repeatPassword, email); - logger.info(result.toString()); Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); } catch (Exception e) { - logger.error(Status.CREATE_USER_ERROR.getMsg(),e); Assert.assertTrue(false); } } + + @Test + public void testActivateUser() { + User user = new User(); + user.setUserType(UserType.GENERAL_USER); + String userName = "userTest0002~"; + try { + //not admin + Map result = usersService.activateUser(user, userName); + Assert.assertEquals(Status.USER_NO_OPERATION_PERM, result.get(Constants.STATUS)); + + //userName error + user.setUserType(UserType.ADMIN_USER); + result = usersService.activateUser(user, userName); + Assert.assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR, result.get(Constants.STATUS)); + + //user not exist + userName = "userTest10013"; + result = usersService.activateUser(user, userName); + Assert.assertEquals(Status.USER_NOT_EXIST, result.get(Constants.STATUS)); + + //user state error + userName = "userTest0001"; + when(userMapper.queryByUserNameAccurately(userName)).thenReturn(getUser()); + result = usersService.activateUser(user, userName); + Assert.assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR, result.get(Constants.STATUS)); + + //success + when(userMapper.queryByUserNameAccurately(userName)).thenReturn(getDisabledUser()); + result = usersService.activateUser(user, userName); + Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); + } catch (Exception e) { + Assert.assertTrue(false); + } + } + + @Test + public void testBatchActivateUser() { + User user = new User(); + user.setUserType(UserType.GENERAL_USER); + List userNames = new ArrayList<>(); + userNames.add("userTest0001"); + userNames.add("userTest0002"); + userNames.add("userTest0003~"); + userNames.add("userTest0004"); + + try { + //not admin + Map result = usersService.batchActivateUser(user, userNames); + Assert.assertEquals(Status.USER_NO_OPERATION_PERM, result.get(Constants.STATUS)); + + //batch activate user names + user.setUserType(UserType.ADMIN_USER); + when(userMapper.queryByUserNameAccurately("userTest0001")).thenReturn(getUser()); + when(userMapper.queryByUserNameAccurately("userTest0002")).thenReturn(getDisabledUser()); + result = usersService.batchActivateUser(user, userNames); + Map responseData = (Map) result.get(Constants.DATA_LIST); + Map successData = (Map) responseData.get("success"); + int totalSuccess = (Integer) successData.get("sum"); + + Map failedData = (Map) responseData.get("failed"); + int totalFailed = (Integer) failedData.get("sum"); + + Assert.assertEquals(1, totalSuccess); + Assert.assertEquals(3, totalFailed); + Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); + } catch (Exception e) { + Assert.assertTrue(false); + } + } + + /** + * get disabled user + * @return + */ + private User getDisabledUser() { + + User user = new User(); + user.setUserType(UserType.GENERAL_USER); + user.setUserName("userTest0001"); + user.setUserPassword("userTest0001"); + user.setState(0); + return user; + } + + /** * get user * @return diff --git a/dolphinscheduler-common/pom.xml b/dolphinscheduler-common/pom.xml index 130f9dfa99..2ade59550f 100644 --- a/dolphinscheduler-common/pom.xml +++ b/dolphinscheduler-common/pom.xml @@ -580,6 +580,11 @@ + + com.facebook.presto + presto-jdbc + + com.baomidou mybatis-plus-annotation diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java index 4cb09a1a56..3b12748888 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java @@ -14,6 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.common; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; @@ -27,7 +28,7 @@ import java.util.regex.Pattern; public final class Constants { private Constants() { - throw new IllegalStateException("Constants class"); + throw new UnsupportedOperationException("Construct Constants"); } /** @@ -138,7 +139,7 @@ public final class Constants { /** * python home */ - public static final String PYTHON_HOME="PYTHON_HOME"; + public static final String PYTHON_HOME = "PYTHON_HOME"; /** * resource.view.suffixs @@ -366,7 +367,6 @@ public final class Constants { public static final double DEFAULT_WORKER_RESERVED_MEMORY = OSUtils.totalMemorySize() / 10; - /** * default log cache rows num,output when reach the number */ @@ -752,7 +752,7 @@ public final class Constants { /** - * preview schedule execute count + * preview schedule execute count */ public static final int PREVIEW_SCHEDULE_EXECUTE_COUNT = 5; @@ -832,6 +832,7 @@ public final class Constants { public static final int[] NOT_TERMINATED_STATES = new int[]{ ExecutionStatus.SUBMITTED_SUCCESS.ordinal(), ExecutionStatus.RUNNING_EXECUTION.ordinal(), + ExecutionStatus.DELAY_EXECUTION.ordinal(), ExecutionStatus.READY_PAUSE.ordinal(), ExecutionStatus.READY_STOP.ordinal(), ExecutionStatus.NEED_FAULT_TOLERANCE.ordinal(), @@ -852,18 +853,17 @@ public final class Constants { /** * data total */ - public static final String COUNT = "count"; + public static final String COUNT = "count"; /** * page size */ - public static final String PAGE_SIZE = "pageSize"; + public static final String PAGE_SIZE = "pageSize"; /** * current page no */ - public static final String PAGE_NUMBER = "pageNo"; - + public static final String PAGE_NUMBER = "pageNo"; /** @@ -898,6 +898,7 @@ public final class Constants { public static final String COM_ORACLE_JDBC_DRIVER = "oracle.jdbc.driver.OracleDriver"; public static final String COM_SQLSERVER_JDBC_DRIVER = "com.microsoft.sqlserver.jdbc.SQLServerDriver"; public static final String COM_DB2_JDBC_DRIVER = "com.ibm.db2.jcc.DB2Driver"; + public static final String COM_PRESTO_JDBC_DRIVER = "com.facebook.presto.jdbc.PrestoDriver"; /** * database type @@ -910,6 +911,7 @@ public final class Constants { public static final String ORACLE = "ORACLE"; public static final String SQLSERVER = "SQLSERVER"; public static final String DB2 = "DB2"; + public static final String PRESTO = "PRESTO"; /** * jdbc url @@ -922,6 +924,7 @@ public final class Constants { public static final String JDBC_ORACLE_SERVICE_NAME = "jdbc:oracle:thin:@//"; public static final String JDBC_SQLSERVER = "jdbc:sqlserver://"; public static final String JDBC_DB2 = "jdbc:db2://"; + public static final String JDBC_PRESTO = "jdbc:presto://"; public static final String ADDRESS = "address"; @@ -963,11 +966,11 @@ public final class Constants { /** * authorize writable perm */ - public static final int AUTHORIZE_WRITABLE_PERM=7; + public static final int AUTHORIZE_WRITABLE_PERM = 7; /** * authorize readable perm */ - public static final int AUTHORIZE_READABLE_PERM=4; + public static final int AUTHORIZE_READABLE_PERM = 4; /** diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AlertEvent.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AlertEvent.java new file mode 100644 index 0000000000..0c8ed89fd7 --- /dev/null +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AlertEvent.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.common.enums; + +public enum AlertEvent { + + SERVER_DOWN,TIME_OUT +} diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AlertWarnLevel.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AlertWarnLevel.java new file mode 100644 index 0000000000..71579a9611 --- /dev/null +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AlertWarnLevel.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.common.enums; + +public enum AlertWarnLevel { + + MIDDLE,SERIOUS +} diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/DbType.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/DbType.java index 1d28a759c0..8ff2c70bba 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/DbType.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/DbType.java @@ -33,6 +33,7 @@ public enum DbType { * 5 oracle * 6 sqlserver * 7 db2 + * 8 presto */ MYSQL(0, "mysql"), POSTGRESQL(1, "postgresql"), @@ -41,7 +42,8 @@ public enum DbType { CLICKHOUSE(4, "clickhouse"), ORACLE(5, "oracle"), SQLSERVER(6, "sqlserver"), - DB2(7, "db2"); + DB2(7, "db2"), + PRESTO(8, "presto"); DbType(int code, String descp) { this.code = code; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ExecutionStatus.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ExecutionStatus.java index 6ea02ef096..f6ac2cf5ab 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ExecutionStatus.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ExecutionStatus.java @@ -14,16 +14,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.common.enums; - -import com.baomidou.mybatisplus.annotation.EnumValue; - import java.util.HashMap; +import com.baomidou.mybatisplus.annotation.EnumValue; + /** * running status for workflow and task nodes - * */ public enum ExecutionStatus { @@ -41,6 +40,7 @@ public enum ExecutionStatus { * 9 kill * 10 waiting thread * 11 waiting depend node complete + * 12 delay execution */ SUBMITTED_SUCCESS(0, "submit success"), RUNNING_EXECUTION(1, "running"), @@ -53,9 +53,10 @@ public enum ExecutionStatus { NEED_FAULT_TOLERANCE(8, "need fault tolerance"), KILL(9, "kill"), WAITTING_THREAD(10, "waiting thread"), - WAITTING_DEPEND(11, "waiting depend node complete"); + WAITTING_DEPEND(11, "waiting depend node complete"), + DELAY_EXECUTION(12, "delay execution"); - ExecutionStatus(int code, String descp){ + ExecutionStatus(int code, String descp) { this.code = code; this.descp = descp; } @@ -64,77 +65,85 @@ public enum ExecutionStatus { private final int code; private final String descp; - private static HashMap EXECUTION_STATUS_MAP=new HashMap<>(); + private static HashMap EXECUTION_STATUS_MAP = new HashMap<>(); static { - for (ExecutionStatus executionStatus:ExecutionStatus.values()){ - EXECUTION_STATUS_MAP.put(executionStatus.code,executionStatus); - } + for (ExecutionStatus executionStatus : ExecutionStatus.values()) { + EXECUTION_STATUS_MAP.put(executionStatus.code, executionStatus); + } } - /** - * status is success - * @return status - */ - public boolean typeIsSuccess(){ - return this == SUCCESS; - } + /** + * status is success + * + * @return status + */ + public boolean typeIsSuccess() { + return this == SUCCESS; + } - /** - * status is failure - * @return status - */ - public boolean typeIsFailure(){ - return this == FAILURE || this == NEED_FAULT_TOLERANCE || this == KILL; - } + /** + * status is failure + * + * @return status + */ + public boolean typeIsFailure() { + return this == FAILURE || this == NEED_FAULT_TOLERANCE || this == KILL; + } - /** - * status is finished - * @return status - */ - public boolean typeIsFinished(){ - - return typeIsSuccess() || typeIsFailure() || typeIsCancel() || typeIsPause() - || typeIsStop(); - } + /** + * status is finished + * + * @return status + */ + public boolean typeIsFinished() { + return typeIsSuccess() || typeIsFailure() || typeIsCancel() || typeIsPause() + || typeIsStop(); + } /** * status is waiting thread + * * @return status */ - public boolean typeIsWaitingThread(){ - return this == WAITTING_THREAD; - } + public boolean typeIsWaitingThread() { + return this == WAITTING_THREAD; + } /** * status is pause + * * @return status */ - public boolean typeIsPause(){ - return this == PAUSE; - } + public boolean typeIsPause() { + return this == PAUSE; + } + /** * status is pause + * * @return status */ - public boolean typeIsStop(){ + public boolean typeIsStop() { return this == STOP; } /** * status is running + * * @return status */ - public boolean typeIsRunning(){ - return this == RUNNING_EXECUTION || this == WAITTING_DEPEND; - } + public boolean typeIsRunning() { + return this == RUNNING_EXECUTION || this == WAITTING_DEPEND || this == DELAY_EXECUTION; + } /** * status is cancel + * * @return status */ - public boolean typeIsCancel(){ - return this == KILL || this == STOP ; + public boolean typeIsCancel() { + return this == KILL || this == STOP; } public int getCode() { @@ -145,10 +154,10 @@ public enum ExecutionStatus { return descp; } - public static ExecutionStatus of(int status){ - if(EXECUTION_STATUS_MAP.containsKey(status)){ - return EXECUTION_STATUS_MAP.get(status); - } + public static ExecutionStatus of(int status) { + if (EXECUTION_STATUS_MAP.containsKey(status)) { + return EXECUTION_STATUS_MAP.get(status); + } throw new IllegalArgumentException("invalid status : " + status); } } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/TaskStateType.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/TaskStateType.java index 11ab8560b7..36766a7f4d 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/TaskStateType.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/TaskStateType.java @@ -31,12 +31,13 @@ public enum TaskStateType { /** * convert task state to execute status integer array ; + * * @param taskStateType task state type * @return result of execution status */ - public static int[] convert2ExecutStatusIntArray(TaskStateType taskStateType){ + public static int[] convert2ExecutStatusIntArray(TaskStateType taskStateType) { - switch (taskStateType){ + switch (taskStateType) { case SUCCESS: return new int[]{ExecutionStatus.SUCCESS.ordinal()}; case FAILED: @@ -51,14 +52,15 @@ public enum TaskStateType { case RUNNING: return new int[]{ExecutionStatus.SUBMITTED_SUCCESS.ordinal(), ExecutionStatus.RUNNING_EXECUTION.ordinal(), + ExecutionStatus.DELAY_EXECUTION.ordinal(), ExecutionStatus.READY_PAUSE.ordinal(), ExecutionStatus.READY_STOP.ordinal()}; case WAITTING: return new int[]{ ExecutionStatus.SUBMITTED_SUCCESS.ordinal() }; - default: - break; + default: + break; } return new int[0]; } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/TaskNode.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/TaskNode.java index f794396457..cd3e573b16 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/TaskNode.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/TaskNode.java @@ -136,6 +136,11 @@ public class TaskNode { @JsonSerialize(using = JSONUtils.JsonDataSerializer.class) private String timeout; + /** + * delay execution time. + */ + private int delayTime; + public String getId() { return id; } @@ -310,24 +315,25 @@ public class TaskNode { @Override public String toString() { - return "TaskNode{" + - "id='" + id + '\'' + - ", name='" + name + '\'' + - ", desc='" + desc + '\'' + - ", type='" + type + '\'' + - ", runFlag='" + runFlag + '\'' + - ", loc='" + loc + '\'' + - ", maxRetryTimes=" + maxRetryTimes + - ", retryInterval=" + retryInterval + - ", params='" + params + '\'' + - ", preTasks='" + preTasks + '\'' + - ", extras='" + extras + '\'' + - ", depList=" + depList + - ", dependence='" + dependence + '\'' + - ", taskInstancePriority=" + taskInstancePriority + - ", timeout='" + timeout + '\'' + - ", workerGroup='" + workerGroup + '\'' + - '}'; + return "TaskNode{" + + "id='" + id + '\'' + + ", name='" + name + '\'' + + ", desc='" + desc + '\'' + + ", type='" + type + '\'' + + ", runFlag='" + runFlag + '\'' + + ", loc='" + loc + '\'' + + ", maxRetryTimes=" + maxRetryTimes + + ", retryInterval=" + retryInterval + + ", params='" + params + '\'' + + ", preTasks='" + preTasks + '\'' + + ", extras='" + extras + '\'' + + ", depList=" + depList + + ", dependence='" + dependence + '\'' + + ", taskInstancePriority=" + taskInstancePriority + + ", timeout='" + timeout + '\'' + + ", workerGroup='" + workerGroup + '\'' + + ", delayTime=" + delayTime + + '}'; } public String getWorkerGroup() { @@ -353,4 +359,12 @@ public class TaskNode { public void setWorkerGroupId(Integer workerGroupId) { this.workerGroupId = workerGroupId; } + + public int getDelayTime() { + return delayTime; + } + + public void setDelayTime(int delayTime) { + this.delayTime = delayTime; + } } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CollectionUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CollectionUtils.java index d900f0f6bf..bc7c93af3d 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CollectionUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CollectionUtils.java @@ -14,13 +14,19 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.common.utils; import org.apache.commons.beanutils.BeanMap; -import org.apache.commons.lang.StringUtils; - -import java.util.*; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; /** * Provides utility methods and decorators for {@link Collection} instances. @@ -37,8 +43,9 @@ import java.util.*; public class CollectionUtils { private CollectionUtils() { - throw new IllegalStateException("CollectionUtils class"); + throw new UnsupportedOperationException("Construct CollectionUtils"); } + /** * Returns a new {@link Collection} containing a minus a subset of * b. Only the elements of b that satisfy the predicate @@ -71,7 +78,7 @@ public class CollectionUtils { /** * String to map * - * @param str string + * @param str string * @param separator separator * @return string to map */ @@ -82,7 +89,7 @@ public class CollectionUtils { /** * String to map * - * @param str string + * @param str string * @param separator separator * @param keyPrefix prefix * @return string to map @@ -112,7 +119,6 @@ public class CollectionUtils { return map; } - /** * Helper class to easily access cardinality properties of two collections. * @@ -137,8 +143,8 @@ public class CollectionUtils { * @param b the second collection */ public CardinalityHelper(final Iterable a, final Iterable b) { - cardinalityA = CollectionUtils.getCardinalityMap(a); - cardinalityB = CollectionUtils.getCardinalityMap(b); + cardinalityA = CollectionUtils.getCardinalityMap(a); + cardinalityB = CollectionUtils.getCardinalityMap(b); } /** @@ -227,7 +233,7 @@ public class CollectionUtils { * Only those elements present in the collection will appear as * keys in the map. * - * @param the type of object in the returned {@link Map}. This is a super type of O + * @param the type of object in the returned {@link Map}. This is a super type of O * @param coll the collection to get the cardinality map for, must not be null * @return the populated cardinality map */ @@ -239,9 +245,9 @@ public class CollectionUtils { return count; } - /** * Removes certain attributes of each object in the list + * * @param originList origin list * @param exclusionSet exclusion set * @param T @@ -258,8 +264,8 @@ public class CollectionUtils { Map instanceMap; for (T instance : originList) { Map dataMap = new BeanMap(instance); - instanceMap = new LinkedHashMap<>(16,0.75f,true); - for (Map.Entry entry: dataMap.entrySet()) { + instanceMap = new LinkedHashMap<>(16, 0.75f, true); + for (Map.Entry entry : dataMap.entrySet()) { if (exclusionSet.contains(entry.getKey())) { continue; } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CommonUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CommonUtils.java index 6722c23037..45c5aa2c93 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CommonUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CommonUtils.java @@ -14,124 +14,129 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.common.utils; -import org.apache.commons.codec.binary.Base64; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ResUploadType; + +import org.apache.commons.codec.binary.Base64; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.net.URL; import java.nio.charset.StandardCharsets; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + /** * common utils */ public class CommonUtils { - private static final Logger logger = LoggerFactory.getLogger(CommonUtils.class); + private static final Logger logger = LoggerFactory.getLogger(CommonUtils.class); - private static final Base64 BASE64 = new Base64(); + private static final Base64 BASE64 = new Base64(); - private CommonUtils() { - throw new IllegalStateException("CommonUtils class"); - } - - /** - * @return get the path of system environment variables - */ - public static String getSystemEnvPath() { - String envPath = PropertyUtils.getString(Constants.DOLPHINSCHEDULER_ENV_PATH); - if (StringUtils.isEmpty(envPath)) { - URL envDefaultPath = CommonUtils.class.getClassLoader().getResource(Constants.ENV_PATH); - - if (envDefaultPath != null){ - envPath = envDefaultPath.getPath(); - logger.debug("env path :{}", envPath); - }else{ - envPath = "/etc/profile"; - } + private CommonUtils() { + throw new UnsupportedOperationException("Construct CommonUtils"); } - return envPath; - } + /** + * @return get the path of system environment variables + */ + public static String getSystemEnvPath() { + String envPath = PropertyUtils.getString(Constants.DOLPHINSCHEDULER_ENV_PATH); + if (StringUtils.isEmpty(envPath)) { + URL envDefaultPath = CommonUtils.class.getClassLoader().getResource(Constants.ENV_PATH); - /** - * - * @return is develop mode - */ - public static boolean isDevelopMode() { - return PropertyUtils.getBoolean(Constants.DEVELOPMENT_STATE, true); - } + if (envDefaultPath != null) { + envPath = envDefaultPath.getPath(); + logger.debug("env path :{}", envPath); + } else { + envPath = "/etc/profile"; + } + } - - - /** - * if upload resource is HDFS and kerberos startup is true , else false - * @return true if upload resource is HDFS and kerberos startup - */ - public static boolean getKerberosStartupState(){ - String resUploadStartupType = PropertyUtils.getUpperCaseString(Constants.RESOURCE_STORAGE_TYPE); - ResUploadType resUploadType = ResUploadType.valueOf(resUploadStartupType); - Boolean kerberosStartupState = PropertyUtils.getBoolean(Constants.HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE,false); - return resUploadType == ResUploadType.HDFS && kerberosStartupState; - } - - /** - * load kerberos configuration - * @throws Exception errors - */ - public static void loadKerberosConf()throws Exception{ - if (CommonUtils.getKerberosStartupState()) { - System.setProperty(Constants.JAVA_SECURITY_KRB5_CONF, PropertyUtils.getString(Constants.JAVA_SECURITY_KRB5_CONF_PATH)); - Configuration configuration = new Configuration(); - configuration.set(Constants.HADOOP_SECURITY_AUTHENTICATION, Constants.KERBEROS); - UserGroupInformation.setConfiguration(configuration); - UserGroupInformation.loginUserFromKeytab(PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_USERNAME), - PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_PATH)); + return envPath; } - } - /** - * encode password - * @param password - * @return - */ - public static String encodePassword(String password) { - if(StringUtils.isEmpty(password)){return StringUtils.EMPTY; } - //if encryption is not turned on, return directly - boolean encryptionEnable = PropertyUtils.getBoolean(Constants.DATASOURCE_ENCRYPTION_ENABLE,false); - if ( !encryptionEnable){ return password; } - - // Using Base64 + salt to process password - String salt = PropertyUtils.getString(Constants.DATASOURCE_ENCRYPTION_SALT,Constants.DATASOURCE_ENCRYPTION_SALT_DEFAULT); - String passwordWithSalt = salt + new String(BASE64.encode(password.getBytes(StandardCharsets.UTF_8))) ; - return new String(BASE64.encode(passwordWithSalt.getBytes(StandardCharsets.UTF_8))); - } - - /** - * decode password - * @param password - * @return - */ - public static String decodePassword(String password) { - if(StringUtils.isEmpty(password)){return StringUtils.EMPTY ; } - - //if encryption is not turned on, return directly - boolean encryptionEnable = PropertyUtils.getBoolean(Constants.DATASOURCE_ENCRYPTION_ENABLE,false); - if ( !encryptionEnable){ return password; } - - // Using Base64 + salt to process password - String salt = PropertyUtils.getString(Constants.DATASOURCE_ENCRYPTION_SALT,Constants.DATASOURCE_ENCRYPTION_SALT_DEFAULT); - String passwordWithSalt = new String(BASE64.decode(password), StandardCharsets.UTF_8) ; - if(!passwordWithSalt.startsWith(salt)){ - logger.warn("There is a password and salt mismatch: {} ",password); - return password; + /** + * @return is develop mode + */ + public static boolean isDevelopMode() { + return PropertyUtils.getBoolean(Constants.DEVELOPMENT_STATE, true); } - return new String(BASE64.decode(passwordWithSalt.substring(salt.length())), StandardCharsets.UTF_8) ; - } + /** + * if upload resource is HDFS and kerberos startup is true , else false + * + * @return true if upload resource is HDFS and kerberos startup + */ + public static boolean getKerberosStartupState() { + String resUploadStartupType = PropertyUtils.getUpperCaseString(Constants.RESOURCE_STORAGE_TYPE); + ResUploadType resUploadType = ResUploadType.valueOf(resUploadStartupType); + Boolean kerberosStartupState = PropertyUtils.getBoolean(Constants.HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE, false); + return resUploadType == ResUploadType.HDFS && kerberosStartupState; + } + + /** + * load kerberos configuration + * + * @throws Exception errors + */ + public static void loadKerberosConf() throws Exception { + if (CommonUtils.getKerberosStartupState()) { + System.setProperty(Constants.JAVA_SECURITY_KRB5_CONF, PropertyUtils.getString(Constants.JAVA_SECURITY_KRB5_CONF_PATH)); + Configuration configuration = new Configuration(); + configuration.set(Constants.HADOOP_SECURITY_AUTHENTICATION, Constants.KERBEROS); + UserGroupInformation.setConfiguration(configuration); + UserGroupInformation.loginUserFromKeytab(PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_USERNAME), + PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_PATH)); + } + } + + /** + * encode password + */ + public static String encodePassword(String password) { + if (StringUtils.isEmpty(password)) { + return StringUtils.EMPTY; + } + //if encryption is not turned on, return directly + boolean encryptionEnable = PropertyUtils.getBoolean(Constants.DATASOURCE_ENCRYPTION_ENABLE, false); + if (!encryptionEnable) { + return password; + } + + // Using Base64 + salt to process password + String salt = PropertyUtils.getString(Constants.DATASOURCE_ENCRYPTION_SALT, Constants.DATASOURCE_ENCRYPTION_SALT_DEFAULT); + String passwordWithSalt = salt + new String(BASE64.encode(password.getBytes(StandardCharsets.UTF_8))); + return new String(BASE64.encode(passwordWithSalt.getBytes(StandardCharsets.UTF_8))); + } + + /** + * decode password + */ + public static String decodePassword(String password) { + if (StringUtils.isEmpty(password)) { + return StringUtils.EMPTY; + } + + //if encryption is not turned on, return directly + boolean encryptionEnable = PropertyUtils.getBoolean(Constants.DATASOURCE_ENCRYPTION_ENABLE, false); + if (!encryptionEnable) { + return password; + } + + // Using Base64 + salt to process password + String salt = PropertyUtils.getString(Constants.DATASOURCE_ENCRYPTION_SALT, Constants.DATASOURCE_ENCRYPTION_SALT_DEFAULT); + String passwordWithSalt = new String(BASE64.decode(password), StandardCharsets.UTF_8); + if (!passwordWithSalt.startsWith(salt)) { + logger.warn("There is a password and salt mismatch: {} ", password); + return password; + } + return new String(BASE64.decode(passwordWithSalt.substring(salt.length())), StandardCharsets.UTF_8); + } } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ConnectionUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ConnectionUtils.java index f8ea0e7188..f0cd8f2fe0 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ConnectionUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ConnectionUtils.java @@ -14,37 +14,40 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.common.utils; import java.util.Arrays; import java.util.Objects; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class ConnectionUtils { - public static final Logger logger = LoggerFactory.getLogger(ConnectionUtils.class); + public static final Logger logger = LoggerFactory.getLogger(ConnectionUtils.class); - private ConnectionUtils() { - throw new IllegalStateException("ConnectionUtils class"); - } - - /** - * release resource - * @param resources resources - */ - public static void releaseResource(AutoCloseable... resources) { - - if (resources == null || resources.length == 0) { - return; + private ConnectionUtils() { + throw new UnsupportedOperationException("Construct ConnectionUtils"); + } + + /** + * release resource + * + * @param resources resources + */ + public static void releaseResource(AutoCloseable... resources) { + + if (resources == null || resources.length == 0) { + return; + } + Arrays.stream(resources).filter(Objects::nonNull) + .forEach(resource -> { + try { + resource.close(); + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + }); } - Arrays.stream(resources).filter(Objects::nonNull) - .forEach(resource -> { - try { - resource.close(); - } catch (Exception e) { - logger.error(e.getMessage(), e); - } - }); - } } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/DateUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/DateUtils.java index 1033816e8e..283b4e7f80 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/DateUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/DateUtils.java @@ -1 +1,447 @@ -/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.dolphinscheduler.common.utils; import org.apache.dolphinscheduler.common.Constants; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.time.Instant; import java.time.LocalDateTime; import java.time.ZoneId; import java.time.format.DateTimeFormatter; import java.util.Calendar; import java.util.Date; /** * date utils */ public class DateUtils { private static final Logger logger = LoggerFactory.getLogger(DateUtils.class); /** * date to local datetime * * @param date date * @return local datetime */ private static LocalDateTime date2LocalDateTime(Date date) { return LocalDateTime.ofInstant(date.toInstant(), ZoneId.systemDefault()); } /** * local datetime to date * * @param localDateTime local datetime * @return date */ private static Date localDateTime2Date(LocalDateTime localDateTime) { Instant instant = localDateTime.atZone(ZoneId.systemDefault()).toInstant(); return Date.from(instant); } /** * get current date str * * @return date string */ public static String getCurrentTime() { return getCurrentTime(Constants.YYYY_MM_DD_HH_MM_SS); } /** * get the date string in the specified format of the current time * * @param format date format * @return date string */ public static String getCurrentTime(String format) { return LocalDateTime.now().format(DateTimeFormatter.ofPattern(format)); } /** * get the formatted date string * * @param date date * @param format e.g. yyyy-MM-dd HH:mm:ss * @return date string */ public static String format(Date date, String format) { return format(date2LocalDateTime(date), format); } /** * get the formatted date string * * @param localDateTime local data time * @param format yyyy-MM-dd HH:mm:ss * @return date string */ public static String format(LocalDateTime localDateTime, String format) { return localDateTime.format(DateTimeFormatter.ofPattern(format)); } /** * convert time to yyyy-MM-dd HH:mm:ss format * * @param date date * @return date string */ public static String dateToString(Date date) { return format(date, Constants.YYYY_MM_DD_HH_MM_SS); } /** * convert string to date and time * * @param date date * @param format format * @return date */ public static Date parse(String date, String format) { try { LocalDateTime ldt = LocalDateTime.parse(date, DateTimeFormatter.ofPattern(format)); return localDateTime2Date(ldt); } catch (Exception e) { logger.error("error while parse date:" + date, e); } return null; } /** * convert date str to yyyy-MM-dd HH:mm:ss format * * @param str date string * @return yyyy-MM-dd HH:mm:ss format */ public static Date stringToDate(String str) { return parse(str, Constants.YYYY_MM_DD_HH_MM_SS); } /** * get seconds between two dates * * @param d1 date1 * @param d2 date2 * @return differ seconds */ public static long differSec(Date d1, Date d2) { if(d1 == null || d2 == null){ return 0; } return (long) Math.ceil(differMs(d1, d2) / 1000.0); } /** * get ms between two dates * * @param d1 date1 * @param d2 date2 * @return differ ms */ public static long differMs(Date d1, Date d2) { return Math.abs(d1.getTime() - d2.getTime()); } /** * get hours between two dates * * @param d1 date1 * @param d2 date2 * @return differ hours */ public static long diffHours(Date d1, Date d2) { return (long) Math.ceil(diffMin(d1, d2) / 60.0); } /** * get minutes between two dates * * @param d1 date1 * @param d2 date2 * @return differ minutes */ public static long diffMin(Date d1, Date d2) { return (long) Math.ceil(differSec(d1, d2) / 60.0); } /** * get the date of the specified date in the days before and after * * @param date date * @param day day * @return the date of the specified date in the days before and after */ public static Date getSomeDay(Date date, int day) { Calendar calendar = Calendar.getInstance(); calendar.setTime(date); calendar.add(Calendar.DATE, day); return calendar.getTime(); } /** * get the hour of day. * * @param date date * @return hour of day */ public static int getHourIndex(Date date) { Calendar calendar = Calendar.getInstance(); calendar.setTime(date); return calendar.get(Calendar.HOUR_OF_DAY); } /** * compare two dates * * @param future future date * @param old old date * @return true if future time greater than old time */ public static boolean compare(Date future, Date old) { return future.getTime() > old.getTime(); } /** * convert schedule string to date * * @param schedule schedule * @return convert schedule string to date */ public static Date getScheduleDate(String schedule) { return stringToDate(schedule); } /** * format time to readable * * @param ms ms * @return format time */ public static String format2Readable(long ms) { long days = ms / (1000 * 60 * 60 * 24); long hours = (ms % (1000 * 60 * 60 * 24)) / (1000 * 60 * 60); long minutes = (ms % (1000 * 60 * 60)) / (1000 * 60); long seconds = (ms % (1000 * 60)) / 1000; return String.format("%02d %02d:%02d:%02d", days, hours, minutes, seconds); } /** * get monday * * note: Set the first day of the week to Monday, the default is Sunday * @param date date * @return get monday */ public static Date getMonday(Date date) { Calendar cal = Calendar.getInstance(); cal.setTime(date); cal.setFirstDayOfWeek(Calendar.MONDAY); cal.set(Calendar.DAY_OF_WEEK, Calendar.MONDAY); return cal.getTime(); } /** * get sunday * * note: Set the first day of the week to Monday, the default is Sunday * @param date date * @return get sunday */ public static Date getSunday(Date date) { Calendar cal = Calendar.getInstance(); cal.setTime(date); cal.setFirstDayOfWeek(Calendar.MONDAY); cal.set(Calendar.DAY_OF_WEEK, Calendar.SUNDAY); return cal.getTime(); } /** * get first day of month * * @param date date * @return first day of month * */ public static Date getFirstDayOfMonth(Date date) { Calendar cal = Calendar.getInstance(); cal.setTime(date); cal.set(Calendar.DAY_OF_MONTH, 1); return cal.getTime(); } /** * get some hour of day * * @param date date * @param offsetHour hours * @return some hour of day * */ public static Date getSomeHourOfDay(Date date, int offsetHour) { Calendar cal = Calendar.getInstance(); cal.setTime(date); cal.set(Calendar.HOUR_OF_DAY, cal.get(Calendar.HOUR_OF_DAY) + offsetHour); cal.set(Calendar.MINUTE, 0); cal.set(Calendar.SECOND, 0); cal.set(Calendar.MILLISECOND, 0); return cal.getTime(); } /** * get last day of month * * @param date date * @return get last day of month */ public static Date getLastDayOfMonth(Date date) { Calendar cal = Calendar.getInstance(); cal.setTime(date); cal.add(Calendar.MONTH, 1); cal.set(Calendar.DAY_OF_MONTH, 1); cal.add(Calendar.DAY_OF_MONTH, -1); return cal.getTime(); } /** * return YYYY-MM-DD 00:00:00 * * @param inputDay date * @return start day */ public static Date getStartOfDay(Date inputDay) { Calendar cal = Calendar.getInstance(); cal.setTime(inputDay); cal.set(Calendar.HOUR_OF_DAY, 0); cal.set(Calendar.MINUTE, 0); cal.set(Calendar.SECOND, 0); cal.set(Calendar.MILLISECOND, 0); return cal.getTime(); } /** * return YYYY-MM-DD 23:59:59 * * @param inputDay day * @return end of day */ public static Date getEndOfDay(Date inputDay) { Calendar cal = Calendar.getInstance(); cal.setTime(inputDay); cal.set(Calendar.HOUR_OF_DAY, 23); cal.set(Calendar.MINUTE, 59); cal.set(Calendar.SECOND, 59); cal.set(Calendar.MILLISECOND, 999); return cal.getTime(); } /** * return YYYY-MM-DD 00:00:00 * * @param inputDay day * @return start of hour */ public static Date getStartOfHour(Date inputDay) { Calendar cal = Calendar.getInstance(); cal.setTime(inputDay); cal.set(Calendar.MINUTE, 0); cal.set(Calendar.SECOND, 0); cal.set(Calendar.MILLISECOND, 0); return cal.getTime(); } /** * return YYYY-MM-DD 23:59:59 * * @param inputDay day * @return end of hour */ public static Date getEndOfHour(Date inputDay) { Calendar cal = Calendar.getInstance(); cal.setTime(inputDay); cal.set(Calendar.MINUTE, 59); cal.set(Calendar.SECOND, 59); cal.set(Calendar.MILLISECOND, 999); return cal.getTime(); } /** * get current date * @return current date */ public static Date getCurrentDate() { return DateUtils.parse(DateUtils.getCurrentTime(), Constants.YYYY_MM_DD_HH_MM_SS); } /** * get date * @param date date * @param calendarField calendarField * @param amount amount * @return date */ public static Date add(final Date date, final int calendarField, final int amount) { if (date == null) { throw new IllegalArgumentException("The date must not be null"); } final Calendar c = Calendar.getInstance(); c.setTime(date); c.add(calendarField, amount); return c.getTime(); } } \ No newline at end of file +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.common.utils; + +import org.apache.dolphinscheduler.common.Constants; + +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.util.Calendar; +import java.util.Date; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * date utils + */ +public class DateUtils { + + private static final Logger logger = LoggerFactory.getLogger(DateUtils.class); + + private DateUtils() { + throw new UnsupportedOperationException("Construct DateUtils"); + } + + /** + * date to local datetime + * + * @param date date + * @return local datetime + */ + private static LocalDateTime date2LocalDateTime(Date date) { + return LocalDateTime.ofInstant(date.toInstant(), ZoneId.systemDefault()); + } + + /** + * local datetime to date + * + * @param localDateTime local datetime + * @return date + */ + private static Date localDateTime2Date(LocalDateTime localDateTime) { + Instant instant = localDateTime.atZone(ZoneId.systemDefault()).toInstant(); + return Date.from(instant); + } + + /** + * get current date str + * + * @return date string + */ + public static String getCurrentTime() { + return getCurrentTime(Constants.YYYY_MM_DD_HH_MM_SS); + } + + /** + * get the date string in the specified format of the current time + * + * @param format date format + * @return date string + */ + public static String getCurrentTime(String format) { + return LocalDateTime.now().format(DateTimeFormatter.ofPattern(format)); + } + + /** + * get the formatted date string + * + * @param date date + * @param format e.g. yyyy-MM-dd HH:mm:ss + * @return date string + */ + public static String format(Date date, String format) { + return format(date2LocalDateTime(date), format); + } + + /** + * get the formatted date string + * + * @param localDateTime local data time + * @param format yyyy-MM-dd HH:mm:ss + * @return date string + */ + public static String format(LocalDateTime localDateTime, String format) { + return localDateTime.format(DateTimeFormatter.ofPattern(format)); + } + + /** + * convert time to yyyy-MM-dd HH:mm:ss format + * + * @param date date + * @return date string + */ + public static String dateToString(Date date) { + return format(date, Constants.YYYY_MM_DD_HH_MM_SS); + } + + /** + * convert string to date and time + * + * @param date date + * @param format format + * @return date + */ + public static Date parse(String date, String format) { + try { + LocalDateTime ldt = LocalDateTime.parse(date, DateTimeFormatter.ofPattern(format)); + return localDateTime2Date(ldt); + } catch (Exception e) { + logger.error("error while parse date:" + date, e); + } + return null; + } + + /** + * convert date str to yyyy-MM-dd HH:mm:ss format + * + * @param str date string + * @return yyyy-MM-dd HH:mm:ss format + */ + public static Date stringToDate(String str) { + return parse(str, Constants.YYYY_MM_DD_HH_MM_SS); + } + + /** + * get seconds between two dates + * + * @param d1 date1 + * @param d2 date2 + * @return differ seconds + */ + public static long differSec(Date d1, Date d2) { + if (d1 == null || d2 == null) { + return 0; + } + return (long) Math.ceil(differMs(d1, d2) / 1000.0); + } + + /** + * get ms between two dates + * + * @param d1 date1 + * @param d2 date2 + * @return differ ms + */ + public static long differMs(Date d1, Date d2) { + return Math.abs(d1.getTime() - d2.getTime()); + } + + /** + * get hours between two dates + * + * @param d1 date1 + * @param d2 date2 + * @return differ hours + */ + public static long diffHours(Date d1, Date d2) { + return (long) Math.ceil(diffMin(d1, d2) / 60.0); + } + + /** + * get minutes between two dates + * + * @param d1 date1 + * @param d2 date2 + * @return differ minutes + */ + public static long diffMin(Date d1, Date d2) { + return (long) Math.ceil(differSec(d1, d2) / 60.0); + } + + /** + * get the date of the specified date in the days before and after + * + * @param date date + * @param day day + * @return the date of the specified date in the days before and after + */ + public static Date getSomeDay(Date date, int day) { + Calendar calendar = Calendar.getInstance(); + calendar.setTime(date); + calendar.add(Calendar.DATE, day); + return calendar.getTime(); + } + + /** + * get the hour of day. + * + * @param date date + * @return hour of day + */ + public static int getHourIndex(Date date) { + Calendar calendar = Calendar.getInstance(); + calendar.setTime(date); + return calendar.get(Calendar.HOUR_OF_DAY); + } + + /** + * compare two dates + * + * @param future future date + * @param old old date + * @return true if future time greater than old time + */ + public static boolean compare(Date future, Date old) { + return future.getTime() > old.getTime(); + } + + /** + * convert schedule string to date + * + * @param schedule schedule + * @return convert schedule string to date + */ + public static Date getScheduleDate(String schedule) { + return stringToDate(schedule); + } + + /** + * format time to readable + * + * @param ms ms + * @return format time + */ + public static String format2Readable(long ms) { + + long days = ms / (1000 * 60 * 60 * 24); + long hours = (ms % (1000 * 60 * 60 * 24)) / (1000 * 60 * 60); + long minutes = (ms % (1000 * 60 * 60)) / (1000 * 60); + long seconds = (ms % (1000 * 60)) / 1000; + + return String.format("%02d %02d:%02d:%02d", days, hours, minutes, seconds); + + } + + /** + * get monday + *

+ * note: Set the first day of the week to Monday, the default is Sunday + * + * @param date date + * @return get monday + */ + public static Date getMonday(Date date) { + Calendar cal = Calendar.getInstance(); + + cal.setTime(date); + + cal.setFirstDayOfWeek(Calendar.MONDAY); + cal.set(Calendar.DAY_OF_WEEK, Calendar.MONDAY); + + return cal.getTime(); + } + + /** + * get sunday + *

+ * note: Set the first day of the week to Monday, the default is Sunday + * + * @param date date + * @return get sunday + */ + public static Date getSunday(Date date) { + Calendar cal = Calendar.getInstance(); + cal.setTime(date); + + cal.setFirstDayOfWeek(Calendar.MONDAY); + cal.set(Calendar.DAY_OF_WEEK, Calendar.SUNDAY); + + return cal.getTime(); + } + + /** + * get first day of month + * + * @param date date + * @return first day of month + */ + public static Date getFirstDayOfMonth(Date date) { + Calendar cal = Calendar.getInstance(); + + cal.setTime(date); + cal.set(Calendar.DAY_OF_MONTH, 1); + + return cal.getTime(); + } + + /** + * get some hour of day + * + * @param date date + * @param offsetHour hours + * @return some hour of day + */ + public static Date getSomeHourOfDay(Date date, int offsetHour) { + Calendar cal = Calendar.getInstance(); + + cal.setTime(date); + cal.set(Calendar.HOUR_OF_DAY, cal.get(Calendar.HOUR_OF_DAY) + offsetHour); + cal.set(Calendar.MINUTE, 0); + cal.set(Calendar.SECOND, 0); + cal.set(Calendar.MILLISECOND, 0); + + return cal.getTime(); + } + + /** + * get last day of month + * + * @param date date + * @return get last day of month + */ + public static Date getLastDayOfMonth(Date date) { + Calendar cal = Calendar.getInstance(); + + cal.setTime(date); + + cal.add(Calendar.MONTH, 1); + cal.set(Calendar.DAY_OF_MONTH, 1); + cal.add(Calendar.DAY_OF_MONTH, -1); + + return cal.getTime(); + } + + /** + * return YYYY-MM-DD 00:00:00 + * + * @param inputDay date + * @return start day + */ + public static Date getStartOfDay(Date inputDay) { + Calendar cal = Calendar.getInstance(); + cal.setTime(inputDay); + cal.set(Calendar.HOUR_OF_DAY, 0); + cal.set(Calendar.MINUTE, 0); + cal.set(Calendar.SECOND, 0); + cal.set(Calendar.MILLISECOND, 0); + return cal.getTime(); + } + + /** + * return YYYY-MM-DD 23:59:59 + * + * @param inputDay day + * @return end of day + */ + public static Date getEndOfDay(Date inputDay) { + Calendar cal = Calendar.getInstance(); + cal.setTime(inputDay); + cal.set(Calendar.HOUR_OF_DAY, 23); + cal.set(Calendar.MINUTE, 59); + cal.set(Calendar.SECOND, 59); + cal.set(Calendar.MILLISECOND, 999); + return cal.getTime(); + } + + /** + * return YYYY-MM-DD 00:00:00 + * + * @param inputDay day + * @return start of hour + */ + public static Date getStartOfHour(Date inputDay) { + Calendar cal = Calendar.getInstance(); + cal.setTime(inputDay); + cal.set(Calendar.MINUTE, 0); + cal.set(Calendar.SECOND, 0); + cal.set(Calendar.MILLISECOND, 0); + return cal.getTime(); + } + + /** + * return YYYY-MM-DD 23:59:59 + * + * @param inputDay day + * @return end of hour + */ + public static Date getEndOfHour(Date inputDay) { + Calendar cal = Calendar.getInstance(); + cal.setTime(inputDay); + cal.set(Calendar.MINUTE, 59); + cal.set(Calendar.SECOND, 59); + cal.set(Calendar.MILLISECOND, 999); + return cal.getTime(); + } + + /** + * get current date + * + * @return current date + */ + public static Date getCurrentDate() { + return DateUtils.parse(DateUtils.getCurrentTime(), + Constants.YYYY_MM_DD_HH_MM_SS); + } + + /** + * get date + * + * @param date date + * @param calendarField calendarField + * @param amount amount + * @return date + */ + public static Date add(final Date date, final int calendarField, final int amount) { + if (date == null) { + throw new IllegalArgumentException("The date must not be null"); + } + final Calendar c = Calendar.getInstance(); + c.setTime(date); + c.add(calendarField, amount); + return c.getTime(); + } + + /** + * starting from the current time, get how many seconds are left before the target time. + * targetTime = baseTime + intervalSeconds + * + * @param baseTime base time + * @param intervalSeconds a period of time + * @return the number of seconds + */ + public static long getRemainTime(Date baseTime, long intervalSeconds) { + if (baseTime == null) { + return 0; + } + long usedTime = (System.currentTimeMillis() - baseTime.getTime()) / 1000; + return intervalSeconds - usedTime; + } +} diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/DependentUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/DependentUtils.java index 591c16db39..b8b6c1d13e 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/DependentUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/DependentUtils.java @@ -20,8 +20,6 @@ import org.apache.dolphinscheduler.common.enums.DependResult; import org.apache.dolphinscheduler.common.enums.DependentRelation; import org.apache.dolphinscheduler.common.model.DateInterval; import org.apache.dolphinscheduler.common.utils.dependent.DependentDateUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.util.ArrayList; import java.util.Date; @@ -29,32 +27,35 @@ import java.util.List; public class DependentUtils { - private static final Logger logger = LoggerFactory.getLogger(DependentUtils.class); + private DependentUtils() { + throw new UnsupportedOperationException("Construct DependentUtils"); + } public static DependResult getDependResultForRelation(DependentRelation relation, - List dependResultList){ + List dependResultList) { DependResult dependResult = DependResult.SUCCESS; - switch (relation){ + switch (relation) { case AND: - if(dependResultList.contains(DependResult.FAILED)){ + if (dependResultList.contains(DependResult.FAILED)) { dependResult = DependResult.FAILED; - } if(dependResultList.contains(DependResult.WAITING)){ + } + if (dependResultList.contains(DependResult.WAITING)) { dependResult = DependResult.WAITING; } break; case OR: - if(dependResultList.contains(DependResult.SUCCESS)){ + if (dependResultList.contains(DependResult.SUCCESS)) { dependResult = DependResult.SUCCESS; - }else if(dependResultList.contains(DependResult.WAITING)){ + } else if (dependResultList.contains(DependResult.WAITING)) { dependResult = DependResult.WAITING; - }else{ + } else { dependResult = DependResult.FAILED; } break; default: - break; + break; } return dependResult; } @@ -62,36 +63,37 @@ public class DependentUtils { /** * get date interval list by business date and date value. + * * @param businessDate business date * @param dateValue date value * @return date interval list by business date and date value. */ - public static List getDateIntervalList(Date businessDate, String dateValue){ + public static List getDateIntervalList(Date businessDate, String dateValue) { List result = new ArrayList<>(); - switch (dateValue){ + switch (dateValue) { case "currentHour": result = DependentDateUtils.getLastHoursInterval(businessDate, 0); break; case "last1Hour": - result = DependentDateUtils.getLastHoursInterval(businessDate, 1); + result = DependentDateUtils.getLastHoursInterval(businessDate, 1); break; case "last2Hours": - result = DependentDateUtils.getLastHoursInterval(businessDate, 2); + result = DependentDateUtils.getLastHoursInterval(businessDate, 2); break; case "last3Hours": - result = DependentDateUtils.getLastHoursInterval(businessDate, 3); + result = DependentDateUtils.getLastHoursInterval(businessDate, 3); break; case "last24Hours": result = DependentDateUtils.getSpecialLastDayInterval(businessDate); break; case "today": - result = DependentDateUtils.getTodayInterval(businessDate); + result = DependentDateUtils.getTodayInterval(businessDate); break; case "last1Days": - result = DependentDateUtils.getLastDayInterval(businessDate, 1); + result = DependentDateUtils.getLastDayInterval(businessDate, 1); break; case "last2Days": - result = DependentDateUtils.getLastDayInterval(businessDate, 2); + result = DependentDateUtils.getLastDayInterval(businessDate, 2); break; case "last3Days": result = DependentDateUtils.getLastDayInterval(businessDate, 3); @@ -144,5 +146,4 @@ public class DependentUtils { return result; } - } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/EncryptionUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/EncryptionUtils.java index c153ec817a..5d9d540d96 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/EncryptionUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/EncryptionUtils.java @@ -14,6 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.common.utils; import org.apache.commons.codec.digest.DigestUtils; @@ -23,14 +24,16 @@ import org.apache.commons.codec.digest.DigestUtils; */ public class EncryptionUtils { + private EncryptionUtils() { + throw new UnsupportedOperationException("Construct EncryptionUtils"); + } /** - * * @param rawStr raw string * @return md5(rawStr) */ public static String getMd5(String rawStr) { - return DigestUtils.md5Hex(null == rawStr ? StringUtils.EMPTY : rawStr); + return DigestUtils.md5Hex(null == rawStr ? StringUtils.EMPTY : rawStr); } } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/EnumUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/EnumUtils.java index 924e8ff719..10963b486a 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/EnumUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/EnumUtils.java @@ -14,12 +14,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.common.utils; - - public class EnumUtils { + private EnumUtils() { + throw new UnsupportedOperationException("Construct EnumUtils"); + } + public static > E getEnum(final Class enumClass, final String enumName) { if (enumName == null) { return null; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java index de3d42974a..e3e33566e9 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java @@ -14,6 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.common.utils; import static org.apache.dolphinscheduler.common.Constants.DATA_BASEDIR_PATH; @@ -21,6 +22,9 @@ import static org.apache.dolphinscheduler.common.Constants.RESOURCE_VIEW_SUFFIXS import static org.apache.dolphinscheduler.common.Constants.RESOURCE_VIEW_SUFFIXS_DEFAULT_VALUE; import static org.apache.dolphinscheduler.common.Constants.YYYYMMDDHHMMSS; +import org.apache.commons.io.Charsets; +import org.apache.commons.io.IOUtils; + import java.io.BufferedReader; import java.io.BufferedWriter; import java.io.ByteArrayOutputStream; @@ -36,9 +40,6 @@ import java.nio.charset.Charset; import java.nio.charset.UnsupportedCharsetException; import java.util.Optional; -import org.apache.commons.io.Charsets; -import org.apache.commons.io.IOUtils; -import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -46,12 +47,17 @@ import org.slf4j.LoggerFactory; * file utils */ public class FileUtils { + public static final Logger logger = LoggerFactory.getLogger(FileUtils.class); - public static final String DATA_BASEDIR = PropertyUtils.getString(DATA_BASEDIR_PATH,"/tmp/dolphinscheduler"); + public static final String DATA_BASEDIR = PropertyUtils.getString(DATA_BASEDIR_PATH, "/tmp/dolphinscheduler"); public static final ThreadLocal taskLoggerThreadLocal = new ThreadLocal<>(); + private FileUtils() { + throw new UnsupportedOperationException("Construct FileUtils"); + } + /** * get file suffix * @@ -80,7 +86,7 @@ public class FileUtils { String fileName = String.format("%s/download/%s/%s", DATA_BASEDIR, DateUtils.getCurrentTime(YYYYMMDDHHMMSS), filename); File file = new File(fileName); - if (!file.getParentFile().exists()){ + if (!file.getParentFile().exists()) { file.getParentFile().mkdirs(); } @@ -97,7 +103,7 @@ public class FileUtils { public static String getUploadFilename(String tenantCode, String filename) { String fileName = String.format("%s/%s/resources/%s", DATA_BASEDIR, tenantCode, filename); File file = new File(fileName); - if (!file.getParentFile().exists()){ + if (!file.getParentFile().exists()) { file.getParentFile().mkdirs(); } @@ -106,6 +112,7 @@ public class FileUtils { /** * directory of process execution + * * @param projectId project id * @param processDefineId process definition id * @param processInstanceId process instance id @@ -114,9 +121,9 @@ public class FileUtils { */ public static String getProcessExecDir(int projectId, int processDefineId, int processInstanceId, int taskInstanceId) { String fileName = String.format("%s/exec/process/%s/%s/%s/%s", DATA_BASEDIR, Integer.toString(projectId), - Integer.toString(processDefineId), Integer.toString(processInstanceId),Integer.toString(taskInstanceId)); + Integer.toString(processDefineId), Integer.toString(processInstanceId), Integer.toString(taskInstanceId)); File file = new File(fileName); - if (!file.getParentFile().exists()){ + if (!file.getParentFile().exists()) { file.getParentFile().mkdirs(); } @@ -125,6 +132,7 @@ public class FileUtils { /** * directory of process instances + * * @param projectId project id * @param processDefineId process definition id * @param processInstanceId process instance id @@ -150,6 +158,7 @@ public class FileUtils { /** * create directory and user + * * @param execLocalPath execute local path * @param userName user name * @throws IOException errors @@ -190,12 +199,11 @@ public class FileUtils { OSUtils.taskLoggerThreadLocal.remove(); } - /** * write content to file ,if parent path not exists, it will do one's utmost to mkdir * - * @param content content - * @param filePath target file path + * @param content content + * @param filePath target file path * @return true if write success */ public static boolean writeContent2File(String content, String filePath) { @@ -231,13 +239,13 @@ public class FileUtils { /** * Writes a String to a file creating the file if it does not exist. - * + *

* NOTE: As from v1.3, the parent directories of the file will be created * if they do not exist. * - * @param file the file to write - * @param data the content to write to the file - * @param encoding the encoding to use, {@code null} means platform default + * @param file the file to write + * @param data the content to write to the file + * @param encoding the encoding to use, {@code null} means platform default * @throws IOException in case of an I/O error * @throws java.io.UnsupportedEncodingException if the encoding is not supported by the VM * @since 2.4 @@ -248,13 +256,13 @@ public class FileUtils { /** * Writes a String to a file creating the file if it does not exist. - * + *

* NOTE: As from v1.3, the parent directories of the file will be created * if they do not exist. * - * @param file the file to write - * @param data the content to write to the file - * @param encoding the encoding to use, {@code null} means platform default + * @param file the file to write + * @param data the content to write to the file + * @param encoding the encoding to use, {@code null} means platform default * @throws IOException in case of an I/O error * @throws java.io.UnsupportedEncodingException if the encoding is not supported by the VM */ @@ -265,9 +273,9 @@ public class FileUtils { /** * Writes a String to a file creating the file if it does not exist. * - * @param file the file to write - * @param data the content to write to the file - * @param encoding the encoding to use, {@code null} means platform default + * @param file the file to write + * @param data the content to write to the file + * @param encoding the encoding to use, {@code null} means platform default * @param append if {@code true}, then the String will be added to the * end of the file rather than overwriting * @throws IOException in case of an I/O error @@ -287,15 +295,14 @@ public class FileUtils { /** * Writes a String to a file creating the file if it does not exist. * - * @param file the file to write - * @param data the content to write to the file - * @param encoding the encoding to use, {@code null} means platform default + * @param file the file to write + * @param data the content to write to the file + * @param encoding the encoding to use, {@code null} means platform default * @param append if {@code true}, then the String will be added to the * end of the file rather than overwriting * @throws IOException in case of an I/O error - * @throws UnsupportedCharsetException - * thrown instead of {@link UnsupportedEncodingException} in version 2.2 if the encoding is not - * supported by the VM + * @throws UnsupportedCharsetException thrown instead of {@link UnsupportedEncodingException} in version 2.2 if the encoding is not + * supported by the VM * @since 2.1 */ public static void writeStringToFile(File file, String data, String encoding, boolean append) throws IOException { @@ -305,8 +312,8 @@ public class FileUtils { /** * Writes a String to a file creating the file if it does not exist using the default encoding for the VM. * - * @param file the file to write - * @param data the content to write to the file + * @param file the file to write + * @param data the content to write to the file * @throws IOException in case of an I/O error */ public static void writeStringToFile(File file, String data) throws IOException { @@ -316,8 +323,8 @@ public class FileUtils { /** * Writes a String to a file creating the file if it does not exist using the default encoding for the VM. * - * @param file the file to write - * @param data the content to write to the file + * @param file the file to write + * @param data the content to write to the file * @param append if {@code true}, then the String will be added to the * end of the file rather than overwriting * @throws IOException in case of an I/O error @@ -340,7 +347,7 @@ public class FileUtils { * An exception is thrown if the file exists but cannot be written to. * An exception is thrown if the parent directory cannot be created. * - * @param file the file to open for output, must not be {@code null} + * @param file the file to open for output, must not be {@code null} * @return a new {@link FileOutputStream} for the specified file * @throws IOException if the file object is a directory * @throws IOException if the file cannot be written to @@ -364,7 +371,7 @@ public class FileUtils { * An exception is thrown if the file exists but cannot be written to. * An exception is thrown if the parent directory cannot be created. * - * @param file the file to open for output, must not be {@code null} + * @param file the file to open for output, must not be {@code null} * @param append if {@code true}, then bytes will be added to the * end of the file rather than overwriting * @return a new {@link FileOutputStream} for the specified file @@ -384,15 +391,15 @@ public class FileUtils { } else { File parent = file.getParentFile(); if (parent != null && !parent.mkdirs() && !parent.isDirectory()) { - throw new IOException("Directory '" + parent + "' could not be created"); + throw new IOException("Directory '" + parent + "' could not be created"); } } return new FileOutputStream(file, append); } - /** * deletes a directory recursively + * * @param dir directory * @throws IOException in case deletion is unsuccessful */ @@ -420,17 +427,18 @@ public class FileUtils { /** * Gets all the parent subdirectories of the parentDir directory + * * @param parentDir parent dir * @return all dirs */ - public static File[] getAllDir(String parentDir){ - if(parentDir == null || "".equals(parentDir)) { + public static File[] getAllDir(String parentDir) { + if (parentDir == null || "".equals(parentDir)) { throw new RuntimeException("parentDir can not be empty"); } File file = new File(parentDir); - if(!file.exists() || !file.isDirectory()) { - throw new RuntimeException("parentDir not exist, or is not a directory:"+parentDir); + if (!file.exists() || !file.isDirectory()) { + throw new RuntimeException("parentDir not exist, or is not a directory:" + parentDir); } return file.listFiles(File::isDirectory); @@ -438,6 +446,7 @@ public class FileUtils { /** * Get Content + * * @param inputStream input stream * @return string of input stream */ @@ -447,15 +456,14 @@ public class FileUtils { ByteArrayOutputStream output = new ByteArrayOutputStream(); byte[] buffer = new byte[1024]; int length; - while ((length= inputStream.read(buffer)) != -1) { - output.write(buffer,0,length); + while ((length = inputStream.read(buffer)) != -1) { + output.write(buffer, 0, length); } return output.toString(); } catch (Exception e) { - logger.error(e.getMessage(),e); + logger.error(e.getMessage(), e); throw new RuntimeException(e); } } - } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java index cf432a17d5..68d03506c6 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java @@ -195,7 +195,7 @@ public class HadoopUtils implements Closeable { */ String appUrl = ""; - if (StringUtils.isEmpty(rmHaIds)){ + if (StringUtils.isEmpty(rmHaIds)) { //single resourcemanager enabled appUrl = appAddress; yarnEnabled = true; @@ -206,7 +206,7 @@ public class HadoopUtils implements Closeable { logger.info("application url : {}", appUrl); } - if(StringUtils.isBlank(appUrl)){ + if (StringUtils.isBlank(appUrl)) { throw new Exception("application url is blank"); } return String.format(appUrl, applicationId); @@ -417,20 +417,33 @@ public class HadoopUtils implements Closeable { String applicationUrl = getApplicationUrl(applicationId); logger.info("applicationUrl={}", applicationUrl); - String responseContent = HttpUtils.get(applicationUrl); + String responseContent; + if (PropertyUtils.getBoolean(Constants.HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE, false)) { + responseContent = KerberosHttpClient.get(applicationUrl); + } else { + responseContent = HttpUtils.get(applicationUrl); + } if (responseContent != null) { ObjectNode jsonObject = JSONUtils.parseObject(responseContent); + if (!jsonObject.has("app")) { + return ExecutionStatus.FAILURE; + } result = jsonObject.path("app").path("finalStatus").asText(); + } else { //may be in job history String jobHistoryUrl = getJobHistoryUrl(applicationId); logger.info("jobHistoryUrl={}", jobHistoryUrl); responseContent = HttpUtils.get(jobHistoryUrl); - ObjectNode jsonObject = JSONUtils.parseObject(responseContent); - if (!jsonObject.has("job")){ + if (null != responseContent) { + ObjectNode jsonObject = JSONUtils.parseObject(responseContent); + if (!jsonObject.has("job")) { + return ExecutionStatus.FAILURE; + } + result = jsonObject.path("job").path("state").asText(); + } else { return ExecutionStatus.FAILURE; } - result = jsonObject.path("job").path("state").asText(); } switch (result) { @@ -469,7 +482,7 @@ public class HadoopUtils implements Closeable { /** * hdfs resource dir * - * @param tenantCode tenant code + * @param tenantCode tenant code * @param resourceType resource type * @return hdfs resource dir */ @@ -674,7 +687,7 @@ public class HadoopUtils implements Closeable { ObjectNode jsonObject = JSONUtils.parseObject(retStr); //get ResourceManager state - if (!jsonObject.has("clusterInfo")){ + if (!jsonObject.has("clusterInfo")) { return null; } return jsonObject.get("clusterInfo").path("haState").asText(); diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HttpUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HttpUtils.java index 8ea15314a8..36b437f312 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HttpUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HttpUtils.java @@ -14,9 +14,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.common.utils; import org.apache.dolphinscheduler.common.Constants; + import org.apache.http.HttpEntity; import org.apache.http.client.config.AuthSchemes; import org.apache.http.client.config.CookieSpecs; @@ -30,138 +32,148 @@ import org.apache.http.conn.socket.PlainConnectionSocketFactory; import org.apache.http.conn.ssl.NoopHostnameVerifier; import org.apache.http.conn.ssl.SSLConnectionSocketFactory; import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.DefaultHttpClient; import org.apache.http.impl.client.HttpClients; import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; import org.apache.http.util.EntityUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import javax.net.ssl.SSLContext; -import javax.net.ssl.TrustManager; -import javax.net.ssl.X509TrustManager; import java.io.IOException; import java.security.KeyManagementException; import java.security.NoSuchAlgorithmException; import java.security.cert.X509Certificate; import java.util.Arrays; + +import javax.net.ssl.SSLContext; +import javax.net.ssl.TrustManager; +import javax.net.ssl.X509TrustManager; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + /** * http utils */ public class HttpUtils { + public static final Logger logger = LoggerFactory.getLogger(HttpUtils.class); - public static final Logger logger = LoggerFactory.getLogger(HttpUtils.class); + private HttpUtils() { + throw new UnsupportedOperationException("Construct HttpUtils"); + } - private HttpUtils() { + public static CloseableHttpClient getInstance() { + return HttpClientInstance.httpClient; + } - } - - public static CloseableHttpClient getInstance(){ - return HttpClientInstance.httpClient; - } - - private static class HttpClientInstance{ - private static final CloseableHttpClient httpClient = HttpClients.custom().setConnectionManager(cm).setDefaultRequestConfig(requestConfig).build(); - } + private static class HttpClientInstance { + private static final CloseableHttpClient httpClient = HttpClients.custom().setConnectionManager(cm).setDefaultRequestConfig(requestConfig).build(); + } - private static PoolingHttpClientConnectionManager cm; + private static PoolingHttpClientConnectionManager cm; - private static SSLContext ctx = null; + private static SSLContext ctx = null; - private static SSLConnectionSocketFactory socketFactory; + private static SSLConnectionSocketFactory socketFactory; - private static RequestConfig requestConfig; + private static RequestConfig requestConfig; - private static Registry socketFactoryRegistry; + private static Registry socketFactoryRegistry; - private static X509TrustManager xtm = new X509TrustManager() { - @Override - public void checkClientTrusted(X509Certificate[] chain, String authType) { - } + private static X509TrustManager xtm = new X509TrustManager() { + @Override + public void checkClientTrusted(X509Certificate[] chain, String authType) { + } - @Override - public void checkServerTrusted(X509Certificate[] chain, String authType) { - } + @Override + public void checkServerTrusted(X509Certificate[] chain, String authType) { + } - @Override - public X509Certificate[] getAcceptedIssuers() { - return null; - } - }; + @Override + public X509Certificate[] getAcceptedIssuers() { + return null; + } + }; - static { - try { - ctx = SSLContext.getInstance(SSLConnectionSocketFactory.TLS); - ctx.init(null, new TrustManager[] { xtm }, null); - } catch (NoSuchAlgorithmException e) { - logger.error("SSLContext init with NoSuchAlgorithmException", e); - } catch (KeyManagementException e) { - logger.error("SSLContext init with KeyManagementException", e); - } - socketFactory = new SSLConnectionSocketFactory(ctx, NoopHostnameVerifier.INSTANCE); - /** set timeout、request time、socket timeout */ - requestConfig = RequestConfig.custom().setCookieSpec(CookieSpecs.IGNORE_COOKIES) - .setExpectContinueEnabled(Boolean.TRUE) - .setTargetPreferredAuthSchemes(Arrays.asList(AuthSchemes.NTLM, AuthSchemes.DIGEST)) - .setProxyPreferredAuthSchemes(Arrays.asList(AuthSchemes.BASIC)) - .setConnectTimeout(Constants.HTTP_CONNECT_TIMEOUT).setSocketTimeout(Constants.SOCKET_TIMEOUT) - .setConnectionRequestTimeout(Constants.HTTP_CONNECTION_REQUEST_TIMEOUT).setRedirectsEnabled(true) - .build(); - socketFactoryRegistry = RegistryBuilder.create() - .register("http", PlainConnectionSocketFactory.INSTANCE).register("https", socketFactory).build(); - cm = new PoolingHttpClientConnectionManager(socketFactoryRegistry); - cm.setDefaultMaxPerRoute(60); - cm.setMaxTotal(100); + static { + try { + ctx = SSLContext.getInstance(SSLConnectionSocketFactory.TLS); + ctx.init(null, new TrustManager[]{xtm}, null); + } catch (NoSuchAlgorithmException e) { + logger.error("SSLContext init with NoSuchAlgorithmException", e); + } catch (KeyManagementException e) { + logger.error("SSLContext init with KeyManagementException", e); + } + socketFactory = new SSLConnectionSocketFactory(ctx, NoopHostnameVerifier.INSTANCE); + /** set timeout、request time、socket timeout */ + requestConfig = RequestConfig.custom().setCookieSpec(CookieSpecs.IGNORE_COOKIES) + .setExpectContinueEnabled(Boolean.TRUE) + .setTargetPreferredAuthSchemes(Arrays.asList(AuthSchemes.NTLM, AuthSchemes.DIGEST)) + .setProxyPreferredAuthSchemes(Arrays.asList(AuthSchemes.BASIC)) + .setConnectTimeout(Constants.HTTP_CONNECT_TIMEOUT).setSocketTimeout(Constants.SOCKET_TIMEOUT) + .setConnectionRequestTimeout(Constants.HTTP_CONNECTION_REQUEST_TIMEOUT).setRedirectsEnabled(true) + .build(); + socketFactoryRegistry = RegistryBuilder.create() + .register("http", PlainConnectionSocketFactory.INSTANCE).register("https", socketFactory).build(); + cm = new PoolingHttpClientConnectionManager(socketFactoryRegistry); + cm.setDefaultMaxPerRoute(60); + cm.setMaxTotal(100); - } + } + /** + * get http request content + * + * @param url url + * @return http get request response content + */ + public static String get(String url) { + CloseableHttpClient httpclient = HttpUtils.getInstance(); + HttpGet httpget = new HttpGet(url); + return getResponseContentString(httpget, httpclient); + } - /** - * get http request content - * @param url url - * @return http get request response content - */ - public static String get(String url){ - CloseableHttpClient httpclient = HttpUtils.getInstance(); + /** + * get http response content + * + * @param httpget httpget + * @param httpClient httpClient + * @return http get request response content + */ + public static String getResponseContentString(HttpGet httpget, CloseableHttpClient httpClient) { + String responseContent = null; + CloseableHttpResponse response = null; + try { + response = httpClient.execute(httpget); + // check response status is 200 + if (response.getStatusLine().getStatusCode() == 200) { + HttpEntity entity = response.getEntity(); + if (entity != null) { + responseContent = EntityUtils.toString(entity, Constants.UTF_8); + } else { + logger.warn("http entity is null"); + } + } else { + logger.error("http get:{} response status code is not 200!", response.getStatusLine().getStatusCode()); + } + } catch (IOException ioe) { + logger.error(ioe.getMessage(), ioe); + } finally { + try { + if (response != null) { + EntityUtils.consume(response.getEntity()); + response.close(); + } + } catch (IOException e) { + logger.error(e.getMessage(), e); + } + if (!httpget.isAborted()) { + httpget.releaseConnection(); + httpget.abort(); + } - HttpGet httpget = new HttpGet(url); - String responseContent = null; - CloseableHttpResponse response = null; - - try { - response = httpclient.execute(httpget); - //check response status is 200 - if (response.getStatusLine().getStatusCode() == 200) { - HttpEntity entity = response.getEntity(); - if (entity != null) { - responseContent = EntityUtils.toString(entity, Constants.UTF_8); - }else{ - logger.warn("http entity is null"); - } - }else{ - logger.error("http get:{} response status code is not 200!", response.getStatusLine().getStatusCode()); - } - }catch (Exception e){ - logger.error(e.getMessage(),e); - }finally { - try { - if (response != null) { - EntityUtils.consume(response.getEntity()); - response.close(); - } - } catch (IOException e) { - logger.error(e.getMessage(),e); - } - - if (!httpget.isAborted()) { - httpget.releaseConnection(); - httpget.abort(); - } - } - return responseContent; - } + } + return responseContent; + } } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/IOUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/IOUtils.java index ce551d8405..96366d539f 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/IOUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/IOUtils.java @@ -1,4 +1,3 @@ - /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with @@ -18,14 +17,17 @@ package org.apache.dolphinscheduler.common.utils; - import java.io.Closeable; import java.io.IOException; public class IOUtils { - public static void closeQuietly(Closeable closeable){ - if(closeable != null){ + private IOUtils() { + throw new UnsupportedOperationException("Construct IOUtils"); + } + + public static void closeQuietly(Closeable closeable) { + if (closeable != null) { try { closeable.close(); } catch (IOException ignore) { diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/IpUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/IpUtils.java index 858e5b44b5..63d43e7b69 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/IpUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/IpUtils.java @@ -14,46 +14,50 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.dolphinscheduler.common.utils; +package org.apache.dolphinscheduler.common.utils; /** * http utils */ public class IpUtils { - public static final String DOT = "."; + private IpUtils() { + throw new UnsupportedOperationException("Construct IpUtils"); + } - /** - * ip str to long

- * - * @param ipStr ip string - * @return ip to long - */ - public static Long ipToLong(String ipStr) { - String[] ipSet = ipStr.split("\\" + DOT); + public static final String DOT = "."; - return Long.parseLong(ipSet[0]) << 24 | Long.parseLong(ipSet[1]) << 16 | Long.parseLong(ipSet[2]) << 8 | Long.parseLong(ipSet[3]); - } + /** + * ip str to long

+ * + * @param ipStr ip string + * @return ip to long + */ + public static Long ipToLong(String ipStr) { + String[] ipSet = ipStr.split("\\" + DOT); - /** - * long to ip - * @param ipLong the long number converted from IP - * @return String - */ - public static String longToIp(long ipLong) { - long[] ipNumbers = new long[4]; - long tmp = 0xFF; - ipNumbers[0] = ipLong >> 24 & tmp; - ipNumbers[1] = ipLong >> 16 & tmp; - ipNumbers[2] = ipLong >> 8 & tmp; - ipNumbers[3] = ipLong & tmp; + return Long.parseLong(ipSet[0]) << 24 | Long.parseLong(ipSet[1]) << 16 | Long.parseLong(ipSet[2]) << 8 | Long.parseLong(ipSet[3]); + } - String sb = ipNumbers[0] + DOT + - ipNumbers[1] + DOT + - ipNumbers[2] + DOT + - ipNumbers[3]; - return sb; - } + /** + * long to ip + * + * @param ipLong the long number converted from IP + * @return String + */ + public static String longToIp(long ipLong) { + long[] ipNumbers = new long[4]; + long tmp = 0xFF; + ipNumbers[0] = ipLong >> 24 & tmp; + ipNumbers[1] = ipLong >> 16 & tmp; + ipNumbers[2] = ipLong >> 8 & tmp; + ipNumbers[3] = ipLong & tmp; + + return ipNumbers[0] + DOT + + ipNumbers[1] + DOT + + ipNumbers[2] + DOT + + ipNumbers[3]; + } } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java index 65a078778e..56ef74d6ee 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java @@ -14,25 +14,39 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.common.utils; +import static com.fasterxml.jackson.databind.DeserializationFeature.ACCEPT_EMPTY_ARRAY_AS_NULL_OBJECT; +import static com.fasterxml.jackson.databind.DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES; +import static com.fasterxml.jackson.databind.DeserializationFeature.READ_UNKNOWN_ENUM_VALUES_AS_NULL; +import static com.fasterxml.jackson.databind.MapperFeature.REQUIRE_SETTERS_FOR_GETTERS; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.TimeZone; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.*; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonDeserializer; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.JsonSerializer; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectWriter; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.databind.SerializerProvider; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.fasterxml.jackson.databind.node.TextNode; import com.fasterxml.jackson.databind.type.CollectionType; -import org.apache.commons.lang.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.util.*; - -import static com.fasterxml.jackson.databind.DeserializationFeature.*; - /** * json utils @@ -48,13 +62,13 @@ public class JSONUtils { .configure(FAIL_ON_UNKNOWN_PROPERTIES, false) .configure(ACCEPT_EMPTY_ARRAY_AS_NULL_OBJECT, true) .configure(READ_UNKNOWN_ENUM_VALUES_AS_NULL, true) - .setTimeZone(TimeZone.getDefault()) - ; + .configure(REQUIRE_SETTERS_FOR_GETTERS, true) + .setTimeZone(TimeZone.getDefault()); private JSONUtils() { + throw new UnsupportedOperationException("Construct JSONUtils"); } - public static ArrayNode createArrayNode() { return objectMapper.createArrayNode(); } @@ -93,9 +107,9 @@ public class JSONUtils { * the fields of the specified object are generics, just the object itself should not be a * generic type. * - * @param json the string from which the object is to be deserialized + * @param json the string from which the object is to be deserialized * @param clazz the class of T - * @param T + * @param T * @return an object of type T from the string * classOfT */ @@ -115,9 +129,9 @@ public class JSONUtils { /** * json to list * - * @param json json string + * @param json json string * @param clazz class - * @param T + * @param T * @return list */ public static List toList(String json, Class clazz) { @@ -136,7 +150,6 @@ public class JSONUtils { return Collections.emptyList(); } - /** * check json object valid * @@ -159,13 +172,12 @@ public class JSONUtils { return false; } - /** * Method for finding a JSON Object field with specified name in this * node or its child nodes, and returning value it has. * If no matching field is found in this node or its descendants, returns null. * - * @param jsonNode json node + * @param jsonNode json node * @param fieldName Name of field to look for * @return Value of first matching node found, if any; null if none */ @@ -179,7 +191,6 @@ public class JSONUtils { return node.toString(); } - /** * json to map *

@@ -194,7 +205,8 @@ public class JSONUtils { } try { - return objectMapper.readValue(json, new TypeReference>() {}); + return objectMapper.readValue(json, new TypeReference>() { + }); } catch (Exception e) { logger.error("json to map exception!", e); } @@ -205,11 +217,11 @@ public class JSONUtils { /** * json to map * - * @param json json + * @param json json * @param classK classK * @param classV classV - * @param K - * @param V + * @param K + * @param V * @return to map */ public static Map toMap(String json, Class classK, Class classV) { @@ -257,7 +269,6 @@ public class JSONUtils { } } - /** * json serializer */ diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/KerberosHttpClient.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/KerberosHttpClient.java new file mode 100644 index 0000000000..5c1fd41900 --- /dev/null +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/KerberosHttpClient.java @@ -0,0 +1,156 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.common.utils; + +import org.apache.dolphinscheduler.common.Constants; +import org.apache.http.auth.AuthSchemeProvider; +import org.apache.http.auth.AuthScope; +import org.apache.http.auth.Credentials; +import org.apache.http.client.config.AuthSchemes; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.config.Lookup; +import org.apache.http.config.RegistryBuilder; +import org.apache.http.impl.auth.SPNegoSchemeFactory; +import org.apache.http.impl.client.BasicCredentialsProvider; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClientBuilder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.security.auth.Subject; +import javax.security.auth.kerberos.KerberosPrincipal; +import javax.security.auth.login.AppConfigurationEntry; +import javax.security.auth.login.Configuration; +import javax.security.auth.login.LoginContext; +import javax.security.auth.login.LoginException; +import java.security.Principal; +import java.security.PrivilegedAction; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +/** + * kerberos http client + */ +public class KerberosHttpClient { + + public static final Logger logger = LoggerFactory.getLogger(KerberosHttpClient.class); + + private String principal; + private String keyTabLocation; + + public KerberosHttpClient(String principal, String keyTabLocation) { + super(); + this.principal = principal; + this.keyTabLocation = keyTabLocation; + } + + public KerberosHttpClient(String principal, String keyTabLocation, boolean isDebug) { + this(principal, keyTabLocation); + if (isDebug) { + System.setProperty("sun.security.spnego.debug", "true"); + System.setProperty("sun.security.krb5.debug", "true"); + } + } + + public KerberosHttpClient(String principal, String keyTabLocation, String krb5Location, boolean isDebug) { + this(principal, keyTabLocation, isDebug); + System.setProperty("java.security.krb5.conf", krb5Location); + } + + private static CloseableHttpClient buildSpengoHttpClient() { + HttpClientBuilder builder = HttpClientBuilder.create(); + Lookup authSchemeRegistry = RegistryBuilder.create() + .register(AuthSchemes.SPNEGO, new SPNegoSchemeFactory(true)).build(); + builder.setDefaultAuthSchemeRegistry(authSchemeRegistry); + BasicCredentialsProvider credentialsProvider = new BasicCredentialsProvider(); + credentialsProvider.setCredentials(new AuthScope(null, -1, null), new Credentials() { + @Override + public Principal getUserPrincipal() { + return null; + } + + @Override + public String getPassword() { + return null; + } + }); + builder.setDefaultCredentialsProvider(credentialsProvider); + return builder.build(); + } + + public String get(final String url, final String userId) { + logger.info("Calling KerberosHttpClient {} {} {}", this.principal, this.keyTabLocation, url); + Configuration config = new Configuration() { + @SuppressWarnings("serial") + @Override + public AppConfigurationEntry[] getAppConfigurationEntry(String name) { + Map options = new HashMap<>(9); + options.put("useTicketCache", "false"); + options.put("useKeyTab", "true"); + options.put("keyTab", keyTabLocation); + options.put("refreshKrb5Config", "true"); + options.put("principal", principal); + options.put("storeKey", "true"); + options.put("doNotPrompt", "true"); + options.put("isInitiator", "true"); + options.put("debug", "true"); + return new AppConfigurationEntry[] { + new AppConfigurationEntry("com.sun.security.auth.module.Krb5LoginModule", + AppConfigurationEntry.LoginModuleControlFlag.REQUIRED, options) }; + } + }; + Set princ = new HashSet<>(1); + princ.add(new KerberosPrincipal(userId)); + Subject sub = new Subject(false, princ, new HashSet<>(), new HashSet<>()); + + LoginContext lc; + try { + lc = new LoginContext("", sub, null, config); + lc.login(); + Subject serviceSubject = lc.getSubject(); + return Subject.doAs(serviceSubject, (PrivilegedAction) () -> { + CloseableHttpClient httpClient = buildSpengoHttpClient(); + HttpGet httpget = new HttpGet(url); + return HttpUtils.getResponseContentString(httpget, httpClient); + }); + } catch (LoginException le) { + logger.error("Kerberos authentication failed ", le); + } + return null; + } + + /** + * get http request content by kerberosClient + * + * @param url url + * @return http get request response content + */ + public static String get(String url) { + + String responseContent; + KerberosHttpClient kerberosHttpClient = new KerberosHttpClient( + PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_USERNAME), + PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_PATH), + PropertyUtils.getString(Constants.JAVA_SECURITY_KRB5_CONF_PATH), true); + responseContent = kerberosHttpClient.get(url, PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_USERNAME)); + return responseContent; + + } + +} diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/LoggerUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/LoggerUtils.java index e3cf652efb..211f0a08a8 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/LoggerUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/LoggerUtils.java @@ -14,22 +14,28 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.common.utils; +import org.apache.dolphinscheduler.common.Constants; + import java.util.ArrayList; import java.util.List; import java.util.Optional; import java.util.regex.Matcher; import java.util.regex.Pattern; -import org.apache.dolphinscheduler.common.Constants; import org.slf4j.Logger; /** - * logger utils + * logger utils */ public class LoggerUtils { + private LoggerUtils() { + throw new UnsupportedOperationException("Construct LoggerUtils"); + } + /** * rules for extracting application ID */ @@ -53,29 +59,29 @@ public class LoggerUtils { /** * build job id * - * @param affix Task Logger's prefix - * @param processDefId process define id + * @param affix Task Logger's prefix + * @param processDefId process define id * @param processInstId process instance id - * @param taskId task id + * @param taskId task id * @return task id format */ public static String buildTaskId(String affix, - int processDefId, - int processInstId, - int taskId){ + int processDefId, + int processInstId, + int taskId) { // - [taskAppId=TASK_79_4084_15210] - return String.format(" - %s%s-%s-%s-%s]",TASK_APPID_LOG_FORMAT,affix, + return String.format(" - %s%s-%s-%s-%s]", TASK_APPID_LOG_FORMAT, affix, processDefId, processInstId, taskId); } - /** * processing log * get yarn application id list - * @param log log content - * @param logger logger + * + * @param log log content + * @param logger logger * @return app id list */ public static List getAppIds(String log, Logger logger) { @@ -87,7 +93,7 @@ public class LoggerUtils { // analyse logs to get all submit yarn application id while (matcher.find()) { String appId = matcher.group(); - if(!appIds.contains(appId)){ + if (!appIds.contains(appId)) { logger.info("find app id: {}", appId); appIds.add(appId); } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/NetUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/NetUtils.java index 13a25dc636..b001825ce1 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/NetUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/NetUtils.java @@ -14,27 +14,36 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.common.utils; +import static org.apache.dolphinscheduler.common.Constants.DOLPHIN_SCHEDULER_PREFERRED_NETWORK_INTERFACE; + +import static java.util.Collections.emptyList; + +import java.io.IOException; +import java.net.Inet6Address; +import java.net.InetAddress; +import java.net.NetworkInterface; +import java.net.SocketException; +import java.net.UnknownHostException; +import java.util.Enumeration; +import java.util.LinkedList; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.regex.Pattern; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; -import java.net.*; -import java.util.*; -import java.util.regex.Pattern; - -import static java.util.Collections.emptyList; -import static org.apache.dolphinscheduler.common.Constants.DOLPHIN_SCHEDULER_PREFERRED_NETWORK_INTERFACE; - /** * NetUtils */ public class NetUtils { - private NetUtils() { - throw new IllegalStateException("Utility class"); + throw new UnsupportedOperationException("Construct NetUtils"); } private static Logger logger = LoggerFactory.getLogger(NetUtils.class); diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java index c3fcb50602..08e092d20d 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java @@ -14,8 +14,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.common.utils; +import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.shell.ShellExecutor; + +import org.apache.commons.configuration.Configuration; + import java.io.BufferedReader; import java.io.FileInputStream; import java.io.IOException; @@ -32,9 +38,6 @@ import java.util.Optional; import java.util.StringTokenizer; import java.util.regex.Pattern; -import org.apache.commons.configuration.Configuration; -import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.common.shell.ShellExecutor; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -45,431 +48,444 @@ import oshi.hardware.HardwareAbstractionLayer; /** * os utils - * */ public class OSUtils { - private static final Logger logger = LoggerFactory.getLogger(OSUtils.class); + private static final Logger logger = LoggerFactory.getLogger(OSUtils.class); - public static final ThreadLocal taskLoggerThreadLocal = new ThreadLocal<>(); + public static final ThreadLocal taskLoggerThreadLocal = new ThreadLocal<>(); - private static final SystemInfo SI = new SystemInfo(); - public static final String TWO_DECIMAL = "0.00"; + private static final SystemInfo SI = new SystemInfo(); + public static final String TWO_DECIMAL = "0.00"; - /** - * return -1 when the function can not get hardware env info - * e.g {@link OSUtils#loadAverage()} {@link OSUtils#cpuUsage()} - */ - public static final double NEGATIVE_ONE = -1; + /** + * return -1 when the function can not get hardware env info + * e.g {@link OSUtils#loadAverage()} {@link OSUtils#cpuUsage()} + */ + public static final double NEGATIVE_ONE = -1; - private static HardwareAbstractionLayer hal = SI.getHardware(); + private static HardwareAbstractionLayer hal = SI.getHardware(); - private OSUtils() {} - - /** - * Initialization regularization, solve the problem of pre-compilation performance, - * avoid the thread safety problem of multi-thread operation - */ - private static final Pattern PATTERN = Pattern.compile("\\s+"); - - - /** - * get memory usage - * Keep 2 decimal - * @return percent % - */ - public static double memoryUsage() { - GlobalMemory memory = hal.getMemory(); - double memoryUsage = (memory.getTotal() - memory.getAvailable() - memory.getSwapUsed()) * 0.1 / memory.getTotal() * 10; - - DecimalFormat df = new DecimalFormat(TWO_DECIMAL); - df.setRoundingMode(RoundingMode.HALF_UP); - return Double.parseDouble(df.format(memoryUsage)); - } - - - /** - * get available physical memory size - * - * Keep 2 decimal - * @return available Physical Memory Size, unit: G - */ - public static double availablePhysicalMemorySize() { - GlobalMemory memory = hal.getMemory(); - double availablePhysicalMemorySize = (memory.getAvailable() + memory.getSwapUsed()) /1024.0/1024/1024; - - DecimalFormat df = new DecimalFormat(TWO_DECIMAL); - df.setRoundingMode(RoundingMode.HALF_UP); - return Double.parseDouble(df.format(availablePhysicalMemorySize)); - - } - - /** - * get total physical memory size - * - * Keep 2 decimal - * @return available Physical Memory Size, unit: G - */ - public static double totalMemorySize() { - GlobalMemory memory = hal.getMemory(); - double availablePhysicalMemorySize = memory.getTotal() /1024.0/1024/1024; - - DecimalFormat df = new DecimalFormat(TWO_DECIMAL); - df.setRoundingMode(RoundingMode.HALF_UP); - return Double.parseDouble(df.format(availablePhysicalMemorySize)); - } - - - /** - * load average - * - * @return load average - */ - public static double loadAverage() { - double loadAverage = hal.getProcessor().getSystemLoadAverage(); - if (Double.isNaN(loadAverage)) { - return NEGATIVE_ONE; + private OSUtils() { + throw new UnsupportedOperationException("Construct OSUtils"); } - DecimalFormat df = new DecimalFormat(TWO_DECIMAL); - df.setRoundingMode(RoundingMode.HALF_UP); - return Double.parseDouble(df.format(loadAverage)); - } + /** + * Initialization regularization, solve the problem of pre-compilation performance, + * avoid the thread safety problem of multi-thread operation + */ + private static final Pattern PATTERN = Pattern.compile("\\s+"); - /** - * get cpu usage - * - * @return cpu usage - */ - public static double cpuUsage() { - CentralProcessor processor = hal.getProcessor(); - double cpuUsage = processor.getSystemCpuLoad(); - if (Double.isNaN(cpuUsage)) { - return NEGATIVE_ONE; + /** + * get memory usage + * Keep 2 decimal + * + * @return percent % + */ + public static double memoryUsage() { + GlobalMemory memory = hal.getMemory(); + double memoryUsage = (memory.getTotal() - memory.getAvailable() - memory.getSwapUsed()) * 0.1 / memory.getTotal() * 10; + + DecimalFormat df = new DecimalFormat(TWO_DECIMAL); + df.setRoundingMode(RoundingMode.HALF_UP); + return Double.parseDouble(df.format(memoryUsage)); } - DecimalFormat df = new DecimalFormat(TWO_DECIMAL); - df.setRoundingMode(RoundingMode.HALF_UP); - return Double.parseDouble(df.format(cpuUsage)); - } + /** + * get available physical memory size + *

+ * Keep 2 decimal + * + * @return available Physical Memory Size, unit: G + */ + public static double availablePhysicalMemorySize() { + GlobalMemory memory = hal.getMemory(); + double availablePhysicalMemorySize = (memory.getAvailable() + memory.getSwapUsed()) / 1024.0 / 1024 / 1024; + + DecimalFormat df = new DecimalFormat(TWO_DECIMAL); + df.setRoundingMode(RoundingMode.HALF_UP); + return Double.parseDouble(df.format(availablePhysicalMemorySize)); - public static List getUserList() { - try { - if (isMacOS()) { - return getUserListFromMac(); - } else if (isWindows()) { - return getUserListFromWindows(); - } else { - return getUserListFromLinux(); - } - } catch (Exception e) { - logger.error(e.getMessage(), e); } - return Collections.emptyList(); - } + /** + * get total physical memory size + *

+ * Keep 2 decimal + * + * @return available Physical Memory Size, unit: G + */ + public static double totalMemorySize() { + GlobalMemory memory = hal.getMemory(); + double availablePhysicalMemorySize = memory.getTotal() / 1024.0 / 1024 / 1024; - /** - * get user list from linux - * - * @return user list - */ - private static List getUserListFromLinux() throws IOException { - List userList = new ArrayList<>(); + DecimalFormat df = new DecimalFormat(TWO_DECIMAL); + df.setRoundingMode(RoundingMode.HALF_UP); + return Double.parseDouble(df.format(availablePhysicalMemorySize)); + } - try (BufferedReader bufferedReader = new BufferedReader( - new InputStreamReader(new FileInputStream("/etc/passwd")))) { - String line; - - while ((line = bufferedReader.readLine()) != null) { - if (line.contains(":")) { - String[] userInfo = line.split(":"); - userList.add(userInfo[0]); + /** + * load average + * + * @return load average + */ + public static double loadAverage() { + double loadAverage = hal.getProcessor().getSystemLoadAverage(); + if (Double.isNaN(loadAverage)) { + return NEGATIVE_ONE; } - } + + DecimalFormat df = new DecimalFormat(TWO_DECIMAL); + df.setRoundingMode(RoundingMode.HALF_UP); + return Double.parseDouble(df.format(loadAverage)); } - return userList; - } - - /** - * get user list from mac - * @return user list - */ - private static List getUserListFromMac() throws IOException { - String result = exeCmd("dscl . list /users"); - if (StringUtils.isNotEmpty(result)) { - return Arrays.asList(result.split( "\n")); - } - - return Collections.emptyList(); - } - - /** - * get user list from windows - * @return user list - * @throws IOException - */ - private static List getUserListFromWindows() throws IOException { - String result = exeCmd("net user"); - String[] lines = result.split("\n"); - - int startPos = 0; - int endPos = lines.length - 2; - for (int i = 0; i < lines.length; i++) { - if (lines[i].isEmpty()) { - continue; - } - - int count = 0; - if (lines[i].charAt(0) == '-') { - for (int j = 0; j < lines[i].length(); j++) { - if (lines[i].charAt(i) == '-') { - count++; - } + /** + * get cpu usage + * + * @return cpu usage + */ + public static double cpuUsage() { + CentralProcessor processor = hal.getProcessor(); + double cpuUsage = processor.getSystemCpuLoad(); + if (Double.isNaN(cpuUsage)) { + return NEGATIVE_ONE; } - } - if (count == lines[i].length()) { - startPos = i + 1; - break; - } + DecimalFormat df = new DecimalFormat(TWO_DECIMAL); + df.setRoundingMode(RoundingMode.HALF_UP); + return Double.parseDouble(df.format(cpuUsage)); } - List users = new ArrayList<>(); - while (startPos <= endPos) { - users.addAll(Arrays.asList(PATTERN.split(lines[startPos]))); - startPos++; + public static List getUserList() { + try { + if (isMacOS()) { + return getUserListFromMac(); + } else if (isWindows()) { + return getUserListFromWindows(); + } else { + return getUserListFromLinux(); + } + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + + return Collections.emptyList(); } - return users; - } + /** + * get user list from linux + * + * @return user list + */ + private static List getUserListFromLinux() throws IOException { + List userList = new ArrayList<>(); + + try (BufferedReader bufferedReader = new BufferedReader( + new InputStreamReader(new FileInputStream("/etc/passwd")))) { + String line; + + while ((line = bufferedReader.readLine()) != null) { + if (line.contains(":")) { + String[] userInfo = line.split(":"); + userList.add(userInfo[0]); + } + } + } + + return userList; + } + + /** + * get user list from mac + * + * @return user list + */ + private static List getUserListFromMac() throws IOException { + String result = exeCmd("dscl . list /users"); + if (StringUtils.isNotEmpty(result)) { + return Arrays.asList(result.split("\n")); + } + + return Collections.emptyList(); + } + + /** + * get user list from windows + * + * @return user list + */ + private static List getUserListFromWindows() throws IOException { + String result = exeCmd("net user"); + String[] lines = result.split("\n"); + + int startPos = 0; + int endPos = lines.length - 2; + for (int i = 0; i < lines.length; i++) { + if (lines[i].isEmpty()) { + continue; + } + + int count = 0; + if (lines[i].charAt(0) == '-') { + for (int j = 0; j < lines[i].length(); j++) { + if (lines[i].charAt(i) == '-') { + count++; + } + } + } + + if (count == lines[i].length()) { + startPos = i + 1; + break; + } + } + + List users = new ArrayList<>(); + while (startPos <= endPos) { + users.addAll(Arrays.asList(PATTERN.split(lines[startPos]))); + startPos++; + } + + return users; + } + + /** + * create user + * + * @param userName user name + * @return true if creation was successful, otherwise false + */ + public static boolean createUser(String userName) { + try { + String userGroup = OSUtils.getGroup(); + if (StringUtils.isEmpty(userGroup)) { + String errorLog = String.format("%s group does not exist for this operating system.", userGroup); + LoggerUtils.logError(Optional.ofNullable(logger), errorLog); + LoggerUtils.logError(Optional.ofNullable(taskLoggerThreadLocal.get()), errorLog); + return false; + } + if (isMacOS()) { + createMacUser(userName, userGroup); + } else if (isWindows()) { + createWindowsUser(userName, userGroup); + } else { + createLinuxUser(userName, userGroup); + } + return true; + } catch (Exception e) { + LoggerUtils.logError(Optional.ofNullable(logger), e); + LoggerUtils.logError(Optional.ofNullable(taskLoggerThreadLocal.get()), e); + } - /** - * create user - * @param userName user name - * @return true if creation was successful, otherwise false - */ - public static boolean createUser(String userName) { - try { - String userGroup = OSUtils.getGroup(); - if (StringUtils.isEmpty(userGroup)) { - String errorLog = String.format("%s group does not exist for this operating system.", userGroup); - LoggerUtils.logError(Optional.ofNullable(logger), errorLog); - LoggerUtils.logError(Optional.ofNullable(taskLoggerThreadLocal.get()), errorLog); return false; - } - if (isMacOS()) { - createMacUser(userName, userGroup); - } else if (isWindows()) { - createWindowsUser(userName, userGroup); - } else { - createLinuxUser(userName, userGroup); - } - return true; - } catch (Exception e) { - LoggerUtils.logError(Optional.ofNullable(logger), e); - LoggerUtils.logError(Optional.ofNullable(taskLoggerThreadLocal.get()), e); } - return false; - } + /** + * create linux user + * + * @param userName user name + * @param userGroup user group + * @throws IOException in case of an I/O error + */ + private static void createLinuxUser(String userName, String userGroup) throws IOException { + String infoLog1 = String.format("create linux os user : %s", userName); + LoggerUtils.logInfo(Optional.ofNullable(logger), infoLog1); + LoggerUtils.logInfo(Optional.ofNullable(taskLoggerThreadLocal.get()), infoLog1); - /** - * create linux user - * @param userName user name - * @param userGroup user group - * @throws IOException in case of an I/O error - */ - private static void createLinuxUser(String userName, String userGroup) throws IOException { - String infoLog1 = String.format("create linux os user : %s", userName); - LoggerUtils.logInfo(Optional.ofNullable(logger), infoLog1); - LoggerUtils.logInfo(Optional.ofNullable(taskLoggerThreadLocal.get()), infoLog1); - - String cmd = String.format("sudo useradd -g %s %s", userGroup, userName); - String infoLog2 = String.format("execute cmd : %s", cmd); - LoggerUtils.logInfo(Optional.ofNullable(logger), infoLog2); - LoggerUtils.logInfo(Optional.ofNullable(taskLoggerThreadLocal.get()), infoLog2); - OSUtils.exeCmd(cmd); - } - - /** - * create mac user (Supports Mac OSX 10.10+) - * @param userName user name - * @param userGroup user group - * @throws IOException in case of an I/O error - */ - private static void createMacUser(String userName, String userGroup) throws IOException { - - Optional optionalLogger = Optional.ofNullable(logger); - Optional optionalTaskLogger = Optional.ofNullable(taskLoggerThreadLocal.get()); - - String infoLog1 = String.format("create mac os user : %s", userName); - LoggerUtils.logInfo(optionalLogger, infoLog1); - LoggerUtils.logInfo(optionalTaskLogger, infoLog1); - - String createUserCmd = String.format("sudo sysadminctl -addUser %s -password %s", userName, userName); - String infoLog2 = String.format("create user command : %s", createUserCmd); - LoggerUtils.logInfo(optionalLogger, infoLog2); - LoggerUtils.logInfo(optionalTaskLogger, infoLog2); - OSUtils.exeCmd(createUserCmd); - - String appendGroupCmd = String.format("sudo dseditgroup -o edit -a %s -t user %s", userName, userGroup); - String infoLog3 = String.format("append user to group : %s", appendGroupCmd); - LoggerUtils.logInfo(optionalLogger, infoLog3); - LoggerUtils.logInfo(optionalTaskLogger, infoLog3); - OSUtils.exeCmd(appendGroupCmd); - } - - /** - * create windows user - * @param userName user name - * @param userGroup user group - * @throws IOException in case of an I/O error - */ - private static void createWindowsUser(String userName, String userGroup) throws IOException { - String infoLog1 = String.format("create windows os user : %s", userName); - LoggerUtils.logInfo(Optional.ofNullable(logger), infoLog1); - LoggerUtils.logInfo(Optional.ofNullable(taskLoggerThreadLocal.get()), infoLog1); - - String userCreateCmd = String.format("net user \"%s\" /add", userName); - String infoLog2 = String.format("execute create user command : %s", userCreateCmd); - LoggerUtils.logInfo(Optional.ofNullable(logger), infoLog2); - LoggerUtils.logInfo(Optional.ofNullable(taskLoggerThreadLocal.get()), infoLog2); - OSUtils.exeCmd(userCreateCmd); - - String appendGroupCmd = String.format("net localgroup \"%s\" \"%s\" /add", userGroup, userName); - String infoLog3 = String.format("execute append user to group : %s", appendGroupCmd); - LoggerUtils.logInfo(Optional.ofNullable(logger), infoLog3); - LoggerUtils.logInfo(Optional.ofNullable(taskLoggerThreadLocal.get()), infoLog3); - OSUtils.exeCmd(appendGroupCmd); - } - - /** - * get system group information - * @return system group info - * @throws IOException errors - */ - public static String getGroup() throws IOException { - if (isWindows()) { - String currentProcUserName = System.getProperty("user.name"); - String result = exeCmd(String.format("net user \"%s\"", currentProcUserName)); - String line = result.split("\n")[22]; - String group = PATTERN.split(line)[1]; - if (group.charAt(0) == '*') { - return group.substring(1); - } else { - return group; - } - } else { - String result = exeCmd("groups"); - if (StringUtils.isNotEmpty(result)) { - String[] groupInfo = result.split(" "); - return groupInfo[0]; - } + String cmd = String.format("sudo useradd -g %s %s", userGroup, userName); + String infoLog2 = String.format("execute cmd : %s", cmd); + LoggerUtils.logInfo(Optional.ofNullable(logger), infoLog2); + LoggerUtils.logInfo(Optional.ofNullable(taskLoggerThreadLocal.get()), infoLog2); + OSUtils.exeCmd(cmd); } - return null; - } + /** + * create mac user (Supports Mac OSX 10.10+) + * + * @param userName user name + * @param userGroup user group + * @throws IOException in case of an I/O error + */ + private static void createMacUser(String userName, String userGroup) throws IOException { - /** - * Execute the corresponding command of Linux or Windows - * - * @param command command - * @return result of execute command - * @throws IOException errors - */ - public static String exeCmd(String command) throws IOException { - StringTokenizer st = new StringTokenizer(command); - String[] cmdArray = new String[st.countTokens()]; - for (int i = 0; st.hasMoreTokens(); i++) { - cmdArray[i] = st.nextToken(); + Optional optionalLogger = Optional.ofNullable(logger); + Optional optionalTaskLogger = Optional.ofNullable(taskLoggerThreadLocal.get()); + + String infoLog1 = String.format("create mac os user : %s", userName); + LoggerUtils.logInfo(optionalLogger, infoLog1); + LoggerUtils.logInfo(optionalTaskLogger, infoLog1); + + String createUserCmd = String.format("sudo sysadminctl -addUser %s -password %s", userName, userName); + String infoLog2 = String.format("create user command : %s", createUserCmd); + LoggerUtils.logInfo(optionalLogger, infoLog2); + LoggerUtils.logInfo(optionalTaskLogger, infoLog2); + OSUtils.exeCmd(createUserCmd); + + String appendGroupCmd = String.format("sudo dseditgroup -o edit -a %s -t user %s", userName, userGroup); + String infoLog3 = String.format("append user to group : %s", appendGroupCmd); + LoggerUtils.logInfo(optionalLogger, infoLog3); + LoggerUtils.logInfo(optionalTaskLogger, infoLog3); + OSUtils.exeCmd(appendGroupCmd); } - return exeShell(cmdArray); - } - /** - * Execute the shell - * @param command command - * @return result of execute the shell - * @throws IOException errors - */ - public static String exeShell(String[] command) throws IOException { - return ShellExecutor.execCommand(command); - } + /** + * create windows user + * + * @param userName user name + * @param userGroup user group + * @throws IOException in case of an I/O error + */ + private static void createWindowsUser(String userName, String userGroup) throws IOException { + String infoLog1 = String.format("create windows os user : %s", userName); + LoggerUtils.logInfo(Optional.ofNullable(logger), infoLog1); + LoggerUtils.logInfo(Optional.ofNullable(taskLoggerThreadLocal.get()), infoLog1); - /** - * get process id - * @return process id - */ - public static int getProcessID() { - RuntimeMXBean runtimeMXBean = ManagementFactory.getRuntimeMXBean(); - return Integer.parseInt(runtimeMXBean.getName().split("@")[0]); - } + String userCreateCmd = String.format("net user \"%s\" /add", userName); + String infoLog2 = String.format("execute create user command : %s", userCreateCmd); + LoggerUtils.logInfo(Optional.ofNullable(logger), infoLog2); + LoggerUtils.logInfo(Optional.ofNullable(taskLoggerThreadLocal.get()), infoLog2); + OSUtils.exeCmd(userCreateCmd); - /** - * whether is macOS - * @return true if mac - */ - public static boolean isMacOS() { - return getOSName().startsWith("Mac"); - } - - - /** - * whether is windows - * @return true if windows - */ - public static boolean isWindows() { - return getOSName().startsWith("Windows"); - } - - /** - * get current OS name - * @return current OS name - */ - public static String getOSName() { - return System.getProperty("os.name"); - } - - /** - * check memory and cpu usage - * @param systemCpuLoad systemCpuLoad - * @param systemReservedMemory systemReservedMemory - * @return check memory and cpu usage - */ - public static Boolean checkResource(double systemCpuLoad, double systemReservedMemory){ - // system load average - double loadAverage = OSUtils.loadAverage(); - // system available physical memory - double availablePhysicalMemorySize = OSUtils.availablePhysicalMemorySize(); - - if(loadAverage > systemCpuLoad || availablePhysicalMemorySize < systemReservedMemory){ - logger.warn("load is too high or availablePhysicalMemorySize(G) is too low, it's availablePhysicalMemorySize(G):{},loadAvg:{}", availablePhysicalMemorySize , loadAverage); - return false; - }else{ - return true; + String appendGroupCmd = String.format("net localgroup \"%s\" \"%s\" /add", userGroup, userName); + String infoLog3 = String.format("execute append user to group : %s", appendGroupCmd); + LoggerUtils.logInfo(Optional.ofNullable(logger), infoLog3); + LoggerUtils.logInfo(Optional.ofNullable(taskLoggerThreadLocal.get()), infoLog3); + OSUtils.exeCmd(appendGroupCmd); } - } - /** - * check memory and cpu usage - * @param conf conf - * @param isMaster is master - * @return check memory and cpu usage - */ - public static Boolean checkResource(Configuration conf, Boolean isMaster){ - double systemCpuLoad; - double systemReservedMemory; + /** + * get system group information + * + * @return system group info + * @throws IOException errors + */ + public static String getGroup() throws IOException { + if (isWindows()) { + String currentProcUserName = System.getProperty("user.name"); + String result = exeCmd(String.format("net user \"%s\"", currentProcUserName)); + String line = result.split("\n")[22]; + String group = PATTERN.split(line)[1]; + if (group.charAt(0) == '*') { + return group.substring(1); + } else { + return group; + } + } else { + String result = exeCmd("groups"); + if (StringUtils.isNotEmpty(result)) { + String[] groupInfo = result.split(" "); + return groupInfo[0]; + } + } - if(Boolean.TRUE.equals(isMaster)){ - systemCpuLoad = conf.getDouble(Constants.MASTER_MAX_CPULOAD_AVG, Constants.DEFAULT_MASTER_CPU_LOAD); - systemReservedMemory = conf.getDouble(Constants.MASTER_RESERVED_MEMORY, Constants.DEFAULT_MASTER_RESERVED_MEMORY); - }else{ - systemCpuLoad = conf.getDouble(Constants.WORKER_MAX_CPULOAD_AVG, Constants.DEFAULT_WORKER_CPU_LOAD); - systemReservedMemory = conf.getDouble(Constants.WORKER_RESERVED_MEMORY, Constants.DEFAULT_WORKER_RESERVED_MEMORY); + return null; + } + + /** + * Execute the corresponding command of Linux or Windows + * + * @param command command + * @return result of execute command + * @throws IOException errors + */ + public static String exeCmd(String command) throws IOException { + StringTokenizer st = new StringTokenizer(command); + String[] cmdArray = new String[st.countTokens()]; + for (int i = 0; st.hasMoreTokens(); i++) { + cmdArray[i] = st.nextToken(); + } + return exeShell(cmdArray); + } + + /** + * Execute the shell + * + * @param command command + * @return result of execute the shell + * @throws IOException errors + */ + public static String exeShell(String[] command) throws IOException { + return ShellExecutor.execCommand(command); + } + + /** + * get process id + * + * @return process id + */ + public static int getProcessID() { + RuntimeMXBean runtimeMXBean = ManagementFactory.getRuntimeMXBean(); + return Integer.parseInt(runtimeMXBean.getName().split("@")[0]); + } + + /** + * whether is macOS + * + * @return true if mac + */ + public static boolean isMacOS() { + return getOSName().startsWith("Mac"); + } + + /** + * whether is windows + * + * @return true if windows + */ + public static boolean isWindows() { + return getOSName().startsWith("Windows"); + } + + /** + * get current OS name + * + * @return current OS name + */ + public static String getOSName() { + return System.getProperty("os.name"); + } + + /** + * check memory and cpu usage + * + * @param systemCpuLoad systemCpuLoad + * @param systemReservedMemory systemReservedMemory + * @return check memory and cpu usage + */ + public static Boolean checkResource(double systemCpuLoad, double systemReservedMemory) { + // system load average + double loadAverage = OSUtils.loadAverage(); + // system available physical memory + double availablePhysicalMemorySize = OSUtils.availablePhysicalMemorySize(); + + if (loadAverage > systemCpuLoad || availablePhysicalMemorySize < systemReservedMemory) { + logger.warn("load is too high or availablePhysicalMemorySize(G) is too low, it's availablePhysicalMemorySize(G):{},loadAvg:{}", availablePhysicalMemorySize, loadAverage); + return false; + } else { + return true; + } + } + + /** + * check memory and cpu usage + * + * @param conf conf + * @param isMaster is master + * @return check memory and cpu usage + */ + public static Boolean checkResource(Configuration conf, Boolean isMaster) { + double systemCpuLoad; + double systemReservedMemory; + + if (Boolean.TRUE.equals(isMaster)) { + systemCpuLoad = conf.getDouble(Constants.MASTER_MAX_CPULOAD_AVG, Constants.DEFAULT_MASTER_CPU_LOAD); + systemReservedMemory = conf.getDouble(Constants.MASTER_RESERVED_MEMORY, Constants.DEFAULT_MASTER_RESERVED_MEMORY); + } else { + systemCpuLoad = conf.getDouble(Constants.WORKER_MAX_CPULOAD_AVG, Constants.DEFAULT_WORKER_CPU_LOAD); + systemReservedMemory = conf.getDouble(Constants.WORKER_RESERVED_MEMORY, Constants.DEFAULT_WORKER_RESERVED_MEMORY); + } + return checkResource(systemCpuLoad, systemReservedMemory); } - return checkResource(systemCpuLoad,systemReservedMemory); - } } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ParameterUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ParameterUtils.java index 2d624de1fe..39ec04afcf 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ParameterUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ParameterUtils.java @@ -14,10 +14,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.common.utils; -import org.apache.commons.lang.StringUtils; -import org.apache.commons.lang.time.DateUtils; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.common.enums.DataType; @@ -25,231 +24,228 @@ import org.apache.dolphinscheduler.common.process.Property; import org.apache.dolphinscheduler.common.utils.placeholder.BusinessTimeUtils; import org.apache.dolphinscheduler.common.utils.placeholder.PlaceholderUtils; import org.apache.dolphinscheduler.common.utils.placeholder.TimePlaceholderUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.sql.PreparedStatement; -import java.text.ParseException; -import java.util.*; +import java.util.Date; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * parameter parse utils */ public class ParameterUtils { - private static final Logger logger = LoggerFactory.getLogger(ParameterUtils.class); + private static final Logger logger = LoggerFactory.getLogger(ParameterUtils.class); - /** - * convert parameters place holders - * - * @param parameterString parameter - * @param parameterMap parameter map - * @return convert parameters place holders - */ - public static String convertParameterPlaceholders(String parameterString, Map parameterMap) { - if (StringUtils.isEmpty(parameterString) || parameterMap == null) { - return parameterString; + private ParameterUtils() { + throw new UnsupportedOperationException("Construct ParameterUtils"); } - //Get current time, schedule execute time - String cronTimeStr = parameterMap.get(Constants.PARAMETER_DATETIME); + /** + * convert parameters place holders + * + * @param parameterString parameter + * @param parameterMap parameter map + * @return convert parameters place holders + */ + public static String convertParameterPlaceholders(String parameterString, Map parameterMap) { + if (StringUtils.isEmpty(parameterString) || parameterMap == null) { + return parameterString; + } - Date cronTime = null; + //Get current time, schedule execute time + String cronTimeStr = parameterMap.get(Constants.PARAMETER_DATETIME); - if (StringUtils.isNotEmpty(cronTimeStr)) { - try { - cronTime = DateUtils.parseDate(cronTimeStr, new String[]{Constants.PARAMETER_FORMAT_TIME}); - } catch (ParseException e) { - logger.error("parse {} exception", cronTimeStr, e); - } - } else { - cronTime = new Date(); + Date cronTime = null; + + if (StringUtils.isNotEmpty(cronTimeStr)) { + cronTime = DateUtils.parse(cronTimeStr, Constants.PARAMETER_FORMAT_TIME); + } else { + cronTime = new Date(); + } + + // replace variable ${} form,refers to the replacement of system variables and custom variables + parameterString = PlaceholderUtils.replacePlaceholders(parameterString, parameterMap, true); + + // replace time $[...] form, eg. $[yyyyMMdd] + if (cronTime != null) { + parameterString = TimePlaceholderUtils.replacePlaceholders(parameterString, cronTime, true); + } + + return parameterString; } - // replace variable ${} form,refers to the replacement of system variables and custom variables - parameterString = PlaceholderUtils.replacePlaceholders(parameterString, parameterMap, true); + /** + * new + * convert parameters place holders + * + * @param parameterString parameter + * @param parameterMap parameter map + * @return convert parameters place holders + */ + public static String convertParameterPlaceholders2(String parameterString, Map parameterMap) { + if (StringUtils.isEmpty(parameterString)) { + return parameterString; + } + //Get current time, schedule execute time + String cronTimeStr = parameterMap.get(Constants.PARAMETER_SHECDULE_TIME); + Date cronTime = null; - // replace time $[...] form, eg. $[yyyyMMdd] - if (cronTime != null) { - parameterString = TimePlaceholderUtils.replacePlaceholders(parameterString, cronTime, true); + if (StringUtils.isNotEmpty(cronTimeStr)) { + cronTime = DateUtils.parse(cronTimeStr, Constants.PARAMETER_FORMAT_TIME); + + } else { + cronTime = new Date(); + } + + // replace variable ${} form,refers to the replacement of system variables and custom variables + parameterString = PlaceholderUtils.replacePlaceholders(parameterString, parameterMap, true); + + // replace time $[...] form, eg. $[yyyyMMdd] + if (cronTime != null) { + parameterString = TimePlaceholderUtils.replacePlaceholders(parameterString, cronTime, true); + + } + return parameterString; } - return parameterString; - } - - /** - * new - * convert parameters place holders - * - * @param parameterString parameter - * @param parameterMap parameter map - * @return convert parameters place holders - */ - public static String convertParameterPlaceholders2(String parameterString, Map parameterMap) { - if (StringUtils.isEmpty(parameterString)) { - return parameterString; - } - //Get current time, schedule execute time - String cronTimeStr = parameterMap.get(Constants.PARAMETER_SHECDULE_TIME); - Date cronTime = null; - - if (StringUtils.isNotEmpty(cronTimeStr)) { - try { - cronTime = DateUtils.parseDate(cronTimeStr, new String[]{Constants.PARAMETER_FORMAT_TIME}); - - } catch (ParseException e) { - logger.error(String.format("parse %s exception", cronTimeStr), e); - } - } else { - cronTime = new Date(); + /** + * set in parameter + * + * @param index index + * @param stmt preparedstatement + * @param dataType data type + * @param value value + * @throws Exception errors + */ + public static void setInParameter(int index, PreparedStatement stmt, DataType dataType, String value) throws Exception { + if (dataType.equals(DataType.VARCHAR)) { + stmt.setString(index, value); + } else if (dataType.equals(DataType.INTEGER)) { + stmt.setInt(index, Integer.parseInt(value)); + } else if (dataType.equals(DataType.LONG)) { + stmt.setLong(index, Long.parseLong(value)); + } else if (dataType.equals(DataType.FLOAT)) { + stmt.setFloat(index, Float.parseFloat(value)); + } else if (dataType.equals(DataType.DOUBLE)) { + stmt.setDouble(index, Double.parseDouble(value)); + } else if (dataType.equals(DataType.DATE)) { + stmt.setDate(index, java.sql.Date.valueOf(value)); + } else if (dataType.equals(DataType.TIME)) { + stmt.setString(index, value); + } else if (dataType.equals(DataType.TIMESTAMP)) { + stmt.setTimestamp(index, java.sql.Timestamp.valueOf(value)); + } else if (dataType.equals(DataType.BOOLEAN)) { + stmt.setBoolean(index, Boolean.parseBoolean(value)); + } } - // replace variable ${} form,refers to the replacement of system variables and custom variables - parameterString = PlaceholderUtils.replacePlaceholders(parameterString, parameterMap, true); + /** + * curing user define parameters + * + * @param globalParamMap global param map + * @param globalParamList global param list + * @param commandType command type + * @param scheduleTime schedule time + * @return curing user define parameters + */ + public static String curingGlobalParams(Map globalParamMap, List globalParamList, + CommandType commandType, Date scheduleTime) { - // replace time $[...] form, eg. $[yyyyMMdd] - if (cronTime != null) { - parameterString = TimePlaceholderUtils.replacePlaceholders(parameterString, cronTime, true); + if (globalParamList == null || globalParamList.isEmpty()) { + return null; + } - } - return parameterString; - } + Map globalMap = new HashMap<>(); + if (globalParamMap != null) { + globalMap.putAll(globalParamMap); + } + Map allParamMap = new HashMap<>(); + //If it is a complement, a complement time needs to be passed in, according to the task type + Map timeParams = BusinessTimeUtils + .getBusinessTime(commandType, scheduleTime); + if (timeParams != null) { + allParamMap.putAll(timeParams); + } - /** - * set in parameter - * @param index index - * @param stmt preparedstatement - * @param dataType data type - * @param value value - * @throws Exception errors - */ - public static void setInParameter(int index, PreparedStatement stmt, DataType dataType, String value)throws Exception{ - if (dataType.equals(DataType.VARCHAR)){ - stmt.setString(index,value); - }else if (dataType.equals(DataType.INTEGER)){ - stmt.setInt(index, Integer.parseInt(value)); - }else if (dataType.equals(DataType.LONG)){ - stmt.setLong(index, Long.parseLong(value)); - }else if (dataType.equals(DataType.FLOAT)){ - stmt.setFloat(index, Float.parseFloat(value)); - }else if (dataType.equals(DataType.DOUBLE)){ - stmt.setDouble(index, Double.parseDouble(value)); - }else if (dataType.equals(DataType.DATE)){ - stmt.setDate(index, java.sql.Date.valueOf(value)); - }else if (dataType.equals(DataType.TIME)){ - stmt.setString(index, value); - }else if (dataType.equals(DataType.TIMESTAMP)){ - stmt.setTimestamp(index, java.sql.Timestamp.valueOf(value)); - }else if (dataType.equals(DataType.BOOLEAN)){ - stmt.setBoolean(index,Boolean.parseBoolean(value)); - } - } + allParamMap.putAll(globalMap); - /** - * curing user define parameters - * - * @param globalParamMap global param map - * @param globalParamList global param list - * @param commandType command type - * @param scheduleTime schedule time - * @return curing user define parameters - */ - public static String curingGlobalParams(Map globalParamMap, List globalParamList, - CommandType commandType, Date scheduleTime){ + Set> entries = allParamMap.entrySet(); - if (globalParamList == null || globalParamList.isEmpty()) { - return null; + Map resolveMap = new HashMap<>(); + for (Map.Entry entry : entries) { + String val = entry.getValue(); + if (val.startsWith("$")) { + String str = ParameterUtils.convertParameterPlaceholders(val, allParamMap); + resolveMap.put(entry.getKey(), str); + } + } + globalMap.putAll(resolveMap); + + for (Property property : globalParamList) { + String val = globalMap.get(property.getProp()); + if (val != null) { + property.setValue(val); + } + } + return JSONUtils.toJsonString(globalParamList); } - Map globalMap = new HashMap<>(); - if (globalParamMap!= null){ - globalMap.putAll(globalParamMap); - } - Map allParamMap = new HashMap<>(); - //If it is a complement, a complement time needs to be passed in, according to the task type - Map timeParams = BusinessTimeUtils - .getBusinessTime(commandType, scheduleTime); + /** + * handle escapes + * + * @param inputString input string + * @return string filter escapes + */ + public static String handleEscapes(String inputString) { - if (timeParams != null) { - allParamMap.putAll(timeParams); + if (StringUtils.isNotEmpty(inputString)) { + return inputString.replace("%", "////%").replaceAll("[\n|\r\t]", "_"); + } + return inputString; } - allParamMap.putAll(globalMap); + /** + * $[yyyyMMdd] replace schedule time + */ + public static String replaceScheduleTime(String text, Date scheduleTime) { + Map paramsMap = new HashMap<>(); + //if getScheduleTime null ,is current date + if (null == scheduleTime) { + scheduleTime = new Date(); + } - Set> entries = allParamMap.entrySet(); + String dateTime = org.apache.dolphinscheduler.common.utils.DateUtils.format(scheduleTime, Constants.PARAMETER_FORMAT_TIME); + Property p = new Property(); + p.setValue(dateTime); + p.setProp(Constants.PARAMETER_SHECDULE_TIME); + paramsMap.put(Constants.PARAMETER_SHECDULE_TIME, p); + text = ParameterUtils.convertParameterPlaceholders2(text, convert(paramsMap)); - Map resolveMap = new HashMap<>(); - for (Map.Entry entry : entries){ - String val = entry.getValue(); - if (val.startsWith("$")){ - String str = ParameterUtils.convertParameterPlaceholders(val, allParamMap); - resolveMap.put(entry.getKey(),str); - } + return text; } - globalMap.putAll(resolveMap); - for (Property property : globalParamList){ - String val = globalMap.get(property.getProp()); - if (val != null){ - property.setValue(val); - } + /** + * format convert + * + * @param paramsMap params map + * @return Map of converted + * see org.apache.dolphinscheduler.server.utils.ParamUtils.convert + */ + public static Map convert(Map paramsMap) { + Map map = new HashMap<>(); + Iterator> iter = paramsMap.entrySet().iterator(); + while (iter.hasNext()) { + Map.Entry en = iter.next(); + map.put(en.getKey(), en.getValue().getValue()); + } + return map; } - return JSONUtils.toJsonString(globalParamList); - } - - - /** - * handle escapes - * @param inputString input string - * @return string filter escapes - */ - public static String handleEscapes(String inputString){ - - if(StringUtils.isNotEmpty(inputString)){ - return inputString.replace("%", "////%").replaceAll("[\n|\r\t]", "_"); - } - return inputString; - } - - - /** - * $[yyyyMMdd] replace schedule time - * @param text - * @param scheduleTime - * @return - */ - public static String replaceScheduleTime(String text, Date scheduleTime) { - Map paramsMap = new HashMap<>(); - //if getScheduleTime null ,is current date - if (null == scheduleTime) { - scheduleTime = new Date(); - } - - String dateTime = org.apache.dolphinscheduler.common.utils.DateUtils.format(scheduleTime, Constants.PARAMETER_FORMAT_TIME); - Property p = new Property(); - p.setValue(dateTime); - p.setProp(Constants.PARAMETER_SHECDULE_TIME); - paramsMap.put(Constants.PARAMETER_SHECDULE_TIME, p); - text = ParameterUtils.convertParameterPlaceholders2(text, convert(paramsMap)); - - return text; - } - - - /** - * format convert - * @param paramsMap params map - * @return Map of converted - * see org.apache.dolphinscheduler.server.utils.ParamUtils.convert - */ - public static Map convert(Map paramsMap){ - Map map = new HashMap<>(); - Iterator> iter = paramsMap.entrySet().iterator(); - while (iter.hasNext()){ - Map.Entry en = iter.next(); - map.put(en.getKey(),en.getValue().getValue()); - } - return map; - } } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/Preconditions.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/Preconditions.java index 1fe40b97e3..9db2852644 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/Preconditions.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/Preconditions.java @@ -14,16 +14,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.common.utils; - /** - * utility methods for validating input - * + * utility methods for validating input */ public final class Preconditions { - private Preconditions() {} + private Preconditions() { + throw new UnsupportedOperationException("Construct Preconditions"); + } /** * if obj is null will throw NPE @@ -41,32 +42,30 @@ public final class Preconditions { /** * if obj is null will throw NullPointerException with error message + * * @param obj obj * @param errorMsg error message * @param T * @return T */ - public static T checkNotNull(T obj, String errorMsg) { + public static T checkNotNull(T obj, String errorMsg) { if (obj == null) { throw new NullPointerException(errorMsg); } return obj; } - /** * if condition is false will throw an IllegalArgumentException with the given message * * @param condition condition - * @param errorMsg error message - * + * @param errorMsg error message * @throws IllegalArgumentException Thrown, if the condition is violated. */ - public static void checkArgument(boolean condition, Object errorMsg) { + public static void checkArgument(boolean condition, Object errorMsg) { if (!condition) { throw new IllegalArgumentException(String.valueOf(errorMsg)); } } - } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java index 895270766c..9edf7939db 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java @@ -14,13 +14,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.common.utils; +import static org.apache.dolphinscheduler.common.Constants.COMMON_PROPERTIES_PATH; + import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ResUploadType; + import org.apache.commons.io.IOUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.IOException; import java.io.InputStream; @@ -28,7 +30,8 @@ import java.util.HashMap; import java.util.Map; import java.util.Properties; -import static org.apache.dolphinscheduler.common.Constants.COMMON_PROPERTIES_PATH; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * property utils @@ -44,7 +47,7 @@ public class PropertyUtils { private static final Properties properties = new Properties(); private PropertyUtils() { - throw new IllegalStateException("PropertyUtils class"); + throw new UnsupportedOperationException("Construct PropertyUtils"); } static { @@ -68,10 +71,9 @@ public class PropertyUtils { } /** - * - * @return judge whether resource upload startup + * @return judge whether resource upload startup */ - public static Boolean getResUploadStartupState(){ + public static Boolean getResUploadStartupState() { String resUploadStartupType = PropertyUtils.getUpperCaseString(Constants.RESOURCE_STORAGE_TYPE); ResUploadType resUploadType = ResUploadType.valueOf(resUploadStartupType); return resUploadType == ResUploadType.HDFS || resUploadType == ResUploadType.S3; @@ -113,14 +115,13 @@ public class PropertyUtils { * get property value * * @param key property name - * @return get property int value , if key == null, then return -1 + * @return get property int value , if key == null, then return -1 */ public static int getInt(String key) { return getInt(key, -1); } /** - * * @param key key * @param defaultValue default value * @return property value @@ -134,7 +135,7 @@ public class PropertyUtils { try { return Integer.parseInt(value); } catch (NumberFormatException e) { - logger.info(e.getMessage(),e); + logger.info(e.getMessage(), e); } return defaultValue; } @@ -147,7 +148,7 @@ public class PropertyUtils { */ public static boolean getBoolean(String key) { String value = properties.getProperty(key.trim()); - if(null != value){ + if (null != value) { return Boolean.parseBoolean(value); } @@ -163,7 +164,7 @@ public class PropertyUtils { */ public static Boolean getBoolean(String key, boolean defaultValue) { String value = properties.getProperty(key.trim()); - if(null != value){ + if (null != value) { return Boolean.parseBoolean(value); } @@ -172,6 +173,7 @@ public class PropertyUtils { /** * get property long value + * * @param key key * @param defaultVal default value * @return property value @@ -182,16 +184,14 @@ public class PropertyUtils { } /** - * * @param key key * @return property value */ public static long getLong(String key) { - return getLong(key,-1); + return getLong(key, -1); } /** - * * @param key key * @param defaultVal default value * @return property value @@ -201,11 +201,11 @@ public class PropertyUtils { return val == null ? defaultVal : Double.parseDouble(val); } - /** - * get array - * @param key property name - * @param splitStr separator + * get array + * + * @param key property name + * @param splitStr separator * @return property value through array */ public static String[] getArray(String key, String splitStr) { @@ -217,18 +217,17 @@ public class PropertyUtils { String[] propertyArray = value.split(splitStr); return propertyArray; } catch (NumberFormatException e) { - logger.info(e.getMessage(),e); + logger.info(e.getMessage(), e); } return new String[0]; } /** - * * @param key key * @param type type * @param defaultValue default value * @param T - * @return get enum value + * @return get enum value */ public > T getEnum(String key, Class type, T defaultValue) { @@ -238,6 +237,7 @@ public class PropertyUtils { /** * get all properties with specified prefix, like: fs. + * * @param prefix prefix to search * @return all properties with specified prefix */ @@ -253,11 +253,9 @@ public class PropertyUtils { /** * - * @param key - * @param value */ public static void setValue(String key, String value) { - properties.setProperty(key,value); + properties.setProperty(key, value); } } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/RetryerUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/RetryerUtils.java index a3a935831f..23861c7084 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/RetryerUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/RetryerUtils.java @@ -17,15 +17,21 @@ package org.apache.dolphinscheduler.common.utils; -import com.github.rholder.retry.*; import org.apache.dolphinscheduler.common.Constants; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.github.rholder.retry.RetryException; +import com.github.rholder.retry.Retryer; +import com.github.rholder.retry.RetryerBuilder; +import com.github.rholder.retry.StopStrategies; +import com.github.rholder.retry.WaitStrategies; + /** * The Retryer util. */ @@ -35,7 +41,7 @@ public class RetryerUtils { private static Retryer defaultRetryerResultNoCheck; private RetryerUtils() { - + throw new UnsupportedOperationException("Construct RetryerUtils"); } private static Retryer getDefaultRetryerResultNoCheck() { diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/SchemaUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/SchemaUtils.java index 312421adc6..bbcd9feed5 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/SchemaUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/SchemaUtils.java @@ -14,10 +14,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.dolphinscheduler.common.utils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +package org.apache.dolphinscheduler.common.utils; import java.io.File; import java.io.FileInputStream; @@ -29,113 +27,123 @@ import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + /** * Metadata related common classes - * */ public class SchemaUtils { - private static final Logger logger = LoggerFactory.getLogger(SchemaUtils.class); - private static Pattern p = Pattern.compile("\\s*|\t|\r|\n"); + private static final Logger logger = LoggerFactory.getLogger(SchemaUtils.class); + private static Pattern p = Pattern.compile("\\s*|\t|\r|\n"); - /** - * Gets upgradable schemas for all upgrade directories - * @return all schema list - */ - @SuppressWarnings("unchecked") - public static List getAllSchemaList() { - List schemaDirList = new ArrayList<>(); - File[] schemaDirArr = FileUtils.getAllDir("sql/upgrade"); - if(schemaDirArr == null || schemaDirArr.length == 0) { - return null; - } + private SchemaUtils() { + throw new UnsupportedOperationException("Construct SchemaUtils"); + } - for(File file : schemaDirArr) { - schemaDirList.add(file.getName()); - } + /** + * Gets upgradable schemas for all upgrade directories + * + * @return all schema list + */ + @SuppressWarnings("unchecked") + public static List getAllSchemaList() { + List schemaDirList = new ArrayList<>(); + File[] schemaDirArr = FileUtils.getAllDir("sql/upgrade"); + if (schemaDirArr == null || schemaDirArr.length == 0) { + return null; + } - Collections.sort(schemaDirList , new Comparator() { - @Override - public int compare(Object o1 , Object o2){ - try { - String dir1 = String.valueOf(o1); - String dir2 = String.valueOf(o2); - String version1 = dir1.split("_")[0]; - String version2 = dir2.split("_")[0]; - if(version1.equals(version2)) { - return 0; - } + for (File file : schemaDirArr) { + schemaDirList.add(file.getName()); + } - if(SchemaUtils.isAGreatVersion(version1, version2)) { - return 1; - } + Collections.sort(schemaDirList, new Comparator() { + @Override + public int compare(Object o1, Object o2) { + try { + String dir1 = String.valueOf(o1); + String dir2 = String.valueOf(o2); + String version1 = dir1.split("_")[0]; + String version2 = dir2.split("_")[0]; + if (version1.equals(version2)) { + return 0; + } - return -1; + if (SchemaUtils.isAGreatVersion(version1, version2)) { + return 1; + } - } catch (Exception e) { - logger.error(e.getMessage(),e); - throw new RuntimeException(e); - } - } - }); + return -1; - return schemaDirList; - } + } catch (Exception e) { + logger.error(e.getMessage(), e); + throw new RuntimeException(e); + } + } + }); - /** - * Determine whether schemaVersion is higher than version - * @param schemaVersion schema version - * @param version version - * @return Determine whether schemaVersion is higher than version - */ - public static boolean isAGreatVersion(String schemaVersion, String version) { - if(StringUtils.isEmpty(schemaVersion) || StringUtils.isEmpty(version)) { - throw new RuntimeException("schemaVersion or version is empty"); - } + return schemaDirList; + } - String[] schemaVersionArr = schemaVersion.split("\\."); - String[] versionArr = version.split("\\."); - int arrLength = Math.min(schemaVersionArr.length, versionArr.length); - for(int i = 0 ; i < arrLength ; i++) { - if(Integer.parseInt(schemaVersionArr[i]) > Integer.parseInt(versionArr[i])) { - return true; - }else if(Integer.parseInt(schemaVersionArr[i]) < Integer.parseInt(versionArr[i])) { - return false; - } - } + /** + * Determine whether schemaVersion is higher than version + * + * @param schemaVersion schema version + * @param version version + * @return Determine whether schemaVersion is higher than version + */ + public static boolean isAGreatVersion(String schemaVersion, String version) { + if (StringUtils.isEmpty(schemaVersion) || StringUtils.isEmpty(version)) { + throw new RuntimeException("schemaVersion or version is empty"); + } - // If the version and schema version is the same from 0 up to the arrlength-1 element,whoever has a larger arrLength has a larger version number - return schemaVersionArr.length > versionArr.length; - } + String[] schemaVersionArr = schemaVersion.split("\\."); + String[] versionArr = version.split("\\."); + int arrLength = Math.min(schemaVersionArr.length, versionArr.length); + for (int i = 0; i < arrLength; i++) { + if (Integer.parseInt(schemaVersionArr[i]) > Integer.parseInt(versionArr[i])) { + return true; + } else if (Integer.parseInt(schemaVersionArr[i]) < Integer.parseInt(versionArr[i])) { + return false; + } + } - /** - * Gets the current software version number of the system - * @return current software version - */ - public static String getSoftVersion() { - String soft_version; - try { - soft_version = FileUtils.readFile2Str(new FileInputStream(new File("sql/soft_version"))); - soft_version = replaceBlank(soft_version); - } catch (FileNotFoundException e) { - logger.error(e.getMessage(),e); - throw new RuntimeException("Failed to get the product version description file. The file could not be found", e); - } - return soft_version; - } + // If the version and schema version is the same from 0 up to the arrlength-1 element,whoever has a larger arrLength has a larger version number + return schemaVersionArr.length > versionArr.length; + } - /** - * Strips the string of space carriage returns and tabs - * @param str string - * @return string removed blank - */ - public static String replaceBlank(String str) { - String dest = ""; - if (str!=null) { + /** + * Gets the current software version number of the system + * + * @return current software version + */ + public static String getSoftVersion() { + String softVersion; + try { + softVersion = FileUtils.readFile2Str(new FileInputStream(new File("sql/soft_version"))); + softVersion = replaceBlank(softVersion); + } catch (FileNotFoundException e) { + logger.error(e.getMessage(), e); + throw new RuntimeException("Failed to get the product version description file. The file could not be found", e); + } + return softVersion; + } - Matcher m = p.matcher(str); - dest = m.replaceAll(""); - } - return dest; - } + /** + * Strips the string of space carriage returns and tabs + * + * @param str string + * @return string removed blank + */ + public static String replaceBlank(String str) { + String dest = ""; + if (str != null) { + + Matcher m = p.matcher(str); + dest = m.replaceAll(""); + } + return dest; + } } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/SensitiveLogUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/SensitiveLogUtils.java index eab6c4f124..5706c38254 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/SensitiveLogUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/SensitiveLogUtils.java @@ -14,21 +14,25 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.common.utils; -import org.apache.commons.lang.StringUtils; import org.apache.dolphinscheduler.common.Constants; /** - * sensitive log Util + * sensitive log Util */ public class SensitiveLogUtils { + private SensitiveLogUtils() { + throw new UnsupportedOperationException("Construct SensitiveLogUtils"); + } + /** * @param dataSourcePwd data source password * @return String */ - public static String maskDataSourcePwd(String dataSourcePwd){ + public static String maskDataSourcePwd(String dataSourcePwd) { if (StringUtils.isNotEmpty(dataSourcePwd)) { dataSourcePwd = Constants.PASSWORD_DEFAULT; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/StreamUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/StreamUtils.java index f30638cda2..fb4941a95d 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/StreamUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/StreamUtils.java @@ -14,6 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.common.utils; import java.util.Iterator; @@ -22,7 +23,9 @@ import java.util.stream.StreamSupport; public class StreamUtils { - private StreamUtils() { } + private StreamUtils() { + throw new UnsupportedOperationException("Construct StreamUtils"); + } public static Stream asStream(Iterator sourceIterator) { return asStream(sourceIterator, false); diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/StringUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/StringUtils.java index af2817a8d7..4f4f12766b 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/StringUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/StringUtils.java @@ -14,11 +14,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.common.utils; public class StringUtils { + public static final String EMPTY = ""; + private StringUtils() { + throw new UnsupportedOperationException("Construct StringUtils"); + } + public static boolean isEmpty(final CharSequence cs) { return cs == null || cs.length() == 0; } @@ -27,14 +33,18 @@ public class StringUtils { return !isEmpty(cs); } - public static boolean isBlank(String s){ + public static boolean isBlank(String s) { if (isEmpty(s)) { return true; } return s.trim().length() == 0; } - public static boolean isNotBlank(String s){ + public static boolean isNotBlank(String s) { return !isBlank(s); } + + public static String replaceNRTtoUnderline(String src) { + return src.replaceAll("[\n|\r|\t]", "_"); + } } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/TaskParametersUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/TaskParametersUtils.java index 84cca93b4c..6099a0d49d 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/TaskParametersUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/TaskParametersUtils.java @@ -14,13 +14,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.common.utils; import org.apache.dolphinscheduler.common.enums.TaskType; import org.apache.dolphinscheduler.common.task.AbstractParameters; import org.apache.dolphinscheduler.common.task.conditions.ConditionsParameters; -import org.apache.dolphinscheduler.common.task.dependent.DependentParameters; import org.apache.dolphinscheduler.common.task.datax.DataxParameters; +import org.apache.dolphinscheduler.common.task.dependent.DependentParameters; import org.apache.dolphinscheduler.common.task.flink.FlinkParameters; import org.apache.dolphinscheduler.common.task.http.HttpParameters; import org.apache.dolphinscheduler.common.task.mr.MapreduceParameters; @@ -31,60 +32,65 @@ import org.apache.dolphinscheduler.common.task.spark.SparkParameters; import org.apache.dolphinscheduler.common.task.sql.SqlParameters; import org.apache.dolphinscheduler.common.task.sqoop.SqoopParameters; import org.apache.dolphinscheduler.common.task.subprocess.SubProcessParameters; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; - /** * task parameters utils */ public class TaskParametersUtils { - private static Logger logger = LoggerFactory.getLogger(TaskParametersUtils.class); + private static Logger logger = LoggerFactory.getLogger(TaskParametersUtils.class); - /** - * get task parameters - * @param taskType task type - * @param parameter parameter - * @return task parameters - */ - public static AbstractParameters getParameters(String taskType, String parameter) { - try { - switch (EnumUtils.getEnum(TaskType.class,taskType)) { - case SUB_PROCESS: - return JSONUtils.parseObject(parameter, SubProcessParameters.class); - case WATERDROP: - return JSONUtils.parseObject(parameter, ShellParameters.class); - case SHELL: - return JSONUtils.parseObject(parameter, ShellParameters.class); - case PROCEDURE: - return JSONUtils.parseObject(parameter, ProcedureParameters.class); - case SQL: - return JSONUtils.parseObject(parameter, SqlParameters.class); - case MR: - return JSONUtils.parseObject(parameter, MapreduceParameters.class); - case SPARK: - return JSONUtils.parseObject(parameter, SparkParameters.class); - case PYTHON: - return JSONUtils.parseObject(parameter, PythonParameters.class); - case DEPENDENT: - return JSONUtils.parseObject(parameter, DependentParameters.class); - case FLINK: - return JSONUtils.parseObject(parameter, FlinkParameters.class); - case HTTP: - return JSONUtils.parseObject(parameter, HttpParameters.class); - case DATAX: - return JSONUtils.parseObject(parameter, DataxParameters.class); - case CONDITIONS: - return JSONUtils.parseObject(parameter, ConditionsParameters.class); - case SQOOP: - return JSONUtils.parseObject(parameter, SqoopParameters.class); - default: - return null; - } - } catch (Exception e) { - logger.error(e.getMessage(), e); + private TaskParametersUtils() { + throw new UnsupportedOperationException("Construct TaskParametersUtils"); + } + + /** + * get task parameters + * + * @param taskType task type + * @param parameter parameter + * @return task parameters + */ + public static AbstractParameters getParameters(String taskType, String parameter) { + try { + switch (EnumUtils.getEnum(TaskType.class, taskType)) { + case SUB_PROCESS: + return JSONUtils.parseObject(parameter, SubProcessParameters.class); + case WATERDROP: + return JSONUtils.parseObject(parameter, ShellParameters.class); + case SHELL: + return JSONUtils.parseObject(parameter, ShellParameters.class); + case PROCEDURE: + return JSONUtils.parseObject(parameter, ProcedureParameters.class); + case SQL: + return JSONUtils.parseObject(parameter, SqlParameters.class); + case MR: + return JSONUtils.parseObject(parameter, MapreduceParameters.class); + case SPARK: + return JSONUtils.parseObject(parameter, SparkParameters.class); + case PYTHON: + return JSONUtils.parseObject(parameter, PythonParameters.class); + case DEPENDENT: + return JSONUtils.parseObject(parameter, DependentParameters.class); + case FLINK: + return JSONUtils.parseObject(parameter, FlinkParameters.class); + case HTTP: + return JSONUtils.parseObject(parameter, HttpParameters.class); + case DATAX: + return JSONUtils.parseObject(parameter, DataxParameters.class); + case CONDITIONS: + return JSONUtils.parseObject(parameter, ConditionsParameters.class); + case SQOOP: + return JSONUtils.parseObject(parameter, SqoopParameters.class); + default: + return null; + } + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + return null; } - return null; - } } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/TriFunction.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/TriFunction.java new file mode 100644 index 0000000000..fe873b3475 --- /dev/null +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/TriFunction.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.common.utils; + +/** + * tri function function interface + */ +@FunctionalInterface +public interface TriFunction { + + OUT1 apply(IN1 in1, IN2 in2, IN3 in3); + +} diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/enums/ExecutionStatusTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/enums/ExecutionStatusTest.java new file mode 100644 index 0000000000..6d4be78aef --- /dev/null +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/enums/ExecutionStatusTest.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.common.enums; + +import junit.framework.TestCase; + +/** + * execution status test. + */ +public class ExecutionStatusTest extends TestCase { + + public void testTypeIsRunning() { + assertTrue(ExecutionStatus.RUNNING_EXECUTION.typeIsRunning()); + assertTrue(ExecutionStatus.WAITTING_DEPEND.typeIsRunning()); + assertTrue(ExecutionStatus.DELAY_EXECUTION.typeIsRunning()); + } +} \ No newline at end of file diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HttpUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HttpUtilsTest.java index aee7ac8880..f9ce989f70 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HttpUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HttpUtilsTest.java @@ -17,7 +17,13 @@ package org.apache.dolphinscheduler.common.utils; import com.fasterxml.jackson.databind.node.ObjectNode; + +import org.apache.dolphinscheduler.common.Constants; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.client.methods.HttpGet; import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; + import org.junit.Assert; import org.junit.Test; import org.slf4j.Logger; @@ -28,26 +34,53 @@ import org.slf4j.LoggerFactory; */ public class HttpUtilsTest { + public static final Logger logger = LoggerFactory.getLogger(HttpUtilsTest.class); + private HadoopUtils hadoopUtils = HadoopUtils.getInstance(); - public static final Logger logger = LoggerFactory.getLogger(HttpUtilsTest.class); + @Test + public void testGetTest() { + // success + String result = HttpUtils.get("https://github.com/manifest.json"); + Assert.assertNotNull(result); + ObjectNode jsonObject = JSONUtils.parseObject(result); + Assert.assertEquals("GitHub", jsonObject.path("name").asText()); + result = HttpUtils.get("https://123.333.111.33/ccc"); + Assert.assertNull(result); + } + @Test + public void testGetByKerberos() { + try { + String applicationUrl = hadoopUtils.getApplicationUrl("application_1542010131334_0029"); + String responseContent; + responseContent = HttpUtils.get(applicationUrl); + Assert.assertNull(responseContent); - @Test - public void testGetTest(){ - //success - String result = HttpUtils.get("https://github.com/manifest.json"); - Assert.assertNotNull(result); - ObjectNode jsonObject = JSONUtils.parseObject(result); - Assert.assertEquals("GitHub", jsonObject.path("name").asText()); - - result = HttpUtils.get("https://123.333.111.33/ccc"); - Assert.assertNull(result); + } catch (Exception e) { + logger.error(e.getMessage(), e); } + } + + @Test + public void testGetResponseContentString() { + CloseableHttpClient httpclient = HttpClients.createDefault(); + HttpGet httpget = new HttpGet("https://github.com/manifest.json"); + /** set timeout、request time、socket timeout */ + RequestConfig requestConfig = RequestConfig.custom().setConnectTimeout(Constants.HTTP_CONNECT_TIMEOUT) + .setConnectionRequestTimeout(Constants.HTTP_CONNECTION_REQUEST_TIMEOUT) + .setSocketTimeout(Constants.SOCKET_TIMEOUT).setRedirectsEnabled(true).build(); + httpget.setConfig(requestConfig); + String responseContent = HttpUtils.getResponseContentString(httpget, httpclient); + Assert.assertNotNull(responseContent); + } + + @Test public void testGetHttpClient() { CloseableHttpClient httpClient1 = HttpUtils.getInstance(); CloseableHttpClient httpClient2 = HttpUtils.getInstance(); Assert.assertEquals(httpClient1, httpClient2); } + } diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/KerberosHttpClientTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/KerberosHttpClientTest.java new file mode 100644 index 0000000000..9911961ac0 --- /dev/null +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/KerberosHttpClientTest.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.common.utils; + +import org.apache.dolphinscheduler.common.Constants; +import org.junit.Assert; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * KerberosHttpClient test + */ +public class KerberosHttpClientTest { + public static final Logger logger = LoggerFactory.getLogger(KerberosHttpClientTest.class); + private HadoopUtils hadoopUtils = HadoopUtils.getInstance(); + + @Test + public void get() { + try { + String applicationUrl = hadoopUtils.getApplicationUrl("application_1542010131334_0029"); + String responseContent; + KerberosHttpClient kerberosHttpClient = new KerberosHttpClient(PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_USERNAME), + PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_PATH), PropertyUtils.getString(Constants.JAVA_SECURITY_KRB5_CONF_PATH), true); + responseContent = kerberosHttpClient.get(applicationUrl, + PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_USERNAME)); + Assert.assertNull(responseContent); + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + } +} \ No newline at end of file diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/AlertDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/AlertDao.java index 49b8c01ece..cd101f06b6 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/AlertDao.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/AlertDao.java @@ -14,29 +14,35 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.dao; - -import org.apache.dolphinscheduler.common.utils.StringUtils; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; +import org.apache.dolphinscheduler.common.enums.AlertEvent; import org.apache.dolphinscheduler.common.enums.AlertStatus; import org.apache.dolphinscheduler.common.enums.AlertType; +import org.apache.dolphinscheduler.common.enums.AlertWarnLevel; import org.apache.dolphinscheduler.common.enums.ShowType; +import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.StringUtils; import org.apache.dolphinscheduler.dao.datasource.ConnectionFactory; import org.apache.dolphinscheduler.dao.entity.Alert; +import org.apache.dolphinscheduler.dao.entity.ProcessAlertContent; import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; +import org.apache.dolphinscheduler.dao.entity.ServerAlertContent; import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.mapper.AlertMapper; import org.apache.dolphinscheduler.dao.mapper.UserAlertGroupMapper; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - +import java.util.ArrayList; import java.util.Date; import java.util.List; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + @Component public class AlertDao extends AbstractBaseDao { @@ -56,21 +62,23 @@ public class AlertDao extends AbstractBaseDao { /** * insert alert + * * @param alert alert * @return add alert result */ - public int addAlert(Alert alert){ + public int addAlert(Alert alert) { return alertMapper.insert(alert); } /** * update alert + * * @param alertStatus alertStatus * @param log log * @param id id * @return update alert result */ - public int updateAlert(AlertStatus alertStatus,String log,int id){ + public int updateAlert(AlertStatus alertStatus, String log, int id) { Alert alert = alertMapper.selectById(id); alert.setAlertStatus(alertStatus); alert.setUpdateTime(new Date()); @@ -80,46 +88,60 @@ public class AlertDao extends AbstractBaseDao { /** * query user list by alert group id + * * @param alerGroupId alerGroupId * @return user list */ - public List queryUserByAlertGroupId(int alerGroupId){ + public List queryUserByAlertGroupId(int alerGroupId) { return userAlertGroupMapper.listUserByAlertgroupId(alerGroupId); } /** * MasterServer or WorkerServer stoped + * * @param alertgroupId alertgroupId * @param host host * @param serverType serverType */ - public void sendServerStopedAlert(int alertgroupId,String host,String serverType){ + public void sendServerStopedAlert(int alertgroupId, String host, String serverType) { Alert alert = new Alert(); - String content = String.format("[{'type':'%s','host':'%s','event':'server down','warning level':'serious'}]", - serverType, host); + List serverAlertContents = new ArrayList<>(1); + ServerAlertContent serverStopAlertContent = ServerAlertContent.newBuilder(). + type(serverType).host(host).event(AlertEvent.SERVER_DOWN).warningLevel(AlertWarnLevel.SERIOUS). + build(); + serverAlertContents.add(serverStopAlertContent); + String content = JSONUtils.toJsonString(serverAlertContents); alert.setTitle("Fault tolerance warning"); saveTaskTimeoutAlert(alert, content, alertgroupId, null, null); } /** * process time out alert + * * @param processInstance processInstance * @param processDefinition processDefinition */ - public void sendProcessTimeoutAlert(ProcessInstance processInstance, ProcessDefinition processDefinition){ + public void sendProcessTimeoutAlert(ProcessInstance processInstance, ProcessDefinition processDefinition) { int alertgroupId = processInstance.getWarningGroupId(); String receivers = processDefinition.getReceivers(); String receiversCc = processDefinition.getReceiversCc(); Alert alert = new Alert(); - String content = String.format("[{'id':'%d','name':'%s','event':'timeout','warnLevel':'middle'}]", - processInstance.getId(), processInstance.getName()); + List processAlertContentList = new ArrayList<>(1); + ProcessAlertContent processAlertContent = ProcessAlertContent.newBuilder() + .processId(processInstance.getId()) + .processName(processInstance.getName()) + .event(AlertEvent.TIME_OUT) + .warningLevel(AlertWarnLevel.MIDDLE) + .build(); + processAlertContentList.add(processAlertContent); + String content = JSONUtils.toJsonString(processAlertContentList); alert.setTitle("Process Timeout Warn"); saveTaskTimeoutAlert(alert, content, alertgroupId, receivers, receiversCc); } - private void saveTaskTimeoutAlert(Alert alert, String content, int alertgroupId, - String receivers, String receiversCc){ + private void saveTaskTimeoutAlert(Alert alert, String content, int alertgroupId, + String receivers, String receiversCc) { alert.setShowType(ShowType.TABLE); alert.setContent(content); alert.setAlertType(AlertType.EMAIL); @@ -135,9 +157,9 @@ public class AlertDao extends AbstractBaseDao { alertMapper.insert(alert); } - /** * task timeout warn + * * @param alertgroupId alertgroupId * @param receivers receivers * @param receiversCc receiversCc @@ -146,37 +168,50 @@ public class AlertDao extends AbstractBaseDao { * @param taskId taskId * @param taskName taskName */ - public void sendTaskTimeoutAlert(int alertgroupId,String receivers,String receiversCc, int processInstanceId, - String processInstanceName, int taskId,String taskName){ + public void sendTaskTimeoutAlert(int alertgroupId, String receivers, String receiversCc, int processInstanceId, + String processInstanceName, int taskId, String taskName) { Alert alert = new Alert(); - String content = String.format("[{'process instance id':'%d','task name':'%s','task id':'%d','task name':'%s'," + - "'event':'timeout','warnLevel':'middle'}]", processInstanceId, processInstanceName, taskId, taskName); + List processAlertContentList = new ArrayList<>(1); + ProcessAlertContent processAlertContent = ProcessAlertContent.newBuilder() + .processId(processInstanceId) + .processName(processInstanceName) + .taskId(taskId) + .taskName(taskName) + .event(AlertEvent.TIME_OUT) + .warningLevel(AlertWarnLevel.MIDDLE) + .build(); + processAlertContentList.add(processAlertContent); + String content = JSONUtils.toJsonString(processAlertContentList); alert.setTitle("Task Timeout Warn"); saveTaskTimeoutAlert(alert, content, alertgroupId, receivers, receiversCc); } /** * list the alert information of waiting to be executed + * * @return alert list */ - public List listWaitExecutionAlert(){ + public List listWaitExecutionAlert() { return alertMapper.listAlertByStatus(AlertStatus.WAIT_EXECUTION); } /** * list user information by alert group id + * * @param alertgroupId alertgroupId * @return user list */ - public List listUserByAlertgroupId(int alertgroupId){ + public List listUserByAlertgroupId(int alertgroupId) { return userAlertGroupMapper.listUserByAlertgroupId(alertgroupId); } /** * for test + * * @return AlertMapper */ public AlertMapper getAlertMapper() { return alertMapper; } + } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/BaseDataSource.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/BaseDataSource.java index ccae36aae0..729a17f27b 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/BaseDataSource.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/BaseDataSource.java @@ -133,6 +133,7 @@ public abstract class BaseDataSource { case MYSQL: case ORACLE: case POSTGRESQL: + case PRESTO: separator = "?"; break; case DB2: diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/DataSourceFactory.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/DataSourceFactory.java index 53f2468ea6..f5d07ea693 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/DataSourceFactory.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/DataSourceFactory.java @@ -54,6 +54,8 @@ public class DataSourceFactory { return JSONUtils.parseObject(parameter, SQLServerDataSource.class); case DB2: return JSONUtils.parseObject(parameter, DB2ServerDataSource.class); + case PRESTO: + return JSONUtils.parseObject(parameter, PrestoDataSource.class); default: return null; } @@ -94,6 +96,9 @@ public class DataSourceFactory { case DB2: Class.forName(Constants.COM_DB2_JDBC_DRIVER); break; + case PRESTO: + Class.forName(Constants.COM_PRESTO_JDBC_DRIVER); + break; default: logger.error("not support sql type: {},can't load class", dbType); throw new IllegalArgumentException("not support sql type,can't load class"); diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/PrestoDataSource.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/PrestoDataSource.java new file mode 100644 index 0000000000..93ed3d61a5 --- /dev/null +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/PrestoDataSource.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.dao.datasource; + +import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.enums.DbType; + +public class PrestoDataSource extends BaseDataSource { + + /** + * @return driver class + */ + @Override + public String driverClassSelector() { + return Constants.COM_PRESTO_JDBC_DRIVER; + } + + /** + * @return db type + */ + @Override + public DbType dbTypeSelector() { + return DbType.PRESTO; + } +} diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessAlertContent.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessAlertContent.java new file mode 100644 index 0000000000..71058f4af8 --- /dev/null +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessAlertContent.java @@ -0,0 +1,236 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.dao.entity; + +import org.apache.dolphinscheduler.common.enums.AlertEvent; +import org.apache.dolphinscheduler.common.enums.AlertWarnLevel; +import org.apache.dolphinscheduler.common.enums.CommandType; +import org.apache.dolphinscheduler.common.enums.ExecutionStatus; +import org.apache.dolphinscheduler.common.enums.Flag; + +import java.io.Serializable; +import java.util.Date; + +import com.fasterxml.jackson.annotation.JsonFormat; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonInclude.Include; +import com.fasterxml.jackson.annotation.JsonProperty; + +@JsonInclude(Include.NON_NULL) +public class ProcessAlertContent implements Serializable { + @JsonProperty("processId") + private int processId; + @JsonProperty("processName") + private String processName; + @JsonProperty("processType") + private CommandType processType; + @JsonProperty("processState") + private ExecutionStatus processState; + @JsonProperty("recovery") + private Flag recovery; + @JsonProperty("runTimes") + private int runTimes; + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss", timezone = "GMT+8") + @JsonProperty("processStartTime") + private Date processStartTime; + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss", timezone = "GMT+8") + @JsonProperty("processEndTime") + private Date processEndTime; + @JsonProperty("processHost") + private String processHost; + @JsonProperty("taskId") + private int taskId; + @JsonProperty("taskName") + private String taskName; + @JsonProperty("event") + private AlertEvent event; + @JsonProperty("warnLevel") + private AlertWarnLevel warnLevel; + @JsonProperty("taskType") + private String taskType; + @JsonProperty("retryTimes") + private int retryTimes; + @JsonProperty("taskState") + private ExecutionStatus taskState; + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss", timezone = "GMT+8") + @JsonProperty("taskStartTime") + private Date taskStartTime; + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss", timezone = "GMT+8") + @JsonProperty("taskEndTime") + private Date taskEndTime; + @JsonProperty("taskHost") + private String taskHost; + @JsonProperty("logPath") + private String logPath; + + private ProcessAlertContent(Builder builder) { + this.processId = builder.processId; + this.processName = builder.processName; + this.processType = builder.processType; + this.recovery = builder.recovery; + this.processState = builder.processState; + this.runTimes = builder.runTimes; + this.processStartTime = builder.processStartTime; + this.processEndTime = builder.processEndTime; + this.processHost = builder.processHost; + this.taskId = builder.taskId; + this.taskName = builder.taskName; + this.event = builder.event; + this.warnLevel = builder.warnLevel; + this.taskType = builder.taskType; + this.taskState = builder.taskState; + this.taskStartTime = builder.taskStartTime; + this.taskEndTime = builder.taskEndTime; + this.taskHost = builder.taskHost; + this.logPath = builder.logPath; + this.retryTimes = builder.retryTimes; + + } + + public static Builder newBuilder() { + return new Builder(); + } + + public static class Builder { + + private int processId; + private String processName; + private CommandType processType; + private Flag recovery; + private ExecutionStatus processState; + private int runTimes; + private Date processStartTime; + private Date processEndTime; + private String processHost; + private int taskId; + private String taskName; + private AlertEvent event; + private AlertWarnLevel warnLevel; + private String taskType; + private int retryTimes; + private ExecutionStatus taskState; + private Date taskStartTime; + private Date taskEndTime; + private String taskHost; + private String logPath; + + public Builder processId(int processId) { + this.processId = processId; + return this; + } + + public Builder processName(String processName) { + this.processName = processName; + return this; + } + + public Builder processType(CommandType processType) { + this.processType = processType; + return this; + } + + public Builder recovery(Flag recovery) { + this.recovery = recovery; + return this; + } + + public Builder processState(ExecutionStatus processState) { + this.processState = processState; + return this; + } + + public Builder runTimes(int runTimes) { + this.runTimes = runTimes; + return this; + } + + public Builder processStartTime(Date processStartTime) { + this.processStartTime = processStartTime; + return this; + } + + public Builder processEndTime(Date processEndTime) { + this.processEndTime = processEndTime; + return this; + } + + public Builder processHost(String processHost) { + this.processHost = processHost; + return this; + } + + public Builder taskId(int taskId) { + this.taskId = taskId; + return this; + } + + public Builder taskName(String taskName) { + this.taskName = taskName; + return this; + } + + public Builder event(AlertEvent event) { + this.event = event; + return this; + } + + public Builder warningLevel(AlertWarnLevel warnLevel) { + this.warnLevel = warnLevel; + return this; + } + + public Builder taskType(String taskType) { + this.taskType = taskType; + return this; + } + + public Builder retryTimes(int retryTimes) { + this.retryTimes = retryTimes; + return this; + } + + public Builder taskState(ExecutionStatus taskState) { + this.taskState = taskState; + return this; + } + + public Builder taskStartTime(Date taskStartTime) { + this.taskStartTime = taskStartTime; + return this; + } + + public Builder taskEndTime(Date taskEndTime) { + this.taskEndTime = taskEndTime; + return this; + } + + public Builder taskHost(String taskHost) { + this.taskHost = taskHost; + return this; + } + + public Builder logPath(String logPath) { + this.logPath = logPath; + return this; + } + + public ProcessAlertContent build() { + return new ProcessAlertContent(this); + } + } +} diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinition.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinition.java index 3648482996..56f6cfe905 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinition.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinition.java @@ -14,24 +14,26 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.dao.entity; +import org.apache.dolphinscheduler.common.enums.Flag; +import org.apache.dolphinscheduler.common.enums.ReleaseState; +import org.apache.dolphinscheduler.common.process.Property; +import org.apache.dolphinscheduler.common.utils.JSONUtils; + +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + import com.baomidou.mybatisplus.annotation.IdType; import com.baomidou.mybatisplus.annotation.TableField; import com.baomidou.mybatisplus.annotation.TableId; import com.baomidou.mybatisplus.annotation.TableName; import com.baomidou.mybatisplus.core.toolkit.StringUtils; import com.fasterxml.jackson.annotation.JsonFormat; -import org.apache.dolphinscheduler.common.enums.Flag; -import org.apache.dolphinscheduler.common.enums.ReleaseState; -import org.apache.dolphinscheduler.common.process.Property; -import org.apache.dolphinscheduler.common.utils.*; - -import java.util.ArrayList; -import java.util.Date; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; /** @@ -54,7 +56,7 @@ public class ProcessDefinition { /** * version */ - private int version; + private long version; /** * release state : online/offline @@ -96,13 +98,13 @@ public class ProcessDefinition { /** * create time */ - @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss", timezone = "GMT+8") private Date createTime; /** * update time */ - @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss", timezone = "GMT+8") private Date updateTime; /** @@ -182,11 +184,11 @@ public class ProcessDefinition { this.name = name; } - public int getVersion() { + public long getVersion() { return version; } - public void setVersion(int version) { + public void setVersion(long version) { this.version = version; } @@ -276,9 +278,9 @@ public class ProcessDefinition { } public void setGlobalParams(String globalParams) { - if (globalParams == null){ + if (globalParams == null) { this.globalParamList = new ArrayList<>(); - }else { + } else { this.globalParamList = JSONUtils.toList(globalParams, Property.class); } this.globalParams = globalParams; @@ -295,7 +297,7 @@ public class ProcessDefinition { public Map getGlobalParamMap() { if (globalParamMap == null && StringUtils.isNotEmpty(globalParams)) { - List propList = JSONUtils.toList(globalParams,Property.class); + List propList = JSONUtils.toList(globalParams, Property.class); globalParamMap = propList.stream().collect(Collectors.toMap(Property::getProp, Property::getValue)); } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinitionVersion.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinitionVersion.java new file mode 100644 index 0000000000..26779ba925 --- /dev/null +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinitionVersion.java @@ -0,0 +1,329 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.dao.entity; + +import java.util.Date; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import com.fasterxml.jackson.annotation.JsonFormat; + + +/** + * process definition version + */ +@TableName("t_ds_process_definition_version") +public class ProcessDefinitionVersion { + + /** + * id + */ + @TableId(value = "id", type = IdType.AUTO) + private int id; + + /** + * process definition id + */ + private int processDefinitionId; + + /** + * version + */ + private long version; + + /** + * definition json string + */ + private String processDefinitionJson; + + /** + * description + */ + private String description; + + /** + * receivers + */ + private String receivers; + + /** + * receivers cc + */ + private String receiversCc; + + /** + * process warning time out. unit: minute + */ + private int timeout; + + /** + * resource ids + */ + private String resourceIds; + + /** + * create time + */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss", timezone = "GMT+8") + private Date createTime; + + /** + * user defined parameters + */ + private String globalParams; + + /** + * locations array for web + */ + private String locations; + + /** + * connects array for web + */ + private String connects; + + public String getGlobalParams() { + return globalParams; + } + + public void setGlobalParams(String globalParams) { + this.globalParams = globalParams; + } + + public int getId() { + return id; + } + + public void setId(int id) { + this.id = id; + } + + public int getProcessDefinitionId() { + return processDefinitionId; + } + + public void setProcessDefinitionId(int processDefinitionId) { + this.processDefinitionId = processDefinitionId; + } + + public long getVersion() { + return version; + } + + public void setVersion(long version) { + this.version = version; + } + + public String getProcessDefinitionJson() { + return processDefinitionJson; + } + + public void setProcessDefinitionJson(String processDefinitionJson) { + this.processDefinitionJson = processDefinitionJson; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public Date getCreateTime() { + return createTime; + } + + public void setCreateTime(Date createTime) { + this.createTime = createTime; + } + + public String getLocations() { + return locations; + } + + public void setLocations(String locations) { + this.locations = locations; + } + + public String getConnects() { + return connects; + } + + public void setConnects(String connects) { + this.connects = connects; + } + + public String getReceivers() { + return receivers; + } + + public void setReceivers(String receivers) { + this.receivers = receivers; + } + + public String getReceiversCc() { + return receiversCc; + } + + public void setReceiversCc(String receiversCc) { + this.receiversCc = receiversCc; + } + + public int getTimeout() { + return timeout; + } + + public void setTimeout(int timeout) { + this.timeout = timeout; + } + + public String getResourceIds() { + return resourceIds; + } + + public void setResourceIds(String resourceIds) { + this.resourceIds = resourceIds; + } + + @Override + public String toString() { + return "ProcessDefinitionVersion{" + + "id=" + id + + ", processDefinitionId=" + processDefinitionId + + ", version=" + version + + ", processDefinitionJson='" + processDefinitionJson + '\'' + + ", description='" + description + '\'' + + ", globalParams='" + globalParams + '\'' + + ", createTime=" + createTime + + ", locations='" + locations + '\'' + + ", connects='" + connects + '\'' + + ", receivers='" + receivers + '\'' + + ", receiversCc='" + receiversCc + '\'' + + ", timeout=" + timeout + + ", resourceIds='" + resourceIds + '\'' + + '}'; + } + + public static Builder newBuilder() { + return new Builder(); + } + + public static final class Builder { + private int id; + private int processDefinitionId; + private long version; + private String processDefinitionJson; + private String description; + private String globalParams; + private Date createTime; + private String locations; + private String connects; + private String receivers; + private String receiversCc; + private int timeout; + private String resourceIds; + + private Builder() { + } + + public Builder id(int id) { + this.id = id; + return this; + } + + public Builder processDefinitionId(int processDefinitionId) { + this.processDefinitionId = processDefinitionId; + return this; + } + + public Builder version(long version) { + this.version = version; + return this; + } + + public Builder processDefinitionJson(String processDefinitionJson) { + this.processDefinitionJson = processDefinitionJson; + return this; + } + + public Builder description(String description) { + this.description = description; + return this; + } + + public Builder globalParams(String globalParams) { + this.globalParams = globalParams; + return this; + } + + public Builder createTime(Date createTime) { + this.createTime = createTime; + return this; + } + + public Builder locations(String locations) { + this.locations = locations; + return this; + } + + public Builder connects(String connects) { + this.connects = connects; + return this; + } + + public Builder receivers(String receivers) { + this.receivers = receivers; + return this; + } + + public Builder receiversCc(String receiversCc) { + this.receiversCc = receiversCc; + return this; + } + + public Builder timeout(int timeout) { + this.timeout = timeout; + return this; + } + + public Builder resourceIds(String resourceIds) { + this.resourceIds = resourceIds; + return this; + } + + public ProcessDefinitionVersion build() { + ProcessDefinitionVersion processDefinitionVersion = new ProcessDefinitionVersion(); + processDefinitionVersion.setId(id); + processDefinitionVersion.setProcessDefinitionId(processDefinitionId); + processDefinitionVersion.setVersion(version); + processDefinitionVersion.setProcessDefinitionJson(processDefinitionJson); + processDefinitionVersion.setDescription(description); + processDefinitionVersion.setGlobalParams(globalParams); + processDefinitionVersion.setCreateTime(createTime); + processDefinitionVersion.setLocations(locations); + processDefinitionVersion.setConnects(connects); + processDefinitionVersion.setReceivers(receivers); + processDefinitionVersion.setReceiversCc(receiversCc); + processDefinitionVersion.setTimeout(timeout); + processDefinitionVersion.setResourceIds(resourceIds); + return processDefinitionVersion; + } + } +} diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Project.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Project.java index feddb598f0..6726aa7dad 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Project.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Project.java @@ -16,13 +16,13 @@ */ package org.apache.dolphinscheduler.dao.entity; +import java.util.Date; + import com.baomidou.mybatisplus.annotation.IdType; import com.baomidou.mybatisplus.annotation.TableField; import com.baomidou.mybatisplus.annotation.TableId; import com.baomidou.mybatisplus.annotation.TableName; -import java.util.Date; - /** * project */ @@ -32,7 +32,7 @@ public class Project { /** * id */ - @TableId(value="id", type=IdType.AUTO) + @TableId(value = "id", type = IdType.AUTO) private int id; /** @@ -44,7 +44,7 @@ public class Project { /** * user name */ - @TableField(exist=false) + @TableField(exist = false) private String userName; /** @@ -70,19 +70,19 @@ public class Project { /** * permission */ - @TableField(exist=false) + @TableField(exist = false) private int perm; /** * process define count */ - @TableField(exist=false) + @TableField(exist = false) private int defCount; /** * process instance running count */ - @TableField(exist=false) + @TableField(exist = false) private int instRunningCount; public int getDefCount() { @@ -136,6 +136,7 @@ public class Project { public void setDescription(String description) { this.description = description; } + public String getDescription() { return description; } @@ -163,6 +164,7 @@ public class Project { public void setPerm(int perm) { this.perm = perm; } + @Override public String toString() { return "Project{" + @@ -176,7 +178,6 @@ public class Project { '}'; } - @Override public boolean equals(Object o) { if (this == o) { @@ -202,4 +203,88 @@ public class Project { return result; } + public static Builder newBuilder() { + return new Builder(); + } + + public static final class Builder { + private int id; + private int userId; + private String userName; + private String name; + private String description; + private Date createTime; + private Date updateTime; + private int perm; + private int defCount; + private int instRunningCount; + + private Builder() { + } + + public Builder id(int id) { + this.id = id; + return this; + } + + public Builder userId(int userId) { + this.userId = userId; + return this; + } + + public Builder userName(String userName) { + this.userName = userName; + return this; + } + + public Builder name(String name) { + this.name = name; + return this; + } + + public Builder description(String description) { + this.description = description; + return this; + } + + public Builder createTime(Date createTime) { + this.createTime = createTime; + return this; + } + + public Builder updateTime(Date updateTime) { + this.updateTime = updateTime; + return this; + } + + public Builder perm(int perm) { + this.perm = perm; + return this; + } + + public Builder defCount(int defCount) { + this.defCount = defCount; + return this; + } + + public Builder instRunningCount(int instRunningCount) { + this.instRunningCount = instRunningCount; + return this; + } + + public Project build() { + Project project = new Project(); + project.setId(id); + project.setUserId(userId); + project.setUserName(userName); + project.setName(name); + project.setDescription(description); + project.setCreateTime(createTime); + project.setUpdateTime(updateTime); + project.setPerm(perm); + project.setDefCount(defCount); + project.setInstRunningCount(instRunningCount); + return project; + } + } } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ServerAlertContent.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ServerAlertContent.java new file mode 100644 index 0000000000..211863f73f --- /dev/null +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ServerAlertContent.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.dao.entity; + +import org.apache.dolphinscheduler.common.enums.AlertEvent; +import org.apache.dolphinscheduler.common.enums.AlertWarnLevel; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public class ServerAlertContent { + + /** + * server type :master or worker + */ + @JsonProperty("type") + final String type; + @JsonProperty("host") + final String host; + @JsonProperty("event") + final AlertEvent event; + @JsonProperty("warningLevel") + final AlertWarnLevel warningLevel; + + private ServerAlertContent(Builder builder) { + this.type = builder.type; + this.host = builder.host; + this.event = builder.event; + this.warningLevel = builder.warningLevel; + + } + + public static Builder newBuilder() { + return new Builder(); + } + + public static class Builder { + private String type; + + private String host; + + private AlertEvent event; + + private AlertWarnLevel warningLevel; + + public Builder type(String type) { + this.type = type; + return this; + } + + public Builder host(String host) { + this.host = host; + return this; + } + + public Builder event(AlertEvent event) { + this.event = event; + return this; + } + + public Builder warningLevel(AlertWarnLevel warningLevel) { + this.warningLevel = warningLevel; + return this; + } + + public ServerAlertContent build() { + return new ServerAlertContent(this); + } + } + +} diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskInstance.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskInstance.java index a90d927154..9688200b2c 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskInstance.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskInstance.java @@ -16,24 +16,23 @@ */ package org.apache.dolphinscheduler.dao.entity; -import com.baomidou.mybatisplus.annotation.IdType; -import com.baomidou.mybatisplus.annotation.TableField; -import com.fasterxml.jackson.annotation.JsonFormat; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.enums.Flag; import org.apache.dolphinscheduler.common.enums.Priority; import org.apache.dolphinscheduler.common.enums.TaskType; import org.apache.dolphinscheduler.common.model.TaskNode; -import org.apache.dolphinscheduler.common.utils.*; -import com.baomidou.mybatisplus.annotation.IdType; -import com.baomidou.mybatisplus.annotation.TableId; -import com.baomidou.mybatisplus.annotation.TableName; import org.apache.dolphinscheduler.common.utils.JSONUtils; import java.io.Serializable; import java.util.Date; import java.util.Map; +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableField; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import com.fasterxml.jackson.annotation.JsonFormat; + /** * task instance */ @@ -43,7 +42,7 @@ public class TaskInstance implements Serializable { /** * id */ - @TableId(value="id", type=IdType.AUTO) + @TableId(value = "id", type = IdType.AUTO) private int id; /** @@ -52,7 +51,6 @@ public class TaskInstance implements Serializable { private String name; - /** * task type */ @@ -84,22 +82,28 @@ public class TaskInstance implements Serializable { */ private ExecutionStatus state; + /** + * task first submit time. + */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss", timezone = "GMT+8") + private Date firstSubmitTime; + /** * task submit time */ - @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss", timezone = "GMT+8") private Date submitTime; /** * task start time */ - @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss", timezone = "GMT+8") private Date startTime; /** * task end time */ - @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss", timezone = "GMT+8") private Date endTime; /** @@ -215,11 +219,14 @@ public class TaskInstance implements Serializable { @TableField(exist = false) - private Map resources; + private Map resources; + /** + * delay execution time. + */ + private int delayTime; - - public void init(String host,Date startTime,String executePath){ + public void init(String host, Date startTime, String executePath) { this.host = host; this.startTime = startTime; this.executePath = executePath; @@ -298,6 +305,14 @@ public class TaskInstance implements Serializable { this.state = state; } + public Date getFirstSubmitTime() { + return firstSubmitTime; + } + + public void setFirstSubmitTime(Date firstSubmitTime) { + this.firstSubmitTime = firstSubmitTime; + } + public Date getSubmitTime() { return submitTime; } @@ -362,7 +377,7 @@ public class TaskInstance implements Serializable { this.retryTimes = retryTimes; } - public Boolean isTaskSuccess(){ + public Boolean isTaskSuccess() { return this.state == ExecutionStatus.SUCCESS; } @@ -382,16 +397,16 @@ public class TaskInstance implements Serializable { this.appLink = appLink; } - - - public String getDependency(){ - - if(this.dependency != null){ + public String getDependency() { + if (this.dependency != null) { return this.dependency; } TaskNode taskNode = JSONUtils.parseObject(taskJson, TaskNode.class); + return taskNode == null ? null : taskNode.getDependence(); + } - return taskNode.getDependence(); + public void setDependency(String dependency) { + this.dependency = dependency; } public Flag getFlag() { @@ -401,6 +416,7 @@ public class TaskInstance implements Serializable { public void setFlag(Flag flag) { this.flag = flag; } + public String getProcessInstanceName() { return processInstanceName; } @@ -465,40 +481,36 @@ public class TaskInstance implements Serializable { this.resources = resources; } - public boolean isSubProcess(){ + public boolean isSubProcess() { return TaskType.SUB_PROCESS.equals(TaskType.valueOf(this.taskType)); } - public boolean isDependTask(){ + public boolean isDependTask() { return TaskType.DEPENDENT.equals(TaskType.valueOf(this.taskType)); } - public boolean isConditionsTask(){ + public boolean isConditionsTask() { return TaskType.CONDITIONS.equals(TaskType.valueOf(this.taskType)); } - /** * determine if you can try again + * * @return can try result */ public boolean taskCanRetry() { - if(this.isSubProcess()){ + if (this.isSubProcess()) { return false; } - if(this.getState() == ExecutionStatus.NEED_FAULT_TOLERANCE){ + if (this.getState() == ExecutionStatus.NEED_FAULT_TOLERANCE) { return true; - }else { + } else { return (this.getState().typeIsFailure() - && this.getRetryTimes() < this.getMaxRetryTimes()); + && this.getRetryTimes() < this.getMaxRetryTimes()); } } - public void setDependency(String dependency) { - this.dependency = dependency; - } - public Priority getTaskInstancePriority() { return taskInstancePriority; } @@ -531,40 +543,50 @@ public class TaskInstance implements Serializable { this.dependentResult = dependentResult; } + public int getDelayTime() { + return delayTime; + } + + public void setDelayTime(int delayTime) { + this.delayTime = delayTime; + } + @Override public String toString() { - return "TaskInstance{" + - "id=" + id + - ", name='" + name + '\'' + - ", taskType='" + taskType + '\'' + - ", processDefinitionId=" + processDefinitionId + - ", processInstanceId=" + processInstanceId + - ", processInstanceName='" + processInstanceName + '\'' + - ", taskJson='" + taskJson + '\'' + - ", state=" + state + - ", submitTime=" + submitTime + - ", startTime=" + startTime + - ", endTime=" + endTime + - ", host='" + host + '\'' + - ", executePath='" + executePath + '\'' + - ", logPath='" + logPath + '\'' + - ", retryTimes=" + retryTimes + - ", alertFlag=" + alertFlag + - ", processInstance=" + processInstance + - ", processDefine=" + processDefine + - ", pid=" + pid + - ", appLink='" + appLink + '\'' + - ", flag=" + flag + - ", dependency='" + dependency + '\'' + - ", duration=" + duration + - ", maxRetryTimes=" + maxRetryTimes + - ", retryInterval=" + retryInterval + - ", taskInstancePriority=" + taskInstancePriority + - ", processInstancePriority=" + processInstancePriority + - ", dependentResult='" + dependentResult + '\'' + - ", workerGroup='" + workerGroup + '\'' + - ", executorId=" + executorId + - ", executorName='" + executorName + '\'' + - '}'; + return "TaskInstance{" + + "id=" + id + + ", name='" + name + '\'' + + ", taskType='" + taskType + '\'' + + ", processDefinitionId=" + processDefinitionId + + ", processInstanceId=" + processInstanceId + + ", processInstanceName='" + processInstanceName + '\'' + + ", taskJson='" + taskJson + '\'' + + ", state=" + state + + ", firstSubmitTime=" + firstSubmitTime + + ", submitTime=" + submitTime + + ", startTime=" + startTime + + ", endTime=" + endTime + + ", host='" + host + '\'' + + ", executePath='" + executePath + '\'' + + ", logPath='" + logPath + '\'' + + ", retryTimes=" + retryTimes + + ", alertFlag=" + alertFlag + + ", processInstance=" + processInstance + + ", processDefine=" + processDefine + + ", pid=" + pid + + ", appLink='" + appLink + '\'' + + ", flag=" + flag + + ", dependency='" + dependency + '\'' + + ", duration=" + duration + + ", maxRetryTimes=" + maxRetryTimes + + ", retryInterval=" + retryInterval + + ", taskInstancePriority=" + taskInstancePriority + + ", processInstancePriority=" + processInstancePriority + + ", dependentResult='" + dependentResult + '\'' + + ", workerGroup='" + workerGroup + '\'' + + ", executorId=" + executorId + + ", executorName='" + executorName + '\'' + + ", delayTime=" + delayTime + + '}'; } } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapper.java index 4df93f2e9f..86e3172f23 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapper.java @@ -14,18 +14,21 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.dao.mapper; import org.apache.dolphinscheduler.dao.entity.DefinitionGroupByUser; import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; -import com.baomidou.mybatisplus.core.mapper.BaseMapper; -import com.baomidou.mybatisplus.core.metadata.IPage; + import org.apache.ibatis.annotations.MapKey; import org.apache.ibatis.annotations.Param; import java.util.List; import java.util.Map; +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.baomidou.mybatisplus.core.metadata.IPage; + /** * process definition mapper interface */ @@ -34,6 +37,7 @@ public interface ProcessDefinitionMapper extends BaseMapper { /** * query process definition by name + * * @param projectId projectId * @param name name * @return process definition @@ -43,6 +47,7 @@ public interface ProcessDefinitionMapper extends BaseMapper { /** * query process definition by id + * * @param processDefineId processDefineId * @return process definition */ @@ -50,6 +55,7 @@ public interface ProcessDefinitionMapper extends BaseMapper { /** * process definition page + * * @param page page * @param searchVal searchVal * @param userId userId @@ -65,6 +71,7 @@ public interface ProcessDefinitionMapper extends BaseMapper { /** * query all process definition list + * * @param projectId projectId * @return process definition list */ @@ -72,6 +79,7 @@ public interface ProcessDefinitionMapper extends BaseMapper { /** * query process definition by ids + * * @param ids ids * @return process definition list */ @@ -79,6 +87,7 @@ public interface ProcessDefinitionMapper extends BaseMapper { /** * query process definition by tenant + * * @param tenantId tenantId * @return process definition list */ @@ -86,6 +95,7 @@ public interface ProcessDefinitionMapper extends BaseMapper { /** * count process definition group by user + * * @param userId userId * @param projectIds projectIds * @param isAdmin isAdmin @@ -98,6 +108,7 @@ public interface ProcessDefinitionMapper extends BaseMapper { /** * list all resource ids + * * @return resource ids list */ @MapKey("id") @@ -105,8 +116,17 @@ public interface ProcessDefinitionMapper extends BaseMapper { /** * list all resource ids by user id + * * @return resource ids list */ @MapKey("id") List> listResourcesByUser(@Param("userId") Integer userId); + + /** + * update process definition version by process definitionId + * + * @param processDefinitionId process definition id + * @param version version + */ + void updateVersionByProcessDefinitionId(@Param("processDefinitionId") int processDefinitionId, @Param("version") long version); } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionVersionMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionVersionMapper.java new file mode 100644 index 0000000000..27efda4327 --- /dev/null +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionVersionMapper.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.dao.mapper; + +import org.apache.dolphinscheduler.dao.entity.ProcessDefinitionVersion; + +import org.apache.ibatis.annotations.Param; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; + +/** + * process definition mapper interface + */ +public interface ProcessDefinitionVersionMapper extends BaseMapper { + + /** + * query max version by process definition id + * + * @param processDefinitionId process definition id + * @return the max version of this process definition id + */ + Long queryMaxVersionByProcessDefinitionId(@Param("processDefinitionId") int processDefinitionId); + + /** + * query the paging process definition version list by pagination info + * + * @param page pagination info + * @param processDefinitionId process definition id + * @return the paging process definition version list + */ + IPage queryProcessDefinitionVersionsPaging(Page page, + @Param("processDefinitionId") int processDefinitionId); + + /** + * query the certain process definition version info by process definition id and version number + * + * @param processDefinitionId process definition id + * @param version version number + * @return the process definition version info + */ + ProcessDefinitionVersion queryByProcessDefinitionIdAndVersion(@Param("processDefinitionId") int processDefinitionId, @Param("version") long version); + + /** + * delete the certain process definition version by process definition id and version number + * + * @param processDefinitionId process definition id + * @param version version number + * @return delete result + */ + int deleteByProcessDefinitionIdAndVersion(@Param("processDefinitionId") int processDefinitionId, @Param("version") long version); + +} diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapper.java index b7bd081cfe..8048fda812 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapper.java @@ -14,19 +14,22 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.dao.mapper; +import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.dao.entity.ExecuteStatusCount; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; -import com.baomidou.mybatisplus.core.mapper.BaseMapper; -import com.baomidou.mybatisplus.core.metadata.IPage; -import com.baomidou.mybatisplus.extension.plugins.pagination.Page; -import org.apache.dolphinscheduler.common.enums.ExecutionStatus; + import org.apache.ibatis.annotations.Param; import java.util.Date; import java.util.List; +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; + /** * process instance mapper interface */ @@ -201,9 +204,20 @@ public interface ProcessInstanceMapper extends BaseMapper { * @param endTime * @return ProcessInstance list */ + List queryTopNProcessInstance(@Param("size") int size, @Param("startTime") Date startTime, @Param("endTime") Date endTime, @Param("status")ExecutionStatus status); + /** + * query process instance by processDefinitionId and stateArray + * @param processDefinitionId processDefinitionId + * @param states states array + * @return process instance list + */ + + List queryByProcessDefineIdAndStatus( + @Param("processDefinitionId") int processDefinitionId, + @Param("states") int[] states); } diff --git a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapper.xml b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapper.xml index 3e538a23e0..0481f7deab 100644 --- a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapper.xml +++ b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapper.xml @@ -102,4 +102,10 @@ FROM t_ds_process_definition WHERE user_id = #{userId} and release_state = 1 and resource_ids is not null and resource_ids != '' + + + update t_ds_process_definition + set version = #{version} + where id = #{processDefinitionId} + \ No newline at end of file diff --git a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionVersionMapper.xml b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionVersionMapper.xml new file mode 100644 index 0000000000..b2d0b85982 --- /dev/null +++ b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionVersionMapper.xml @@ -0,0 +1,47 @@ + + + + + + + + + + + + + delete + from t_ds_process_definition_version + where process_definition_id = #{processDefinitionId} + and version = #{version} + + \ No newline at end of file diff --git a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapper.xml b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapper.xml index bbc331a67d..831c4a9c23 100644 --- a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapper.xml +++ b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapper.xml @@ -191,6 +191,16 @@ order by end_time desc limit 1 - + \ No newline at end of file diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/entity/TaskInstanceTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/entity/TaskInstanceTest.java index 9c59670872..5742c95a5d 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/entity/TaskInstanceTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/entity/TaskInstanceTest.java @@ -16,6 +16,9 @@ */ package org.apache.dolphinscheduler.dao.entity; +import org.apache.dolphinscheduler.common.model.TaskNode; +import org.apache.dolphinscheduler.common.utils.JSONUtils; + import org.junit.Assert; import org.junit.Test; @@ -43,7 +46,36 @@ public class TaskInstanceTest { //sub process taskInstance.setTaskType("DEPENDENT"); Assert.assertTrue(taskInstance.isDependTask()); + } + /** + * test for TaskInstance.getDependence + */ + @Test + public void testTaskInstanceGetDependence() { + TaskInstance taskInstance; + TaskNode taskNode; + taskInstance = new TaskInstance(); + taskInstance.setTaskJson(null); + Assert.assertNull(taskInstance.getDependency()); + + taskInstance = new TaskInstance(); + taskNode = new TaskNode(); + taskNode.setDependence(null); + taskInstance.setTaskJson(JSONUtils.toJsonString(taskNode)); + Assert.assertNull(taskInstance.getDependency()); + + taskInstance = new TaskInstance(); + taskNode = new TaskNode(); + // expect a JSON here, and will be unwrap when toJsonString + taskNode.setDependence("\"A\""); + taskInstance.setTaskJson(JSONUtils.toJsonString(taskNode)); + Assert.assertEquals("A", taskInstance.getDependency()); + + taskInstance = new TaskInstance(); + taskInstance.setTaskJson(null); + taskInstance.setDependency("{}"); + Assert.assertEquals("{}", taskInstance.getDependency()); } } diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AccessTokenMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AccessTokenMapperTest.java index 0c1dc20ac9..30c8cdc7b9 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AccessTokenMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AccessTokenMapperTest.java @@ -34,6 +34,7 @@ import org.springframework.transaction.annotation.Transactional; import javax.annotation.Resource; import java.text.SimpleDateFormat; import java.util.*; +import java.util.concurrent.ThreadLocalRandom; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.greaterThan; @@ -45,7 +46,7 @@ import static org.junit.Assert.*; @RunWith(SpringRunner.class) @SpringBootTest @Transactional -@Rollback(true) +@Rollback public class AccessTokenMapperTest { @Autowired @@ -56,10 +57,11 @@ public class AccessTokenMapperTest { /** * test insert + * * @throws Exception */ @Test - public void testInsert() throws Exception{ + public void testInsert() throws Exception { Integer userId = 1; AccessToken accessToken = createAccessToken(userId); @@ -69,10 +71,11 @@ public class AccessTokenMapperTest { /** * test select by id + * * @throws Exception */ @Test - public void testSelectById() throws Exception{ + public void testSelectById() throws Exception { Integer userId = 1; AccessToken accessToken = createAccessToken(userId); AccessToken resultAccessToken = accessTokenMapper.selectById(accessToken.getId()); @@ -81,6 +84,7 @@ public class AccessTokenMapperTest { /** * test hashCode method + * * @throws Exception */ @Test @@ -94,6 +98,7 @@ public class AccessTokenMapperTest { /** * test equals method + * * @throws Exception */ @Test @@ -108,7 +113,7 @@ public class AccessTokenMapperTest { * test page */ @Test - public void testSelectAccessTokenPage() throws Exception{ + public void testSelectAccessTokenPage() throws Exception { Integer count = 4; String userName = "zhangsan"; @@ -120,11 +125,11 @@ public class AccessTokenMapperTest { Page page = new Page(offset, size); IPage accessTokenPage = accessTokenMapper.selectAccessTokenPage(page, userName, 0); - assertEquals(Integer.valueOf(accessTokenPage.getRecords().size()),size); + assertEquals(Integer.valueOf(accessTokenPage.getRecords().size()), size); - for (AccessToken accessToken : accessTokenPage.getRecords()){ + for (AccessToken accessToken : accessTokenPage.getRecords()) { AccessToken resultAccessToken = accessTokenMap.get(accessToken.getId()); - assertEquals(accessToken,resultAccessToken); + assertEquals(accessToken, resultAccessToken); } } @@ -133,14 +138,17 @@ public class AccessTokenMapperTest { * test update */ @Test - public void testUpdate() throws Exception{ + public void testUpdate() throws Exception { Integer userId = 1; AccessToken accessToken = createAccessToken(userId); //update accessToken.setToken("56789"); accessToken.setExpireTime(DateUtils.getCurrentDate()); accessToken.setUpdateTime(DateUtils.getCurrentDate()); - accessTokenMapper.updateById(accessToken); + int status = accessTokenMapper.updateById(accessToken); + if (status != 1) { + Assert.fail("update access token fail"); + } AccessToken resultAccessToken = accessTokenMapper.selectById(accessToken.getId()); assertEquals(accessToken, resultAccessToken); } @@ -149,11 +157,14 @@ public class AccessTokenMapperTest { * test delete */ @Test - public void testDelete() throws Exception{ + public void testDelete() throws Exception { Integer userId = 1; AccessToken accessToken = createAccessToken(userId); - accessTokenMapper.deleteById(accessToken.getId()); + int status = accessTokenMapper.deleteById(accessToken.getId()); + if (status != 1) { + Assert.fail("delete access token data fail"); + } AccessToken resultAccessToken = accessTokenMapper.selectById(accessToken.getId()); @@ -163,21 +174,22 @@ public class AccessTokenMapperTest { /** * create accessTokens - * @param count create accessToken count + * + * @param count create accessToken count * @param userName username * @return accessToken map * @throws Exception */ - private Map createAccessTokens( - Integer count,String userName) throws Exception{ + private Map createAccessTokens( + Integer count, String userName) throws Exception { User user = createUser(userName); - Map accessTokenMap = new HashMap<>(); - for (int i = 1 ; i<= count ; i++){ - AccessToken accessToken = createAccessToken(user.getId(),userName); + Map accessTokenMap = new HashMap<>(); + for (int i = 1; i <= count; i++) { + AccessToken accessToken = createAccessToken(user.getId(), userName); - accessTokenMap.put(accessToken.getId(),accessToken); + accessTokenMap.put(accessToken.getId(), accessToken); } return accessTokenMap; @@ -185,11 +197,12 @@ public class AccessTokenMapperTest { /** * create user + * * @param userName userName * @return user * @throws Exception */ - private User createUser(String userName) throws Exception{ + private User createUser(String userName) throws Exception { User user = new User(); user.setUserName(userName); user.setUserPassword("123"); @@ -201,42 +214,50 @@ public class AccessTokenMapperTest { user.setUpdateTime(DateUtils.getCurrentDate()); user.setQueue("default"); - userMapper.insert(user); + int status = userMapper.insert(user); + + if (status != 1) { + Assert.fail("insert user data error"); + } return user; } /** * create access token - * @param userId userId + * + * @param userId userId * @param userName userName * @return accessToken * @throws Exception */ - private AccessToken createAccessToken(Integer userId,String userName)throws Exception{ - Random random = new Random(); + private AccessToken createAccessToken(Integer userId, String userName) throws Exception { //insertOne AccessToken accessToken = new AccessToken(); accessToken.setUserName(userName); accessToken.setUserId(userId); - accessToken.setToken(String.valueOf(random.nextLong())); + accessToken.setToken(String.valueOf(ThreadLocalRandom.current().nextLong())); accessToken.setCreateTime(DateUtils.getCurrentDate()); accessToken.setUpdateTime(DateUtils.getCurrentDate()); accessToken.setExpireTime(DateUtils.getCurrentDate()); - accessTokenMapper.insert(accessToken); + int status = accessTokenMapper.insert(accessToken); + if (status != 1) { + Assert.fail("insert data error"); + } return accessToken; } /** * create access token + * * @param userId userId * @return accessToken * @throws Exception */ - private AccessToken createAccessToken(Integer userId)throws Exception{ - return createAccessToken(userId,null); + private AccessToken createAccessToken(Integer userId) throws Exception { + return createAccessToken(userId, null); } } \ No newline at end of file diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapperTest.java index ad91e79fb5..c58c92b3bb 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapperTest.java @@ -14,14 +14,22 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.dolphinscheduler.dao.mapper; +package org.apache.dolphinscheduler.dao.mapper; import org.apache.dolphinscheduler.common.enums.ReleaseState; import org.apache.dolphinscheduler.common.enums.UserType; -import com.baomidou.mybatisplus.core.metadata.IPage; -import com.baomidou.mybatisplus.extension.plugins.pagination.Page; -import org.apache.dolphinscheduler.dao.entity.*; +import org.apache.dolphinscheduler.dao.entity.DefinitionGroupByUser; +import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; +import org.apache.dolphinscheduler.dao.entity.Project; +import org.apache.dolphinscheduler.dao.entity.Queue; +import org.apache.dolphinscheduler.dao.entity.Tenant; +import org.apache.dolphinscheduler.dao.entity.User; + +import java.util.Date; +import java.util.List; +import java.util.Map; + import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; @@ -31,9 +39,8 @@ import org.springframework.test.annotation.Rollback; import org.springframework.test.context.junit4.SpringRunner; import org.springframework.transaction.annotation.Transactional; -import java.util.Date; -import java.util.List; -import java.util.Map; +import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; @RunWith(SpringRunner.class) @SpringBootTest @@ -59,9 +66,10 @@ public class ProcessDefinitionMapperTest { /** * insert + * * @return ProcessDefinition */ - private ProcessDefinition insertOne(){ + private ProcessDefinition insertOne() { //insertOne ProcessDefinition processDefinition = new ProcessDefinition(); processDefinition.setName("def 1"); @@ -77,9 +85,10 @@ public class ProcessDefinitionMapperTest { /** * insert + * * @return ProcessDefinition */ - private ProcessDefinition insertTwo(){ + private ProcessDefinition insertTwo() { //insertOne ProcessDefinition processDefinition = new ProcessDefinition(); processDefinition.setName("def 2"); @@ -95,7 +104,7 @@ public class ProcessDefinitionMapperTest { * test update */ @Test - public void testUpdate(){ + public void testUpdate() { //insertOne ProcessDefinition processDefinition = insertOne(); //update @@ -108,7 +117,7 @@ public class ProcessDefinitionMapperTest { * test delete */ @Test - public void testDelete(){ + public void testDelete() { ProcessDefinition processDefinition = insertOne(); int delete = processDefinitionMapper.deleteById(processDefinition.getId()); Assert.assertEquals(1, delete); @@ -175,8 +184,8 @@ public class ProcessDefinitionMapperTest { @Test public void testQueryDefineListPaging() { ProcessDefinition processDefinition = insertOne(); - Page page = new Page(1,3); - IPage processDefinitionIPage = processDefinitionMapper.queryDefineListPaging(page, "def", 101, 1010,true); + Page page = new Page(1, 3); + IPage processDefinitionIPage = processDefinitionMapper.queryDefineListPaging(page, "def", 101, 1010, true); Assert.assertNotEquals(processDefinitionIPage.getTotal(), 0); } @@ -186,7 +195,7 @@ public class ProcessDefinitionMapperTest { @Test public void testQueryAllDefinitionList() { ProcessDefinition processDefinition = insertOne(); - List processDefinitionIPage = processDefinitionMapper.queryAllDefinitionList(1010); + List processDefinitionIPage = processDefinitionMapper.queryAllDefinitionList(1010); Assert.assertNotEquals(processDefinitionIPage.size(), 0); } @@ -214,7 +223,7 @@ public class ProcessDefinitionMapperTest { @Test public void testCountDefinitionGroupByUser() { - User user= new User(); + User user = new User(); user.setUserName("user1"); user.setUserPassword("1"); user.setEmail("xx@123.com"); @@ -239,7 +248,7 @@ public class ProcessDefinitionMapperTest { } @Test - public void listResourcesTest(){ + public void listResourcesTest() { ProcessDefinition processDefinition = insertOne(); processDefinition.setResourceIds("3,5"); processDefinition.setReleaseState(ReleaseState.ONLINE); @@ -248,11 +257,22 @@ public class ProcessDefinitionMapperTest { } @Test - public void listResourcesByUserTest(){ + public void listResourcesByUserTest() { ProcessDefinition processDefinition = insertOne(); processDefinition.setResourceIds("3,5"); processDefinition.setReleaseState(ReleaseState.ONLINE); List> maps = processDefinitionMapper.listResourcesByUser(processDefinition.getUserId()); Assert.assertNotNull(maps); } + + @Test + public void testUpdateVersionByProcessDefinitionId() { + long expectedVersion = 10; + ProcessDefinition processDefinition = insertOne(); + processDefinition.setVersion(expectedVersion); + processDefinitionMapper.updateVersionByProcessDefinitionId( + processDefinition.getId(), processDefinition.getVersion()); + ProcessDefinition processDefinition1 = processDefinitionMapper.selectById(processDefinition.getId()); + Assert.assertEquals(expectedVersion, processDefinition1.getVersion()); + } } \ No newline at end of file diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionVersionMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionVersionMapperTest.java new file mode 100644 index 0000000000..e825e33847 --- /dev/null +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionVersionMapperTest.java @@ -0,0 +1,172 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.dao.mapper; + +import org.apache.dolphinscheduler.common.utils.StringUtils; +import org.apache.dolphinscheduler.dao.entity.ProcessDefinitionVersion; + +import java.util.Date; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.annotation.Rollback; +import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.transaction.annotation.Transactional; + +import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; + +@RunWith(SpringRunner.class) +@SpringBootTest +@Transactional +@Rollback(true) +public class ProcessDefinitionVersionMapperTest { + + + @Autowired + ProcessDefinitionMapper processDefinitionMapper; + + @Autowired + ProcessDefinitionVersionMapper processDefinitionVersionMapper; + + @Autowired + UserMapper userMapper; + + @Autowired + QueueMapper queueMapper; + + @Autowired + TenantMapper tenantMapper; + + @Autowired + ProjectMapper projectMapper; + + /** + * insert + * + * @return ProcessDefinition + */ + private ProcessDefinitionVersion insertOne() { + // insertOne + ProcessDefinitionVersion processDefinitionVersion + = new ProcessDefinitionVersion(); + processDefinitionVersion.setProcessDefinitionId(66); + processDefinitionVersion.setVersion(10); + processDefinitionVersion.setProcessDefinitionJson(StringUtils.EMPTY); + processDefinitionVersion.setDescription(StringUtils.EMPTY); + processDefinitionVersion.setGlobalParams(StringUtils.EMPTY); + processDefinitionVersion.setCreateTime(new Date()); + processDefinitionVersion.setLocations(StringUtils.EMPTY); + processDefinitionVersion.setConnects(StringUtils.EMPTY); + processDefinitionVersion.setReceivers(StringUtils.EMPTY); + processDefinitionVersion.setReceiversCc(StringUtils.EMPTY); + processDefinitionVersion.setTimeout(10); + processDefinitionVersion.setResourceIds("1,2"); + processDefinitionVersionMapper.insert(processDefinitionVersion); + return processDefinitionVersion; + } + + /** + * insert + * + * @return ProcessDefinitionVersion + */ + private ProcessDefinitionVersion insertTwo() { + // insertTwo + ProcessDefinitionVersion processDefinitionVersion + = new ProcessDefinitionVersion(); + processDefinitionVersion.setProcessDefinitionId(67); + processDefinitionVersion.setVersion(11); + processDefinitionVersion.setProcessDefinitionJson(StringUtils.EMPTY); + processDefinitionVersion.setDescription(StringUtils.EMPTY); + processDefinitionVersion.setGlobalParams(StringUtils.EMPTY); + processDefinitionVersion.setCreateTime(new Date()); + processDefinitionVersion.setLocations(StringUtils.EMPTY); + processDefinitionVersion.setConnects(StringUtils.EMPTY); + processDefinitionVersion.setReceivers(StringUtils.EMPTY); + processDefinitionVersion.setReceiversCc(StringUtils.EMPTY); + processDefinitionVersion.setTimeout(10); + processDefinitionVersion.setResourceIds("1,2"); + processDefinitionVersionMapper.insert(processDefinitionVersion); + return processDefinitionVersion; + } + + /** + * test insert + */ + @Test + public void testInsert() { + ProcessDefinitionVersion processDefinitionVersion = insertOne(); + Assert.assertTrue(processDefinitionVersion.getId() > 0); + } + + /** + * test query + */ + @Test + public void testQueryMaxVersionByProcessDefinitionId() { + ProcessDefinitionVersion processDefinitionVersion = insertOne(); + + Long version = processDefinitionVersionMapper.queryMaxVersionByProcessDefinitionId( + processDefinitionVersion.getProcessDefinitionId()); + // query + Assert.assertEquals(10, (long) version); + } + + @Test + public void testQueryProcessDefinitionVersionsPaging() { + insertOne(); + insertTwo(); + + Page page = new Page<>(1, 3); + + IPage processDefinitionVersionIPage = + processDefinitionVersionMapper.queryProcessDefinitionVersionsPaging(page, 10); + + Assert.assertTrue(processDefinitionVersionIPage.getSize() >= 2); + } + + @Test + public void testDeleteByProcessDefinitionIdAndVersion() { + ProcessDefinitionVersion processDefinitionVersion = insertOne(); + int i = processDefinitionVersionMapper.deleteByProcessDefinitionIdAndVersion( + processDefinitionVersion.getProcessDefinitionId(), processDefinitionVersion.getVersion()); + Assert.assertEquals(1, i); + } + + @Test + public void testQueryByProcessDefinitionIdAndVersion() { + ProcessDefinitionVersion processDefinitionVersion1 = insertOne(); + ProcessDefinitionVersion processDefinitionVersion3 = processDefinitionVersionMapper.queryByProcessDefinitionIdAndVersion( + processDefinitionVersion1.getProcessDefinitionId(), 10); + + ProcessDefinitionVersion processDefinitionVersion2 = insertTwo(); + ProcessDefinitionVersion processDefinitionVersion4 = processDefinitionVersionMapper.queryByProcessDefinitionIdAndVersion( + processDefinitionVersion2.getProcessDefinitionId(), 11); + + Assert.assertEquals(processDefinitionVersion1.getProcessDefinitionId(), + processDefinitionVersion3.getProcessDefinitionId()); + Assert.assertEquals(processDefinitionVersion2.getProcessDefinitionId(), + processDefinitionVersion4.getProcessDefinitionId()); + + } + +} \ No newline at end of file diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ResourceMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ResourceMapperTest.java index 818f88fb49..76741a7db9 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ResourceMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ResourceMapperTest.java @@ -65,9 +65,10 @@ public class ResourceMapperTest { /** * insert + * * @return Resource */ - private Resource insertOne(){ + private Resource insertOne() { //insertOne Resource resource = new Resource(); resource.setAlias("ut-resource"); @@ -76,16 +77,20 @@ public class ResourceMapperTest { resource.setDirectory(false); resource.setType(ResourceType.FILE); resource.setUserId(111); - resourceMapper.insert(resource); + int status = resourceMapper.insert(resource); + if (status != 1) { + Assert.fail("insert data error"); + } return resource; } /** * create resource by user + * * @param user user * @return Resource */ - private Resource createResource(User user,boolean isDirectory,ResourceType resourceType,int pid,String alias,String fullName){ + private Resource createResource(User user, boolean isDirectory, ResourceType resourceType, int pid, String alias, String fullName) { //insertOne Resource resource = new Resource(); resource.setDirectory(isDirectory); @@ -93,19 +98,23 @@ public class ResourceMapperTest { resource.setAlias(alias); resource.setFullName(fullName); resource.setUserId(user.getId()); - resourceMapper.insert(resource); + int status = resourceMapper.insert(resource); + if (status != 1) { + Assert.fail("insert data error"); + } return resource; } /** * create resource by user + * * @param user user * @return Resource */ - private Resource createResource(User user){ + private Resource createResource(User user) { //insertOne - String alias = String.format("ut-resource-%s",user.getUserName()); - String fullName = String.format("/%s",alias); + String alias = String.format("ut-resource-%s", user.getUserName()); + String fullName = String.format("/%s", alias); Resource resource = createResource(user, false, ResourceType.FILE, -1, alias, fullName); return resource; @@ -113,9 +122,10 @@ public class ResourceMapperTest { /** * create user + * * @return User */ - private User createGeneralUser(String userName){ + private User createGeneralUser(String userName) { User user = new User(); user.setUserName(userName); user.setUserPassword("1"); @@ -124,15 +134,20 @@ public class ResourceMapperTest { user.setCreateTime(new Date()); user.setTenantId(1); user.setUpdateTime(new Date()); - userMapper.insert(user); + int status = userMapper.insert(user); + + if (status != 1) { + Assert.fail("insert data error"); + } return user; } /** * create resource user + * * @return ResourcesUser */ - private ResourcesUser createResourcesUser(Resource resource,User user){ + private ResourcesUser createResourcesUser(Resource resource, User user) { //insertOne ResourcesUser resourcesUser = new ResourcesUser(); resourcesUser.setCreateTime(new Date()); @@ -145,16 +160,17 @@ public class ResourceMapperTest { } @Test - public void testInsert(){ + public void testInsert() { Resource resource = insertOne(); assertNotNull(resource.getId()); - assertThat(resource.getId(),greaterThan(0)); + assertThat(resource.getId(), greaterThan(0)); } + /** * test update */ @Test - public void testUpdate(){ + public void testUpdate() { //insertOne Resource resource = insertOne(); resource.setCreateTime(new Date()); @@ -167,7 +183,7 @@ public class ResourceMapperTest { * test delete */ @Test - public void testDelete(){ + public void testDelete() { Resource resourceMap = insertOne(); int delete = resourceMapper.deleteById(resourceMap.getId()); Assert.assertEquals(1, delete); @@ -294,19 +310,31 @@ public class ResourceMapperTest { Tenant tenant = new Tenant(); tenant.setTenantName("ut tenant "); tenant.setTenantCode("ut tenant code for resource"); - tenantMapper.insert(tenant); + int tenantInsertStatus = tenantMapper.insert(tenant); + + if (tenantInsertStatus != 1) { + Assert.fail("insert tenant data error"); + } User user = new User(); user.setTenantId(tenant.getId()); user.setUserName("ut user"); - userMapper.insert(user); + int userInsertStatus = userMapper.insert(user); + + if (userInsertStatus != 1) { + Assert.fail("insert user data error"); + } + Resource resource = insertOne(); resource.setUserId(user.getId()); - resourceMapper.updateById(resource); + int userUpdateStatus = resourceMapper.updateById(resource); + if (userUpdateStatus != 1) { + Assert.fail("update user data error"); + } String resource1 = resourceMapper.queryTenantCodeByResourceName( - resource.getFullName(),ResourceType.FILE.ordinal() + resource.getFullName(), ResourceType.FILE.ordinal() ); @@ -315,7 +343,7 @@ public class ResourceMapperTest { } @Test - public void testListAuthorizedResource(){ + public void testListAuthorizedResource() { // create a general user User generalUser1 = createGeneralUser("user1"); User generalUser2 = createGeneralUser("user2"); @@ -328,20 +356,19 @@ public class ResourceMapperTest { List resources = resourceMapper.listAuthorizedResource(generalUser2.getId(), resNames); - Assert.assertEquals(generalUser2.getId(),resource.getUserId()); + Assert.assertEquals(generalUser2.getId(), resource.getUserId()); Assert.assertFalse(resources.stream().map(t -> t.getFullName()).collect(toList()).containsAll(Arrays.asList(resNames))); - // authorize object unauthorizedResource to generalUser - createResourcesUser(unauthorizedResource,generalUser2); + createResourcesUser(unauthorizedResource, generalUser2); List authorizedResources = resourceMapper.listAuthorizedResource(generalUser2.getId(), resNames); Assert.assertTrue(authorizedResources.stream().map(t -> t.getFullName()).collect(toList()).containsAll(Arrays.asList(resNames))); } @Test - public void deleteIdsTest(){ + public void deleteIdsTest() { // create a general user User generalUser1 = createGeneralUser("user1"); @@ -352,11 +379,11 @@ public class ResourceMapperTest { resourceList.add(resource.getId()); resourceList.add(resource1.getId()); int result = resourceMapper.deleteIds(resourceList.toArray(new Integer[resourceList.size()])); - Assert.assertEquals(result,2); + Assert.assertEquals(result, 2); } @Test - public void queryResourceListAuthoredTest(){ + public void queryResourceListAuthoredTest() { // create a general user User generalUser1 = createGeneralUser("user1"); User generalUser2 = createGeneralUser("user2"); @@ -372,16 +399,18 @@ public class ResourceMapperTest { } @Test - public void batchUpdateResourceTest(){ + public void batchUpdateResourceTest() { // create a general user User generalUser1 = createGeneralUser("user1"); // create resource Resource resource = createResource(generalUser1); - resource.setFullName(String.format("%s-update",resource.getFullName())); + resource.setFullName(String.format("%s-update", resource.getFullName())); resource.setUpdateTime(new Date()); List resourceList = new ArrayList<>(); resourceList.add(resource); int result = resourceMapper.batchUpdateResource(resourceList); - Assert.assertTrue(result>0); + if (result != resourceList.size()) { + Assert.fail("batch update resource data error"); + } } } \ No newline at end of file diff --git a/dolphinscheduler-dist/release-docs/LICENSE b/dolphinscheduler-dist/release-docs/LICENSE index 10279872c7..59da2746bf 100644 --- a/dolphinscheduler-dist/release-docs/LICENSE +++ b/dolphinscheduler-dist/release-docs/LICENSE @@ -384,6 +384,7 @@ The text of each license is also included at licenses/LICENSE-[project].txt. xercesImpl 2.9.1: https://mvnrepository.com/artifact/xerces/xercesImpl/2.9.1, Apache 2.0 xml-apis 1.4.01: https://mvnrepository.com/artifact/xml-apis/xml-apis/1.4.01, Apache 2.0 and W3C zookeeper 3.4.14: https://mvnrepository.com/artifact/org.apache.zookeeper/zookeeper/3.4.14, Apache 2.0 + presto-jdbc 0.238.1 https://mvnrepository.com/artifact/com.facebook.presto/presto-jdbc/0.238.1 ======================================================================== diff --git a/dolphinscheduler-dist/release-docs/NOTICE b/dolphinscheduler-dist/release-docs/NOTICE index 6ce789c7fb..901659e689 100644 --- a/dolphinscheduler-dist/release-docs/NOTICE +++ b/dolphinscheduler-dist/release-docs/NOTICE @@ -43,6 +43,13 @@ The following artifacts are EPL and CDDL 1.0. * org.eclipse.jetty.orbit:javax.mail.glassfish +------ +presto-jdbc + +The code for the t-digest was originally authored by Ted Dunning + +Adrien Grand contributed the heart of the AVLTreeDigest (https://github.com/jpountz) + ------ Oracle diff --git a/dolphinscheduler-dist/release-docs/licenses/LICENSE-presto-jdbc.txt b/dolphinscheduler-dist/release-docs/licenses/LICENSE-presto-jdbc.txt new file mode 100644 index 0000000000..f49a4e16e6 --- /dev/null +++ b/dolphinscheduler-dist/release-docs/licenses/LICENSE-presto-jdbc.txt @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/NettyRemotingClient.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/NettyRemotingClient.java index 10e62d8d9b..38f00fb4fd 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/NettyRemotingClient.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/NettyRemotingClient.java @@ -18,10 +18,16 @@ package org.apache.dolphinscheduler.remote; import io.netty.bootstrap.Bootstrap; -import io.netty.channel.*; +import io.netty.channel.Channel; +import io.netty.channel.ChannelFuture; +import io.netty.channel.ChannelFutureListener; +import io.netty.channel.ChannelInitializer; +import io.netty.channel.ChannelOption; +import io.netty.channel.EventLoopGroup; +import io.netty.channel.epoll.EpollEventLoopGroup; import io.netty.channel.nio.NioEventLoopGroup; import io.netty.channel.socket.SocketChannel; -import io.netty.channel.socket.nio.NioSocketChannel; + import org.apache.dolphinscheduler.remote.codec.NettyDecoder; import org.apache.dolphinscheduler.remote.codec.NettyEncoder; import org.apache.dolphinscheduler.remote.command.Command; @@ -38,6 +44,8 @@ import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor; import org.apache.dolphinscheduler.remote.utils.Host; import org.apache.dolphinscheduler.remote.utils.CallerThreadExecutePolicy; import org.apache.dolphinscheduler.remote.utils.NamedThreadFactory; +import org.apache.dolphinscheduler.remote.utils.NettyUtils; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -47,7 +55,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; /** - * remoting netty client + * remoting netty client */ public class NettyRemotingClient { @@ -59,7 +67,7 @@ public class NettyRemotingClient { private final Bootstrap bootstrap = new Bootstrap(); /** - * encoder + * encoder */ private final NettyEncoder encoder = new NettyEncoder(); @@ -69,57 +77,69 @@ public class NettyRemotingClient { private final ConcurrentHashMap channels = new ConcurrentHashMap(128); /** - * started flag + * started flag */ private final AtomicBoolean isStarted = new AtomicBoolean(false); /** - * worker group + * worker group */ - private final NioEventLoopGroup workerGroup; + private final EventLoopGroup workerGroup; /** - * client config + * client config */ private final NettyClientConfig clientConfig; /** - * saync semaphore + * saync semaphore */ private final Semaphore asyncSemaphore = new Semaphore(200, true); /** - * callback thread executor + * callback thread executor */ private final ExecutorService callbackExecutor; /** - * client handler + * client handler */ private final NettyClientHandler clientHandler; /** - * response future executor + * response future executor */ private final ScheduledExecutorService responseFutureExecutor; /** - * client init + * client init + * * @param clientConfig client config */ - public NettyRemotingClient(final NettyClientConfig clientConfig){ + public NettyRemotingClient(final NettyClientConfig clientConfig) { this.clientConfig = clientConfig; - this.workerGroup = new NioEventLoopGroup(clientConfig.getWorkerThreads(), new ThreadFactory() { - private AtomicInteger threadIndex = new AtomicInteger(0); + if (NettyUtils.useEpoll()) { + this.workerGroup = new EpollEventLoopGroup(clientConfig.getWorkerThreads(), new ThreadFactory() { + private AtomicInteger threadIndex = new AtomicInteger(0); - @Override - public Thread newThread(Runnable r) { - return new Thread(r, String.format("NettyClient_%d", this.threadIndex.incrementAndGet())); - } - }); + @Override + public Thread newThread(Runnable r) { + return new Thread(r, String.format("NettyClient_%d", this.threadIndex.incrementAndGet())); + } + }); + } else { + this.workerGroup = new NioEventLoopGroup(clientConfig.getWorkerThreads(), new ThreadFactory() { + private AtomicInteger threadIndex = new AtomicInteger(0); + + @Override + public Thread newThread(Runnable r) { + return new Thread(r, String.format("NettyClient_%d", this.threadIndex.incrementAndGet())); + } + }); + } this.callbackExecutor = new ThreadPoolExecutor(5, 10, 1, TimeUnit.MINUTES, - new LinkedBlockingQueue<>(1000), new NamedThreadFactory("CallbackExecutor", 10), - new CallerThreadExecutePolicy()); + new LinkedBlockingQueue<>(1000), new NamedThreadFactory("CallbackExecutor", 10), + new CallerThreadExecutePolicy()); this.clientHandler = new NettyClientHandler(this, callbackExecutor); this.responseFutureExecutor = Executors.newSingleThreadScheduledExecutor(new NamedThreadFactory("ResponseFutureExecutor")); @@ -128,26 +148,27 @@ public class NettyRemotingClient { } /** - * start + * start */ - private void start(){ + private void start() { this.bootstrap - .group(this.workerGroup) - .channel(NioSocketChannel.class) - .option(ChannelOption.SO_KEEPALIVE, clientConfig.isSoKeepalive()) - .option(ChannelOption.TCP_NODELAY, clientConfig.isTcpNoDelay()) - .option(ChannelOption.SO_SNDBUF, clientConfig.getSendBufferSize()) - .option(ChannelOption.SO_RCVBUF, clientConfig.getReceiveBufferSize()) - .handler(new ChannelInitializer() { - @Override - public void initChannel(SocketChannel ch) throws Exception { - ch.pipeline().addLast( - new NettyDecoder(), - clientHandler, - encoder); - } - }); + .group(this.workerGroup) + .channel(NettyUtils.getSocketChannelClass()) + .option(ChannelOption.SO_KEEPALIVE, clientConfig.isSoKeepalive()) + .option(ChannelOption.TCP_NODELAY, clientConfig.isTcpNoDelay()) + .option(ChannelOption.SO_SNDBUF, clientConfig.getSendBufferSize()) + .option(ChannelOption.SO_RCVBUF, clientConfig.getReceiveBufferSize()) + .option(ChannelOption.CONNECT_TIMEOUT_MILLIS, clientConfig.getConnectTimeoutMillis()) + .handler(new ChannelInitializer() { + @Override + public void initChannel(SocketChannel ch) throws Exception { + ch.pipeline().addLast( + new NettyDecoder(), + clientHandler, + encoder); + } + }); this.responseFutureExecutor.scheduleAtFixedRate(new Runnable() { @Override public void run() { @@ -159,10 +180,11 @@ public class NettyRemotingClient { } /** - * async send - * @param host host - * @param command command - * @param timeoutMillis timeoutMillis + * async send + * + * @param host host + * @param command command + * @param timeoutMillis timeoutMillis * @param invokeCallback callback function * @throws InterruptedException * @throws RemotingException @@ -182,22 +204,22 @@ public class NettyRemotingClient { * control concurrency number */ boolean acquired = this.asyncSemaphore.tryAcquire(timeoutMillis, TimeUnit.MILLISECONDS); - if(acquired){ + if (acquired) { final ReleaseSemaphore releaseSemaphore = new ReleaseSemaphore(this.asyncSemaphore); /** * response future */ final ResponseFuture responseFuture = new ResponseFuture(opaque, - timeoutMillis, - invokeCallback, - releaseSemaphore); + timeoutMillis, + invokeCallback, + releaseSemaphore); try { - channel.writeAndFlush(command).addListener(new ChannelFutureListener(){ + channel.writeAndFlush(command).addListener(new ChannelFutureListener() { @Override public void operationComplete(ChannelFuture future) throws Exception { - if(future.isSuccess()){ + if (future.isSuccess()) { responseFuture.setSendOk(true); return; } else { @@ -207,28 +229,29 @@ public class NettyRemotingClient { responseFuture.putResponse(null); try { responseFuture.executeInvokeCallback(); - } catch (Throwable ex){ + } catch (Throwable ex) { logger.error("execute callback error", ex); - } finally{ + } finally { responseFuture.release(); } } }); - } catch (Throwable ex){ + } catch (Throwable ex) { responseFuture.release(); throw new RemotingException(String.format("send command to host: %s failed", host), ex); } - } else{ + } else { String message = String.format("try to acquire async semaphore timeout: %d, waiting thread num: %d, total permits: %d", - timeoutMillis, asyncSemaphore.getQueueLength(), asyncSemaphore.availablePermits()); + timeoutMillis, asyncSemaphore.getQueueLength(), asyncSemaphore.availablePermits()); throw new RemotingTooMuchRequestException(message); } } /** * sync send - * @param host host - * @param command command + * + * @param host host + * @param command command * @param timeoutMillis timeoutMillis * @return command * @throws InterruptedException @@ -244,7 +267,7 @@ public class NettyRemotingClient { channel.writeAndFlush(command).addListener(new ChannelFutureListener() { @Override public void operationComplete(ChannelFuture future) throws Exception { - if(future.isSuccess()){ + if (future.isSuccess()) { responseFuture.setSendOk(true); return; } else { @@ -259,10 +282,10 @@ public class NettyRemotingClient { * sync wait for result */ Command result = responseFuture.waitResponse(); - if(result == null){ - if(responseFuture.isSendOK()){ + if (result == null) { + if (responseFuture.isSendOK()) { throw new RemotingTimeoutException(host.toString(), timeoutMillis, responseFuture.getCause()); - } else{ + } else { throw new RemotingException(host.toString(), responseFuture.getCause()); } } @@ -270,8 +293,9 @@ public class NettyRemotingClient { } /** - * send task - * @param host host + * send task + * + * @param host host * @param command command * @throws RemotingException */ @@ -296,33 +320,35 @@ public class NettyRemotingClient { } /** - * register processor + * register processor + * * @param commandType command type - * @param processor processor + * @param processor processor */ public void registerProcessor(final CommandType commandType, final NettyRequestProcessor processor) { this.registerProcessor(commandType, processor, null); } /** - * register processor + * register processor * * @param commandType command type - * @param processor processor - * @param executor thread executor + * @param processor processor + * @param executor thread executor */ public void registerProcessor(final CommandType commandType, final NettyRequestProcessor processor, final ExecutorService executor) { this.clientHandler.registerProcessor(commandType, processor, executor); } /** - * get channel + * get channel + * * @param host * @return */ public Channel getChannel(Host host) { Channel channel = channels.get(host); - if(channel != null && channel.isActive()){ + if (channel != null && channel.isActive()) { return channel; } return createChannel(host, true); @@ -330,17 +356,18 @@ public class NettyRemotingClient { /** * create channel - * @param host host + * + * @param host host * @param isSync sync flag * @return channel */ public Channel createChannel(Host host, boolean isSync) { ChannelFuture future; try { - synchronized (bootstrap){ + synchronized (bootstrap) { future = bootstrap.connect(new InetSocketAddress(host.getIp(), host.getPort())); } - if(isSync){ + if (isSync) { future.sync(); } if (future.isSuccess()) { @@ -358,16 +385,16 @@ public class NettyRemotingClient { * close */ public void close() { - if(isStarted.compareAndSet(true, false)){ + if (isStarted.compareAndSet(true, false)) { try { closeChannels(); - if(workerGroup != null){ + if (workerGroup != null) { this.workerGroup.shutdownGracefully(); } - if(callbackExecutor != null){ + if (callbackExecutor != null) { this.callbackExecutor.shutdownNow(); } - if(this.responseFutureExecutor != null){ + if (this.responseFutureExecutor != null) { this.responseFutureExecutor.shutdownNow(); } } catch (Exception ex) { @@ -378,9 +405,9 @@ public class NettyRemotingClient { } /** - * close channels + * close channels */ - private void closeChannels(){ + private void closeChannels() { for (Channel channel : this.channels.values()) { channel.close(); } @@ -389,11 +416,12 @@ public class NettyRemotingClient { /** * close channel + * * @param host host */ - public void closeChannel(Host host){ + public void closeChannel(Host host) { Channel channel = this.channels.remove(host); - if(channel != null){ + if (channel != null) { channel.close(); } } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/NettyRemotingServer.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/NettyRemotingServer.java index 3eed82b1e1..ad5c95bb38 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/NettyRemotingServer.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/NettyRemotingServer.java @@ -17,14 +17,6 @@ package org.apache.dolphinscheduler.remote; -import io.netty.bootstrap.ServerBootstrap; -import io.netty.channel.ChannelFuture; -import io.netty.channel.ChannelInitializer; -import io.netty.channel.ChannelOption; -import io.netty.channel.ChannelPipeline; -import io.netty.channel.nio.NioEventLoopGroup; -import io.netty.channel.socket.nio.NioServerSocketChannel; -import io.netty.channel.socket.nio.NioSocketChannel; import org.apache.dolphinscheduler.remote.codec.NettyDecoder; import org.apache.dolphinscheduler.remote.codec.NettyEncoder; import org.apache.dolphinscheduler.remote.command.CommandType; @@ -32,8 +24,7 @@ import org.apache.dolphinscheduler.remote.config.NettyServerConfig; import org.apache.dolphinscheduler.remote.handler.NettyServerHandler; import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor; import org.apache.dolphinscheduler.remote.utils.Constants; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import org.apache.dolphinscheduler.remote.utils.NettyUtils; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -41,45 +32,58 @@ import java.util.concurrent.ThreadFactory; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import io.netty.bootstrap.ServerBootstrap; +import io.netty.channel.ChannelFuture; +import io.netty.channel.ChannelInitializer; +import io.netty.channel.ChannelOption; +import io.netty.channel.ChannelPipeline; +import io.netty.channel.EventLoopGroup; +import io.netty.channel.epoll.EpollEventLoopGroup; +import io.netty.channel.nio.NioEventLoopGroup; +import io.netty.channel.socket.SocketChannel; + /** - * remoting netty server + * remoting netty server */ public class NettyRemotingServer { private final Logger logger = LoggerFactory.getLogger(NettyRemotingServer.class); /** - * server bootstrap + * server bootstrap */ private final ServerBootstrap serverBootstrap = new ServerBootstrap(); /** - * encoder + * encoder */ private final NettyEncoder encoder = new NettyEncoder(); /** - * default executor + * default executor */ private final ExecutorService defaultExecutor = Executors.newFixedThreadPool(Constants.CPUS); /** * boss group */ - private final NioEventLoopGroup bossGroup; + private final EventLoopGroup bossGroup; /** - * worker group + * worker group */ - private final NioEventLoopGroup workGroup; + private final EventLoopGroup workGroup; /** - * server config + * server config */ private final NettyServerConfig serverConfig; /** - * server handler + * server handler */ private final NettyServerHandler serverHandler = new NettyServerHandler(this); @@ -89,59 +93,78 @@ public class NettyRemotingServer { private final AtomicBoolean isStarted = new AtomicBoolean(false); /** - * server init + * server init * * @param serverConfig server config */ - public NettyRemotingServer(final NettyServerConfig serverConfig){ + public NettyRemotingServer(final NettyServerConfig serverConfig) { this.serverConfig = serverConfig; + if (NettyUtils.useEpoll()) { + this.bossGroup = new EpollEventLoopGroup(1, new ThreadFactory() { + private AtomicInteger threadIndex = new AtomicInteger(0); - this.bossGroup = new NioEventLoopGroup(1, new ThreadFactory() { - private AtomicInteger threadIndex = new AtomicInteger(0); + @Override + public Thread newThread(Runnable r) { + return new Thread(r, String.format("NettyServerBossThread_%d", this.threadIndex.incrementAndGet())); + } + }); - @Override - public Thread newThread(Runnable r) { - return new Thread(r, String.format("NettyServerBossThread_%d", this.threadIndex.incrementAndGet())); - } - }); + this.workGroup = new EpollEventLoopGroup(serverConfig.getWorkerThread(), new ThreadFactory() { + private AtomicInteger threadIndex = new AtomicInteger(0); - this.workGroup = new NioEventLoopGroup(serverConfig.getWorkerThread(), new ThreadFactory() { - private AtomicInteger threadIndex = new AtomicInteger(0); + @Override + public Thread newThread(Runnable r) { + return new Thread(r, String.format("NettyServerWorkerThread_%d", this.threadIndex.incrementAndGet())); + } + }); + } else { + this.bossGroup = new NioEventLoopGroup(1, new ThreadFactory() { + private AtomicInteger threadIndex = new AtomicInteger(0); - @Override - public Thread newThread(Runnable r) { - return new Thread(r, String.format("NettyServerWorkerThread_%d", this.threadIndex.incrementAndGet())); - } - }); + @Override + public Thread newThread(Runnable r) { + return new Thread(r, String.format("NettyServerBossThread_%d", this.threadIndex.incrementAndGet())); + } + }); + + this.workGroup = new NioEventLoopGroup(serverConfig.getWorkerThread(), new ThreadFactory() { + private AtomicInteger threadIndex = new AtomicInteger(0); + + @Override + public Thread newThread(Runnable r) { + return new Thread(r, String.format("NettyServerWorkerThread_%d", this.threadIndex.incrementAndGet())); + } + }); + } } /** - * server start + * server start */ - public void start(){ + public void start() { if (isStarted.compareAndSet(false, true)) { this.serverBootstrap - .group(this.bossGroup, this.workGroup) - .channel(NioServerSocketChannel.class) - .option(ChannelOption.SO_REUSEADDR, true) - .option(ChannelOption.SO_BACKLOG, serverConfig.getSoBacklog()) - .childOption(ChannelOption.SO_KEEPALIVE, serverConfig.isSoKeepalive()) - .childOption(ChannelOption.TCP_NODELAY, serverConfig.isTcpNoDelay()) - .childOption(ChannelOption.SO_SNDBUF, serverConfig.getSendBufferSize()) - .childOption(ChannelOption.SO_RCVBUF, serverConfig.getReceiveBufferSize()) - .childHandler(new ChannelInitializer() { + .group(this.bossGroup, this.workGroup) + .channel(NettyUtils.getServerSocketChannelClass()) + .option(ChannelOption.SO_REUSEADDR, true) + .option(ChannelOption.SO_BACKLOG, serverConfig.getSoBacklog()) + .childOption(ChannelOption.SO_KEEPALIVE, serverConfig.isSoKeepalive()) + .childOption(ChannelOption.TCP_NODELAY, serverConfig.isTcpNoDelay()) + .childOption(ChannelOption.SO_SNDBUF, serverConfig.getSendBufferSize()) + .childOption(ChannelOption.SO_RCVBUF, serverConfig.getReceiveBufferSize()) + .childHandler(new ChannelInitializer() { - @Override - protected void initChannel(NioSocketChannel ch) throws Exception { - initNettyChannel(ch); - } - }); + @Override + protected void initChannel(SocketChannel ch) throws Exception { + initNettyChannel(ch); + } + }); ChannelFuture future; try { future = serverBootstrap.bind(serverConfig.getListenPort()).sync(); } catch (Exception e) { - logger.error("NettyRemotingServer bind fail {}, exit",e.getMessage(), e); + logger.error("NettyRemotingServer bind fail {}, exit", e.getMessage(), e); throw new RuntimeException(String.format("NettyRemotingServer bind %s fail", serverConfig.getListenPort())); } if (future.isSuccess()) { @@ -155,11 +178,11 @@ public class NettyRemotingServer { } /** - * init netty channel + * init netty channel + * * @param ch socket channel - * @throws Exception */ - private void initNettyChannel(NioSocketChannel ch) throws Exception{ + private void initNettyChannel(SocketChannel ch) { ChannelPipeline pipeline = ch.pipeline(); pipeline.addLast("encoder", encoder); pipeline.addLast("decoder", new NettyDecoder()); @@ -167,27 +190,29 @@ public class NettyRemotingServer { } /** - * register processor + * register processor + * * @param commandType command type - * @param processor processor + * @param processor processor */ public void registerProcessor(final CommandType commandType, final NettyRequestProcessor processor) { this.registerProcessor(commandType, processor, null); } /** - * register processor + * register processor * * @param commandType command type - * @param processor processor - * @param executor thread executor + * @param processor processor + * @param executor thread executor */ public void registerProcessor(final CommandType commandType, final NettyRequestProcessor processor, final ExecutorService executor) { this.serverHandler.registerProcessor(commandType, processor, executor); } /** - * get default thread executor + * get default thread executor + * * @return thread executor */ public ExecutorService getDefaultExecutor() { @@ -195,12 +220,12 @@ public class NettyRemotingServer { } public void close() { - if(isStarted.compareAndSet(true, false)){ + if (isStarted.compareAndSet(true, false)) { try { - if(bossGroup != null){ + if (bossGroup != null) { this.bossGroup.shutdownGracefully(); } - if(workGroup != null){ + if (workGroup != null) { this.workGroup.shutdownGracefully(); } defaultExecutor.shutdown(); diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/codec/NettyDecoder.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/codec/NettyDecoder.java index a69022214d..179ae1bef8 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/codec/NettyDecoder.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/codec/NettyDecoder.java @@ -75,6 +75,7 @@ public class NettyDecoder extends ReplayingDecoder { out.add(packet); // checkpoint(State.MAGIC); + break; default: logger.warn("unknown decoder state {}", state()); } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/config/NettyClientConfig.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/config/NettyClientConfig.java index 831e05f7e7..739cbbebe1 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/config/NettyClientConfig.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/config/NettyClientConfig.java @@ -14,22 +14,23 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.remote.config; import org.apache.dolphinscheduler.remote.utils.Constants; /** - * netty client config + * netty client config */ public class NettyClientConfig { /** - * worker threads，default get machine cpus + * worker threads，default get machine cpus */ private int workerThreads = Constants.CPUS; /** - * whether tpc delay + * whether tpc delay */ private boolean tcpNoDelay = true; @@ -39,15 +40,20 @@ public class NettyClientConfig { private boolean soKeepalive = true; /** - * send buffer size + * send buffer size */ private int sendBufferSize = 65535; /** - * receive buffer size + * receive buffer size */ private int receiveBufferSize = 65535; + /** + * connect timeout millis + */ + private int connectTimeoutMillis = 3000; + public int getWorkerThreads() { return workerThreads; } @@ -88,4 +94,11 @@ public class NettyClientConfig { this.receiveBufferSize = receiveBufferSize; } + public int getConnectTimeoutMillis() { + return connectTimeoutMillis; + } + + public void setConnectTimeoutMillis(int connectTimeoutMillis) { + this.connectTimeoutMillis = connectTimeoutMillis; + } } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Constants.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Constants.java index 48736ca694..370467f6ca 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Constants.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Constants.java @@ -14,6 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.remote.utils; import java.nio.charset.Charset; @@ -21,7 +22,7 @@ import java.nio.charset.StandardCharsets; /** - * constant + * constant */ public class Constants { @@ -30,16 +31,26 @@ public class Constants { public static final String SLASH = "/"; /** - * charset + * charset */ public static final Charset UTF8 = StandardCharsets.UTF_8; /** - * cpus + * cpus */ public static final int CPUS = Runtime.getRuntime().availableProcessors(); public static final String LOCAL_ADDRESS = IPUtils.getFirstNoLoopbackIP4Address(); + /** + * netty epoll enable switch + */ + public static final String NETTY_EPOLL_ENABLE = System.getProperty("netty.epoll.enable", "true"); + + /** + * OS Name + */ + public static final String OS_NAME = System.getProperty("os.name"); + } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Host.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Host.java index e9eaabcad6..b905a9fea8 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Host.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Host.java @@ -20,7 +20,7 @@ import java.io.Serializable; import java.util.Objects; /** - * server address + * server address */ public class Host implements Serializable { @@ -39,6 +39,16 @@ public class Host implements Serializable { */ private int port; + /** + * weight + */ + private int weight; + + /** + * workGroup + */ + private String workGroup; + public Host() { } @@ -48,6 +58,21 @@ public class Host implements Serializable { this.address = ip + ":" + port; } + public Host(String ip, int port, int weight) { + this.ip = ip; + this.port = port; + this.address = ip + ":" + port; + this.weight = weight; + } + + public Host(String ip, int port, int weight,String workGroup) { + this.ip = ip; + this.port = port; + this.address = ip + ":" + port; + this.weight = weight; + this.workGroup=workGroup; + } + public String getAddress() { return address; } @@ -65,6 +90,14 @@ public class Host implements Serializable { this.address = ip + ":" + port; } + public int getWeight() { + return weight; + } + + public void setWeight(int weight) { + this.weight = weight; + } + public int getPort() { return port; } @@ -74,31 +107,47 @@ public class Host implements Serializable { this.address = ip + ":" + port; } + public String getWorkGroup() { + return workGroup; + } + + public void setWorkGroup(String workGroup) { + this.workGroup = workGroup; + } + /** * address convert host + * * @param address address * @return host */ - public static Host of(String address){ - if(address == null) { + public static Host of(String address) { + if (address == null) { throw new IllegalArgumentException("Host : address is null."); } String[] parts = address.split(":"); - if (parts.length != 2) { + if (parts.length < 2) { throw new IllegalArgumentException(String.format("Host : %s illegal.", address)); } - Host host = new Host(parts[0], Integer.parseInt(parts[1])); + Host host = null; + if (parts.length == 2) { + host = new Host(parts[0], Integer.parseInt(parts[1])); + } + if (parts.length == 3) { + host = new Host(parts[0], Integer.parseInt(parts[1]), Integer.parseInt(parts[2])); + } return host; } /** * whether old version + * * @param address address * @return old version is true , otherwise is false */ - public static Boolean isOldVersion(String address){ + public static Boolean isOldVersion(String address) { String[] parts = address.split(":"); - return parts.length != 2 ? true : false; + return parts.length != 2 && parts.length != 3; } @Override diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/NettyUtils.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/NettyUtils.java new file mode 100644 index 0000000000..89eb1f9607 --- /dev/null +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/NettyUtils.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.remote.utils; + +import io.netty.channel.epoll.Epoll; +import io.netty.channel.epoll.EpollServerSocketChannel; +import io.netty.channel.epoll.EpollSocketChannel; +import io.netty.channel.socket.ServerSocketChannel; +import io.netty.channel.socket.SocketChannel; +import io.netty.channel.socket.nio.NioServerSocketChannel; +import io.netty.channel.socket.nio.NioSocketChannel; + +/** + * NettyUtils + */ +public class NettyUtils { + + private NettyUtils() { + } + + public static boolean useEpoll() { + String osName = Constants.OS_NAME; + if (!osName.toLowerCase().contains("linux")) { + return false; + } + if (!Epoll.isAvailable()) { + return false; + } + String enableNettyEpoll = Constants.NETTY_EPOLL_ENABLE; + return Boolean.parseBoolean(enableNettyEpoll); + } + + public static Class getServerSocketChannelClass() { + if (useEpoll()) { + return EpollServerSocketChannel.class; + } + return NioServerSocketChannel.class; + } + + public static Class getSocketChannelClass() { + if (useEpoll()) { + return EpollSocketChannel.class; + } + return NioSocketChannel.class; + } + +} diff --git a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/NettyUtilTest.java b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/NettyUtilTest.java new file mode 100644 index 0000000000..e95dbddac9 --- /dev/null +++ b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/NettyUtilTest.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.remote; + +import static org.apache.dolphinscheduler.remote.utils.Constants.OS_NAME; + +import org.apache.dolphinscheduler.remote.utils.NettyUtils; + +import org.junit.Assert; +import org.junit.Test; + +import io.netty.channel.epoll.Epoll; + +/** + * NettyUtilTest + */ +public class NettyUtilTest { + + + @Test + public void testUserEpoll() { + if (OS_NAME.toLowerCase().contains("linux") && Epoll.isAvailable()) { + Assert.assertTrue(NettyUtils.useEpoll()); + } else { + Assert.assertFalse(NettyUtils.useEpoll()); + } + } + +} diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/builder/TaskExecutionContextBuilder.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/builder/TaskExecutionContextBuilder.java index 535c274989..74b0635145 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/builder/TaskExecutionContextBuilder.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/builder/TaskExecutionContextBuilder.java @@ -40,6 +40,7 @@ public class TaskExecutionContextBuilder { public TaskExecutionContextBuilder buildTaskInstanceRelatedInfo(TaskInstance taskInstance){ taskExecutionContext.setTaskInstanceId(taskInstance.getId()); taskExecutionContext.setTaskName(taskInstance.getName()); + taskExecutionContext.setFirstSubmitTime(taskInstance.getFirstSubmitTime()); taskExecutionContext.setStartTime(taskInstance.getStartTime()); taskExecutionContext.setTaskType(taskInstance.getTaskType()); taskExecutionContext.setLogPath(taskInstance.getLogPath()); @@ -48,6 +49,7 @@ public class TaskExecutionContextBuilder { taskExecutionContext.setWorkerGroup(taskInstance.getWorkerGroup()); taskExecutionContext.setHost(taskInstance.getHost()); taskExecutionContext.setResources(taskInstance.getResources()); + taskExecutionContext.setDelayTime(taskInstance.getDelayTime()); return this; } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/entity/TaskExecutionContext.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/entity/TaskExecutionContext.java index 81488fb134..1589c365c2 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/entity/TaskExecutionContext.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/entity/TaskExecutionContext.java @@ -17,7 +17,7 @@ package org.apache.dolphinscheduler.server.entity; -import com.fasterxml.jackson.annotation.JsonFormat; +import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.TaskExecuteRequestCommand; import org.apache.dolphinscheduler.remote.utils.JsonSerializer; @@ -26,30 +26,38 @@ import java.io.Serializable; import java.util.Date; import java.util.Map; +import com.fasterxml.jackson.annotation.JsonFormat; + /** - * master/worker task transport + * master/worker task transport */ -public class TaskExecutionContext implements Serializable{ +public class TaskExecutionContext implements Serializable { /** - * task id + * task id */ private int taskInstanceId; /** - * task name + * task name */ private String taskName; /** - * task start time + * task first submit time. */ - @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss", timezone = "GMT+8") + private Date firstSubmitTime; + + /** + * task start time + */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss", timezone = "GMT+8") private Date startTime; /** - * task type + * task type */ private String taskType; @@ -57,9 +65,9 @@ public class TaskExecutionContext implements Serializable{ * host */ private String host; - + /** - * task execute path + * task execute path */ private String executePath; @@ -69,7 +77,7 @@ public class TaskExecutionContext implements Serializable{ private String logPath; /** - * task json + * task json */ private String taskJson; @@ -84,53 +92,53 @@ public class TaskExecutionContext implements Serializable{ private String appIds; /** - * process instance id + * process instance id */ private int processInstanceId; /** - * process instance schedule time + * process instance schedule time */ - @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss", timezone = "GMT+8") private Date scheduleTime; /** - * process instance global parameters + * process instance global parameters */ private String globalParams; /** - * execute user id + * execute user id */ private int executorId; /** - * command type if complement + * command type if complement */ private int cmdTypeIfComplement; /** - * tenant code + * tenant code */ private String tenantCode; /** - * task queue + * task queue */ private String queue; /** - * process define id + * process define id */ private int processDefineId; /** - * project id + * project id */ private int projectId; @@ -140,12 +148,12 @@ public class TaskExecutionContext implements Serializable{ private String taskParams; /** - * envFile + * envFile */ private String envFile; /** - * definedParams + * definedParams */ private Map definedParams; @@ -155,7 +163,7 @@ public class TaskExecutionContext implements Serializable{ private String taskAppId; /** - * task timeout strategy + * task timeout strategy */ private int taskTimeoutStrategy; @@ -170,17 +178,27 @@ public class TaskExecutionContext implements Serializable{ private String workerGroup; /** - * resources full name and tenant code + * delay execution time. */ - private Map resources; + private int delayTime; /** - * sql TaskExecutionContext + * current execution status + */ + private ExecutionStatus currentExecutionStatus; + + /** + * resources full name and tenant code + */ + private Map resources; + + /** + * sql TaskExecutionContext */ private SQLTaskExecutionContext sqlTaskExecutionContext; /** - * datax TaskExecutionContext + * datax TaskExecutionContext */ private DataxTaskExecutionContext dataxTaskExecutionContext; @@ -195,7 +213,7 @@ public class TaskExecutionContext implements Serializable{ private SqoopTaskExecutionContext sqoopTaskExecutionContext; /** - * procedure TaskExecutionContext + * procedure TaskExecutionContext */ private ProcedureTaskExecutionContext procedureTaskExecutionContext; @@ -215,6 +233,14 @@ public class TaskExecutionContext implements Serializable{ this.taskName = taskName; } + public Date getFirstSubmitTime() { + return firstSubmitTime; + } + + public void setFirstSubmitTime(Date firstSubmitTime) { + this.firstSubmitTime = firstSubmitTime; + } + public Date getStartTime() { return startTime; } @@ -407,6 +433,22 @@ public class TaskExecutionContext implements Serializable{ this.workerGroup = workerGroup; } + public int getDelayTime() { + return delayTime; + } + + public void setDelayTime(int delayTime) { + this.delayTime = delayTime; + } + + public ExecutionStatus getCurrentExecutionStatus() { + return currentExecutionStatus; + } + + public void setCurrentExecutionStatus(ExecutionStatus currentExecutionStatus) { + this.currentExecutionStatus = currentExecutionStatus; + } + public SQLTaskExecutionContext getSqlTaskExecutionContext() { return sqlTaskExecutionContext; } @@ -431,7 +473,7 @@ public class TaskExecutionContext implements Serializable{ this.procedureTaskExecutionContext = procedureTaskExecutionContext; } - public Command toCommand(){ + public Command toCommand() { TaskExecuteRequestCommand requestCommand = new TaskExecuteRequestCommand(); requestCommand.setTaskExecutionContext(JsonSerializer.serializeToString(this)); return requestCommand.convert2Command(); @@ -463,39 +505,42 @@ public class TaskExecutionContext implements Serializable{ @Override public String toString() { - return "TaskExecutionContext{" + - "taskInstanceId=" + taskInstanceId + - ", taskName='" + taskName + '\'' + - ", startTime=" + startTime + - ", taskType='" + taskType + '\'' + - ", host='" + host + '\'' + - ", executePath='" + executePath + '\'' + - ", logPath='" + logPath + '\'' + - ", taskJson='" + taskJson + '\'' + - ", processId=" + processId + - ", appIds='" + appIds + '\'' + - ", processInstanceId=" + processInstanceId + - ", scheduleTime=" + scheduleTime + - ", globalParams='" + globalParams + '\'' + - ", executorId=" + executorId + - ", cmdTypeIfComplement=" + cmdTypeIfComplement + - ", tenantCode='" + tenantCode + '\'' + - ", queue='" + queue + '\'' + - ", processDefineId=" + processDefineId + - ", projectId=" + projectId + - ", taskParams='" + taskParams + '\'' + - ", envFile='" + envFile + '\'' + - ", definedParams=" + definedParams + - ", taskAppId='" + taskAppId + '\'' + - ", taskTimeoutStrategy=" + taskTimeoutStrategy + - ", taskTimeout=" + taskTimeout + - ", workerGroup='" + workerGroup + '\'' + - ", resources=" + resources + - ", sqlTaskExecutionContext=" + sqlTaskExecutionContext + - ", dataxTaskExecutionContext=" + dataxTaskExecutionContext + - ", dependenceTaskExecutionContext=" + dependenceTaskExecutionContext + - ", sqoopTaskExecutionContext=" + sqoopTaskExecutionContext + - ", procedureTaskExecutionContext=" + procedureTaskExecutionContext + - '}'; + return "TaskExecutionContext{" + + "taskInstanceId=" + taskInstanceId + + ", taskName='" + taskName + '\'' + + ", currentExecutionStatus=" + currentExecutionStatus + + ", firstSubmitTime=" + firstSubmitTime + + ", startTime=" + startTime + + ", taskType='" + taskType + '\'' + + ", host='" + host + '\'' + + ", executePath='" + executePath + '\'' + + ", logPath='" + logPath + '\'' + + ", taskJson='" + taskJson + '\'' + + ", processId=" + processId + + ", appIds='" + appIds + '\'' + + ", processInstanceId=" + processInstanceId + + ", scheduleTime=" + scheduleTime + + ", globalParams='" + globalParams + '\'' + + ", executorId=" + executorId + + ", cmdTypeIfComplement=" + cmdTypeIfComplement + + ", tenantCode='" + tenantCode + '\'' + + ", queue='" + queue + '\'' + + ", processDefineId=" + processDefineId + + ", projectId=" + projectId + + ", taskParams='" + taskParams + '\'' + + ", envFile='" + envFile + '\'' + + ", definedParams=" + definedParams + + ", taskAppId='" + taskAppId + '\'' + + ", taskTimeoutStrategy=" + taskTimeoutStrategy + + ", taskTimeout=" + taskTimeout + + ", workerGroup='" + workerGroup + '\'' + + ", delayTime=" + delayTime + + ", resources=" + resources + + ", sqlTaskExecutionContext=" + sqlTaskExecutionContext + + ", dataxTaskExecutionContext=" + dataxTaskExecutionContext + + ", dependenceTaskExecutionContext=" + dependenceTaskExecutionContext + + ", sqoopTaskExecutionContext=" + sqoopTaskExecutionContext + + ", procedureTaskExecutionContext=" + procedureTaskExecutionContext + + '}'; } } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/cache/impl/TaskInstanceCacheManagerImpl.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/cache/impl/TaskInstanceCacheManagerImpl.java index c149ac3335..4d55490a8d 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/cache/impl/TaskInstanceCacheManagerImpl.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/cache/impl/TaskInstanceCacheManagerImpl.java @@ -36,7 +36,7 @@ import java.util.concurrent.ConcurrentHashMap; public class TaskInstanceCacheManagerImpl implements TaskInstanceCacheManager { /** - * taskInstance caceh + * taskInstance cache */ private Map taskInstanceCache = new ConcurrentHashMap<>(); diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/CommonHostManager.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/CommonHostManager.java index 58006bf7f7..4a3d4bd9f1 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/CommonHostManager.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/CommonHostManager.java @@ -71,7 +71,12 @@ public abstract class CommonHostManager implements HostManager { return host; } List candidateHosts = new ArrayList<>(nodes.size()); - nodes.stream().forEach(node -> candidateHosts.add(Host.of(node))); + nodes.forEach(node -> { + Host nodeHost=Host.of(node); + nodeHost.setWorkGroup(context.getWorkerGroup()); + candidateHosts.add(nodeHost); + }); + return select(candidateHosts); } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/RandomHostManager.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/RandomHostManager.java index ef2b6fd22f..241906a7b4 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/RandomHostManager.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/RandomHostManager.java @@ -38,7 +38,7 @@ public class RandomHostManager extends CommonHostManager { * set round robin */ public RandomHostManager(){ - this.selector = new RandomSelector<>(); + this.selector = new RandomSelector(); } @Override diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/RoundRobinHostManager.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/RoundRobinHostManager.java index e9fef49ecf..ec1945e563 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/RoundRobinHostManager.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/RoundRobinHostManager.java @@ -38,7 +38,7 @@ public class RoundRobinHostManager extends CommonHostManager { * set round robin */ public RoundRobinHostManager(){ - this.selector = new RoundRobinSelector<>(); + this.selector = new RoundRobinSelector(); } @Override diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/RandomSelector.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/RandomSelector.java index e00d6f7a65..6975127b9a 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/RandomSelector.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/RandomSelector.java @@ -17,27 +17,44 @@ package org.apache.dolphinscheduler.server.master.dispatch.host.assign; +import org.apache.dolphinscheduler.remote.utils.Host; + +import java.util.ArrayList; import java.util.Collection; -import java.util.Random; +import java.util.List; +import java.util.concurrent.ThreadLocalRandom; /** * random selector - * @param T */ -public class RandomSelector extends AbstractSelector { - - private final Random random = new Random(); +public class RandomSelector extends AbstractSelector { @Override - public T doSelect(final Collection source) { + public Host doSelect(final Collection source) { - int size = source.size(); - /** - * random select - */ - int randomIndex = random.nextInt(size); + List hosts = new ArrayList<>(source); + int size = hosts.size(); + int[] weights = new int[size]; + int totalWeight = 0; + int index = 0; - return (T) source.toArray()[randomIndex]; + for (Host host : hosts) { + totalWeight += host.getWeight(); + weights[index] = host.getWeight(); + index++; + } + + if (totalWeight > 0) { + int offset = ThreadLocalRandom.current().nextInt(totalWeight); + + for (int i = 0; i < size; i++) { + offset -= weights[i]; + if (offset < 0) { + return hosts.get(i); + } + } + } + return hosts.get(ThreadLocalRandom.current().nextInt(size)); } } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/RoundRobinSelector.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/RoundRobinSelector.java index 06e469fe6b..34a79ac6e8 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/RoundRobinSelector.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/RoundRobinSelector.java @@ -16,27 +16,123 @@ */ package org.apache.dolphinscheduler.server.master.dispatch.host.assign; +import org.apache.dolphinscheduler.remote.utils.Host; import org.springframework.stereotype.Service; -import java.util.Collection; -import java.util.concurrent.atomic.AtomicInteger; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; /** - * round robin selector - * @param T + * Smooth Weight Round Robin */ @Service -public class RoundRobinSelector extends AbstractSelector { +public class RoundRobinSelector extends AbstractSelector { + + private ConcurrentMap> workGroupWeightMap = new ConcurrentHashMap<>(); + + private static final int RECYCLE_PERIOD = 100000; + + private AtomicBoolean updateLock = new AtomicBoolean(); + + protected static class WeightedRoundRobin { + private int weight; + private AtomicLong current = new AtomicLong(0); + private long lastUpdate; + + int getWeight() { + return weight; + } + + void setWeight(int weight) { + this.weight = weight; + current.set(0); + } + + long increaseCurrent() { + return current.addAndGet(weight); + } + + void sel(int total) { + current.addAndGet(-1L * total); + } + + long getLastUpdate() { + return lastUpdate; + } + + void setLastUpdate(long lastUpdate) { + this.lastUpdate = lastUpdate; + } + + } - private final AtomicInteger index = new AtomicInteger(0); @Override - public T doSelect(Collection source) { + public Host doSelect(Collection source) { - int size = source.size(); - /** - * round robin - */ - return (T) source.toArray()[index.getAndIncrement() % size]; + List hosts = new ArrayList<>(source); + String key = hosts.get(0).getWorkGroup(); + ConcurrentMap map = workGroupWeightMap.get(key); + if (map == null) { + workGroupWeightMap.putIfAbsent(key, new ConcurrentHashMap<>()); + map = workGroupWeightMap.get(key); + } + + int totalWeight = 0; + long maxCurrent = Long.MIN_VALUE; + long now = System.currentTimeMillis(); + Host selectedHost = null; + WeightedRoundRobin selectWeightRoundRobin = null; + + for (Host host : hosts) { + String workGroupHost = host.getWorkGroup() + host.getAddress(); + WeightedRoundRobin weightedRoundRobin = map.get(workGroupHost); + int weight = host.getWeight(); + if (weight < 0) { + weight = 0; + } + + if (weightedRoundRobin == null) { + weightedRoundRobin = new WeightedRoundRobin(); + // set weight + weightedRoundRobin.setWeight(weight); + map.putIfAbsent(workGroupHost, weightedRoundRobin); + weightedRoundRobin = map.get(workGroupHost); + } + if (weight != weightedRoundRobin.getWeight()) { + weightedRoundRobin.setWeight(weight); + } + + long cur = weightedRoundRobin.increaseCurrent(); + weightedRoundRobin.setLastUpdate(now); + if (cur > maxCurrent) { + maxCurrent = cur; + selectedHost = host; + selectWeightRoundRobin = weightedRoundRobin; + } + + totalWeight += weight; + } + + + if (!updateLock.get() && hosts.size() != map.size() && updateLock.compareAndSet(false, true)) { + try { + ConcurrentMap newMap = new ConcurrentHashMap<>(map); + newMap.entrySet().removeIf(item -> now - item.getValue().getLastUpdate() > RECYCLE_PERIOD); + workGroupWeightMap.put(key, newMap); + } finally { + updateLock.set(false); + } + } + + if (selectedHost != null) { + selectWeightRoundRobin.sel(totalWeight); + return selectedHost; + } + + return hosts.get(0); } } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/future/TaskFuture.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/future/TaskFuture.java index 918ed6764b..bab4acc23e 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/future/TaskFuture.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/future/TaskFuture.java @@ -28,9 +28,10 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; /** - * task fulture + * task future */ public class TaskFuture { @@ -55,11 +56,11 @@ public class TaskFuture { /** * response command */ - private volatile Command responseCommand; + private AtomicReference responseCommandReference = new AtomicReference<>(); private volatile boolean sendOk = true; - private volatile Throwable cause; + private AtomicReference causeReference; public TaskFuture(long opaque, long timeoutMillis) { this.opaque = opaque; @@ -74,7 +75,7 @@ public class TaskFuture { */ public Command waitResponse() throws InterruptedException { this.latch.await(timeoutMillis, TimeUnit.MILLISECONDS); - return this.responseCommand; + return this.responseCommandReference.get(); } /** @@ -83,7 +84,7 @@ public class TaskFuture { * @param responseCommand responseCommand */ public void putResponse(final Command responseCommand) { - this.responseCommand = responseCommand; + responseCommandReference.set(responseCommand); this.latch.countDown(); FUTURE_TABLE.remove(opaque); } @@ -114,11 +115,11 @@ public class TaskFuture { } public void setCause(Throwable cause) { - this.cause = cause; + causeReference.set(cause); } public Throwable getCause() { - return cause; + return causeReference.get(); } public long getOpaque() { @@ -134,11 +135,11 @@ public class TaskFuture { } public Command getResponseCommand() { - return responseCommand; + return responseCommandReference.get(); } public void setResponseCommand(Command responseCommand) { - this.responseCommand = responseCommand; + responseCommandReference.set(responseCommand); } @@ -166,9 +167,9 @@ public class TaskFuture { ", timeoutMillis=" + timeoutMillis + ", latch=" + latch + ", beginTimestamp=" + beginTimestamp + - ", responseCommand=" + responseCommand + + ", responseCommand=" + responseCommandReference.get() + ", sendOk=" + sendOk + - ", cause=" + cause + + ", cause=" + causeReference.get() + '}'; } } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskResponseService.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskResponseService.java index 7abb31b31c..ba07313a9a 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskResponseService.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskResponseService.java @@ -108,6 +108,7 @@ public class TaskResponseService { TaskResponseEvent taskResponseEvent = eventQueue.take(); persist(taskResponseEvent); } catch (InterruptedException e){ + Thread.currentThread().interrupt(); break; } catch (Exception e){ logger.error("persist task error",e); diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistry.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistry.java index 040ea5a43f..01218e5d8b 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistry.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistry.java @@ -14,8 +14,20 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.server.master.registry; +import org.apache.dolphinscheduler.common.utils.DateUtils; +import org.apache.dolphinscheduler.common.utils.NetUtils; +import org.apache.dolphinscheduler.remote.utils.NamedThreadFactory; +import org.apache.dolphinscheduler.server.master.config.MasterConfig; +import org.apache.dolphinscheduler.server.registry.HeartBeatTask; +import org.apache.dolphinscheduler.server.registry.ZookeeperRegistryCenter; + +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.state.ConnectionState; +import org.apache.curator.framework.state.ConnectionStateListener; + import java.util.Date; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; @@ -23,15 +35,6 @@ import java.util.concurrent.TimeUnit; import javax.annotation.PostConstruct; -import org.apache.curator.framework.CuratorFramework; -import org.apache.curator.framework.state.ConnectionState; -import org.apache.curator.framework.state.ConnectionStateListener; -import org.apache.dolphinscheduler.common.utils.DateUtils; -import org.apache.dolphinscheduler.common.utils.NetUtils; -import org.apache.dolphinscheduler.remote.utils.NamedThreadFactory; -import org.apache.dolphinscheduler.server.master.config.MasterConfig; -import org.apache.dolphinscheduler.server.registry.HeartBeatTask; -import org.apache.dolphinscheduler.server.registry.ZookeeperRegistryCenter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -40,7 +43,7 @@ import org.springframework.stereotype.Service; import com.google.common.collect.Sets; /** - * master registry + * master registry */ @Service public class MasterRegistry { @@ -48,7 +51,7 @@ public class MasterRegistry { private final Logger logger = LoggerFactory.getLogger(MasterRegistry.class); /** - * zookeeper registry center + * zookeeper registry center */ @Autowired private ZookeeperRegistryCenter zookeeperRegistryCenter; @@ -65,19 +68,18 @@ public class MasterRegistry { private ScheduledExecutorService heartBeatExecutor; /** - * worker start time + * master start time */ private String startTime; - @PostConstruct - public void init(){ + public void init() { this.startTime = DateUtils.dateToString(new Date()); this.heartBeatExecutor = Executors.newSingleThreadScheduledExecutor(new NamedThreadFactory("HeartBeatExecutor")); } /** - * registry + * registry */ public void registry() { String address = NetUtils.getHost(); @@ -86,12 +88,12 @@ public class MasterRegistry { zookeeperRegistryCenter.getZookeeperCachedOperator().getZkClient().getConnectionStateListenable().addListener(new ConnectionStateListener() { @Override public void stateChanged(CuratorFramework client, ConnectionState newState) { - if(newState == ConnectionState.LOST){ + if (newState == ConnectionState.LOST) { logger.error("master : {} connection lost from zookeeper", address); - } else if(newState == ConnectionState.RECONNECTED){ + } else if (newState == ConnectionState.RECONNECTED) { logger.info("master : {} reconnected to zookeeper", address); zookeeperRegistryCenter.getZookeeperCachedOperator().persistEphemeral(localNodePath, ""); - } else if(newState == ConnectionState.SUSPENDED){ + } else if (newState == ConnectionState.SUSPENDED) { logger.warn("master : {} connection SUSPENDED ", address); } } @@ -103,36 +105,35 @@ public class MasterRegistry { Sets.newHashSet(getMasterPath()), zookeeperRegistryCenter); - this.heartBeatExecutor.scheduleAtFixedRate(heartBeatTask, masterHeartbeatInterval, masterHeartbeatInterval, TimeUnit.SECONDS); - logger.info("master node : {} registry to ZK successfully with heartBeatInterval : {}s", address, masterHeartbeatInterval); + this.heartBeatExecutor.scheduleAtFixedRate(heartBeatTask, 0, masterHeartbeatInterval, TimeUnit.SECONDS); + logger.info("master node : {} registry to ZK path {} successfully with heartBeatInterval : {}s" + , address, localNodePath, masterHeartbeatInterval); } /** - * remove registry info + * remove registry info */ public void unRegistry() { String address = getLocalAddress(); String localNodePath = getMasterPath(); heartBeatExecutor.shutdownNow(); zookeeperRegistryCenter.getZookeeperCachedOperator().remove(localNodePath); - logger.info("master node : {} unRegistry to ZK.", address); + logger.info("master node : {} unRegistry from ZK path {}." + , address, localNodePath); } /** - * get master path - * @return + * get master path */ private String getMasterPath() { String address = getLocalAddress(); - String localNodePath = this.zookeeperRegistryCenter.getMasterPath() + "/" + address; - return localNodePath; + return this.zookeeperRegistryCenter.getMasterPath() + "/" + address; } /** - * get local address - * @return + * get local address */ - private String getLocalAddress(){ + private String getLocalAddress() { return NetUtils.getHost() + ":" + masterConfig.getListenPort(); diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/ConditionsTaskExecThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/ConditionsTaskExecThread.java index a410f99fc1..021f10d444 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/ConditionsTaskExecThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/ConditionsTaskExecThread.java @@ -27,6 +27,8 @@ import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.common.utils.LoggerUtils; import org.apache.dolphinscheduler.common.utils.NetUtils; import org.apache.dolphinscheduler.dao.entity.TaskInstance; +import org.apache.dolphinscheduler.server.utils.LogUtils; + import org.slf4j.LoggerFactory; import java.util.ArrayList; @@ -60,6 +62,7 @@ public class ConditionsTaskExecThread extends MasterBaseTaskExecThread { */ public ConditionsTaskExecThread(TaskInstance taskInstance) { super(taskInstance); + taskInstance.setStartTime(new Date()); } @Override @@ -122,7 +125,7 @@ public class ConditionsTaskExecThread extends MasterBaseTaskExecThread { } private void initTaskParameters() { - this.taskInstance.setLogPath(getTaskLogPath(taskInstance)); + this.taskInstance.setLogPath(LogUtils.getTaskLogPath(taskInstance)); this.taskInstance.setHost(NetUtils.getHost() + Constants.COLON + masterConfig.getListenPort()); taskInstance.setState(ExecutionStatus.RUNNING_EXECUTION); taskInstance.setStartTime(new Date()); diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/DependentTaskExecThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/DependentTaskExecThread.java index 183f1aac42..319afedd7b 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/DependentTaskExecThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/DependentTaskExecThread.java @@ -14,9 +14,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.server.master.runner; -import com.fasterxml.jackson.annotation.JsonFormat; +import static org.apache.dolphinscheduler.common.Constants.DEPENDENT_SPLIT; + import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.DependResult; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; @@ -24,16 +26,22 @@ import org.apache.dolphinscheduler.common.model.DependentTaskModel; import org.apache.dolphinscheduler.common.task.dependent.DependentParameters; import org.apache.dolphinscheduler.common.thread.Stopper; import org.apache.dolphinscheduler.common.utils.DependentUtils; -import org.apache.dolphinscheduler.common.utils.*; +import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.LoggerUtils; import org.apache.dolphinscheduler.common.utils.NetUtils; import org.apache.dolphinscheduler.dao.entity.TaskInstance; +import org.apache.dolphinscheduler.server.utils.LogUtils; import org.apache.dolphinscheduler.server.utils.DependentExecute; + +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + import org.slf4j.LoggerFactory; -import java.util.*; - -import static org.apache.dolphinscheduler.common.Constants.DEPENDENT_SPLIT; +import com.fasterxml.jackson.annotation.JsonFormat; public class DependentTaskExecThread extends MasterBaseTaskExecThread { @@ -64,6 +72,7 @@ public class DependentTaskExecThread extends MasterBaseTaskExecThread { */ public DependentTaskExecThread(TaskInstance taskInstance) { super(taskInstance); + taskInstance.setStartTime(new Date()); } @@ -171,7 +180,7 @@ public class DependentTaskExecThread extends MasterBaseTaskExecThread { } private void initTaskParameters() { - taskInstance.setLogPath(getTaskLogPath(taskInstance)); + taskInstance.setLogPath(LogUtils.getTaskLogPath(taskInstance)); taskInstance.setHost(NetUtils.getHost() + Constants.COLON + masterConfig.getListenPort()); taskInstance.setState(ExecutionStatus.RUNNING_EXECUTION); taskInstance.setStartTime(new Date()); diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterBaseTaskExecThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterBaseTaskExecThread.java index aac201c403..ea3ad19950 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterBaseTaskExecThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterBaseTaskExecThread.java @@ -14,28 +14,26 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.server.master.runner; -import ch.qos.logback.classic.LoggerContext; -import ch.qos.logback.classic.sift.SiftingAppender; -import org.apache.dolphinscheduler.common.Constants; +import static org.apache.dolphinscheduler.common.Constants.UNDERLINE; + import org.apache.dolphinscheduler.common.enums.ExecutionStatus; -import org.apache.dolphinscheduler.common.utils.*; +import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.dao.AlertDao; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.TaskInstance; -import org.apache.dolphinscheduler.server.log.TaskLogDiscriminator; import org.apache.dolphinscheduler.server.master.config.MasterConfig; import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; import org.apache.dolphinscheduler.service.process.ProcessService; import org.apache.dolphinscheduler.service.queue.TaskPriorityQueue; import org.apache.dolphinscheduler.service.queue.TaskPriorityQueueImpl; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import static org.apache.dolphinscheduler.common.Constants.*; import java.util.concurrent.Callable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * master task exec base class @@ -82,11 +80,13 @@ public class MasterBaseTaskExecThread implements Callable { * taskUpdateQueue */ private TaskPriorityQueue taskUpdateQueue; + /** * constructor of MasterBaseTaskExecThread - * @param taskInstance task instance + * + * @param taskInstance task instance */ - public MasterBaseTaskExecThread(TaskInstance taskInstance){ + public MasterBaseTaskExecThread(TaskInstance taskInstance) { this.processService = SpringApplicationContext.getBean(ProcessService.class); this.alertDao = SpringApplicationContext.getBean(AlertDao.class); this.cancel = false; @@ -97,24 +97,26 @@ public class MasterBaseTaskExecThread implements Callable { /** * get task instance + * * @return TaskInstance */ - public TaskInstance getTaskInstance(){ + public TaskInstance getTaskInstance() { return this.taskInstance; } /** * kill master base task exec thread */ - public void kill(){ + public void kill() { this.cancel = true; } /** * submit master base task exec thread + * * @return TaskInstance */ - protected TaskInstance submit(){ + protected TaskInstance submit() { Integer commitRetryTimes = masterConfig.getMasterTaskCommitRetryTimes(); Integer commitRetryInterval = masterConfig.getMasterTaskCommitInterval(); @@ -153,14 +155,13 @@ public class MasterBaseTaskExecThread implements Callable { } - /** * dispatcht task + * * @param taskInstance taskInstance * @return whether submit task success */ public Boolean dispatchTask(TaskInstance taskInstance) { - try{ if(taskInstance.isConditionsTask() || taskInstance.isDependTask() @@ -171,9 +172,10 @@ public class MasterBaseTaskExecThread implements Callable { logger.info(String.format("submit task , but task [%s] state [%s] is already finished. ", taskInstance.getName(), taskInstance.getState().toString())); return true; } - // task cannot submit when running - if(taskInstance.getState() == ExecutionStatus.RUNNING_EXECUTION){ - logger.info(String.format("submit to task, but task [%s] state already be running. ", taskInstance.getName())); + // task cannot be submitted because its execution state is RUNNING or DELAY. + if (taskInstance.getState() == ExecutionStatus.RUNNING_EXECUTION + || taskInstance.getState() == ExecutionStatus.DELAY_EXECUTION) { + logger.info("submit task, but the status of the task {} is already running or delayed.", taskInstance.getName()); return true; } logger.info("task ready to submit: {}", taskInstance); @@ -198,7 +200,7 @@ public class MasterBaseTaskExecThread implements Callable { /** - * buildTaskPriorityInfo + * buildTaskPriorityInfo * * @param processInstancePriority processInstancePriority * @param processInstanceId processInstanceId @@ -211,7 +213,7 @@ public class MasterBaseTaskExecThread implements Callable { int processInstanceId, int taskInstancePriority, int taskInstanceId, - String workerGroup){ + String workerGroup) { return processInstancePriority + UNDERLINE + processInstanceId + @@ -225,14 +227,16 @@ public class MasterBaseTaskExecThread implements Callable { /** * submit wait complete + * * @return true */ - protected Boolean submitWaitComplete(){ + protected Boolean submitWaitComplete() { return true; } /** * call + * * @return boolean * @throws Exception exception */ @@ -242,35 +246,4 @@ public class MasterBaseTaskExecThread implements Callable { return submitWaitComplete(); } - /** - * get task log path - * @return log path - */ - public String getTaskLogPath(TaskInstance task) { - String logPath; - try{ - String baseLog = ((TaskLogDiscriminator) ((SiftingAppender) ((LoggerContext) LoggerFactory.getILoggerFactory()) - .getLogger("ROOT") - .getAppender("TASKLOGFILE")) - .getDiscriminator()).getLogBase(); - if (baseLog.startsWith(Constants.SINGLE_SLASH)){ - logPath = baseLog + Constants.SINGLE_SLASH + - task.getProcessDefinitionId() + Constants.SINGLE_SLASH + - task.getProcessInstanceId() + Constants.SINGLE_SLASH + - task.getId() + ".log"; - }else{ - logPath = System.getProperty("user.dir") + Constants.SINGLE_SLASH + - baseLog + Constants.SINGLE_SLASH + - task.getProcessDefinitionId() + Constants.SINGLE_SLASH + - task.getProcessInstanceId() + Constants.SINGLE_SLASH + - task.getId() + ".log"; - } - }catch (Exception e){ - logger.error("logger", e); - logPath = ""; - } - return logPath; - } - - } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java index 5be6050f87..788b30638e 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java @@ -16,10 +16,21 @@ */ package org.apache.dolphinscheduler.server.master.runner; -import com.google.common.collect.Lists; -import org.apache.commons.io.FileUtils; +import static org.apache.dolphinscheduler.common.Constants.CMDPARAM_COMPLEMENT_DATA_END_DATE; +import static org.apache.dolphinscheduler.common.Constants.CMDPARAM_COMPLEMENT_DATA_START_DATE; +import static org.apache.dolphinscheduler.common.Constants.CMDPARAM_RECOVERY_START_NODE_STRING; +import static org.apache.dolphinscheduler.common.Constants.CMDPARAM_START_NODE_NAMES; +import static org.apache.dolphinscheduler.common.Constants.DEFAULT_WORKER_GROUP; +import static org.apache.dolphinscheduler.common.Constants.SEC_2_MINUTES_TIME_UNIT; + import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.common.enums.*; +import org.apache.dolphinscheduler.common.enums.CommandType; +import org.apache.dolphinscheduler.common.enums.DependResult; +import org.apache.dolphinscheduler.common.enums.ExecutionStatus; +import org.apache.dolphinscheduler.common.enums.FailureStrategy; +import org.apache.dolphinscheduler.common.enums.Flag; +import org.apache.dolphinscheduler.common.enums.Priority; +import org.apache.dolphinscheduler.common.enums.TaskDependType; import org.apache.dolphinscheduler.common.graph.DAG; import org.apache.dolphinscheduler.common.model.TaskNode; import org.apache.dolphinscheduler.common.model.TaskNodeRelation; @@ -27,7 +38,13 @@ import org.apache.dolphinscheduler.common.process.ProcessDag; import org.apache.dolphinscheduler.common.task.conditions.ConditionsParameters; import org.apache.dolphinscheduler.common.thread.Stopper; import org.apache.dolphinscheduler.common.thread.ThreadUtils; -import org.apache.dolphinscheduler.common.utils.*; +import org.apache.dolphinscheduler.common.utils.CollectionUtils; +import org.apache.dolphinscheduler.common.utils.CommonUtils; +import org.apache.dolphinscheduler.common.utils.DateUtils; +import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.OSUtils; +import org.apache.dolphinscheduler.common.utils.ParameterUtils; +import org.apache.dolphinscheduler.common.utils.StringUtils; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.Schedule; import org.apache.dolphinscheduler.dao.entity.TaskInstance; @@ -37,17 +54,26 @@ import org.apache.dolphinscheduler.server.master.config.MasterConfig; import org.apache.dolphinscheduler.server.utils.AlertManager; import org.apache.dolphinscheduler.service.process.ProcessService; import org.apache.dolphinscheduler.service.quartz.cron.CronUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; + +import org.apache.commons.io.FileUtils; import java.io.File; import java.io.IOException; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Date; +import java.util.Iterator; +import java.util.List; +import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; -import static org.apache.dolphinscheduler.common.Constants.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.collect.Lists; /** * master exec thread,split dag @@ -470,9 +496,6 @@ public class MasterExecThread implements Runnable { // task instance whether alert taskInstance.setAlertFlag(Flag.NO); - // task instance start time - taskInstance.setStartTime(new Date()); - // task instance flag taskInstance.setFlag(Flag.YES); @@ -501,6 +524,8 @@ public class MasterExecThread implements Runnable { taskInstance.setWorkerGroup(taskWorkerGroup); } + // delay execution time + taskInstance.setDelayTime(taskNode.getDelayTime()); } return taskInstance; } @@ -719,9 +744,10 @@ public class MasterExecThread implements Runnable { * @return ExecutionStatus */ private ExecutionStatus runningState(ExecutionStatus state){ - if(state == ExecutionStatus.READY_STOP || - state == ExecutionStatus.READY_PAUSE || - state == ExecutionStatus.WAITTING_THREAD){ + if (state == ExecutionStatus.READY_STOP + || state == ExecutionStatus.READY_PAUSE + || state == ExecutionStatus.WAITTING_THREAD + || state == ExecutionStatus.DELAY_EXECUTION) { // if the running task is not completed, the state remains unchanged return state; }else{ diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThread.java index a2dd7cea8d..72ee0fcb89 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThread.java @@ -24,6 +24,8 @@ import org.apache.dolphinscheduler.common.model.TaskNode; import org.apache.dolphinscheduler.common.task.TaskTimeoutParameter; import org.apache.dolphinscheduler.common.thread.Stopper; import org.apache.dolphinscheduler.common.utils.CollectionUtils; +import org.apache.dolphinscheduler.common.utils.DateUtils; +import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.StringUtils; import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; import org.apache.dolphinscheduler.dao.entity.TaskInstance; @@ -39,7 +41,6 @@ import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; import java.util.Date; import java.util.Set; -import org.apache.dolphinscheduler.common.utils.*; /** @@ -150,7 +151,7 @@ public class MasterTaskExecThread extends MasterBaseTaskExecThread { break; } if(checkTimeout){ - long remainTime = getRemaintime(taskTimeoutParameter.getInterval() * 60L); + long remainTime = DateUtils.getRemainTime(taskInstance.getStartTime(), taskTimeoutParameter.getInterval() * 60L); if (remainTime < 0) { logger.warn("task id: {} execution time out",taskInstance.getId()); // process define @@ -256,16 +257,4 @@ public class MasterTaskExecThread extends MasterBaseTaskExecThread { TaskNode taskNode = JSONUtils.parseObject(taskJson, TaskNode.class); return taskNode.getTaskTimeoutParameter(); } - - - /** - * get remain time?s? - * - * @return remain time - */ - private long getRemaintime(long timeoutSeconds) { - Date startTime = taskInstance.getStartTime(); - long usedTime = (System.currentTimeMillis() - startTime.getTime()) / 1000; - return timeoutSeconds - usedTime; - } } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/registry/ZookeeperNodeManager.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/registry/ZookeeperNodeManager.java index f039fb5532..b1a5edee38 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/registry/ZookeeperNodeManager.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/registry/ZookeeperNodeManager.java @@ -151,10 +151,10 @@ public class ZookeeperNodeManager implements InitializingBean { private String parseGroup(String path){ String[] parts = path.split("\\/"); - if(parts.length != 6){ + if (parts.length < 6) { throw new IllegalArgumentException(String.format("worker group path : %s is not valid, ignore", path)); } - String group = parts[4]; + String group = parts[parts.length - 2]; return group; } } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/AlertManager.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/AlertManager.java index 49ec9d3fdd..58ade83d68 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/AlertManager.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/AlertManager.java @@ -14,29 +14,29 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.dolphinscheduler.server.utils; +package org.apache.dolphinscheduler.server.utils; import org.apache.dolphinscheduler.common.enums.AlertType; import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.common.enums.ShowType; import org.apache.dolphinscheduler.common.enums.WarningType; -import org.apache.dolphinscheduler.common.utils.DateUtils; -import org.apache.dolphinscheduler.common.utils.*; +import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.dao.AlertDao; import org.apache.dolphinscheduler.dao.DaoFactory; import org.apache.dolphinscheduler.dao.entity.Alert; +import org.apache.dolphinscheduler.dao.entity.ProcessAlertContent; import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.TaskInstance; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.util.ArrayList; import java.util.Date; -import java.util.LinkedHashMap; import java.util.List; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + /** * alert manager */ @@ -50,8 +50,7 @@ public class AlertManager { /** * alert dao */ - private AlertDao alertDao = DaoFactory.getDaoInstance(AlertDao.class); - + private final AlertDao alertDao = DaoFactory.getDaoInstance(AlertDao.class); /** * command type convert chinese @@ -86,64 +85,52 @@ public class AlertManager { } } - /** - * process instance format - */ - private static final String PROCESS_INSTANCE_FORMAT = - "\"id:%d\"," + - "\"name:%s\"," + - "\"job type: %s\"," + - "\"state: %s\"," + - "\"recovery:%s\"," + - "\"run time: %d\"," + - "\"start time: %s\"," + - "\"end time: %s\"," + - "\"host: %s\"" ; - /** * get process instance content - * @param processInstance process instance - * @param taskInstances task instance list + * + * @param processInstance process instance + * @param taskInstances task instance list * @return process instance format content */ public String getContentProcessInstance(ProcessInstance processInstance, - List taskInstances){ + List taskInstances) { String res = ""; - if(processInstance.getState().typeIsSuccess()){ - res = String.format(PROCESS_INSTANCE_FORMAT, - processInstance.getId(), - processInstance.getName(), - getCommandCnName(processInstance.getCommandType()), - processInstance.getState().toString(), - processInstance.getRecovery().toString(), - processInstance.getRunTimes(), - DateUtils.dateToString(processInstance.getStartTime()), - DateUtils.dateToString(processInstance.getEndTime()), - processInstance.getHost() + if (processInstance.getState().typeIsSuccess()) { + List successTaskList = new ArrayList<>(1); + ProcessAlertContent processAlertContent = ProcessAlertContent.newBuilder() + .processId(processInstance.getId()) + .processName(processInstance.getName()) + .processType(processInstance.getCommandType()) + .processState(processInstance.getState()) + .recovery(processInstance.getRecovery()) + .runTimes(processInstance.getRunTimes()) + .processStartTime(processInstance.getStartTime()) + .processEndTime(processInstance.getEndTime()) + .processHost(processInstance.getHost()) + .build(); + successTaskList.add(processAlertContent); + res = JSONUtils.toJsonString(successTaskList); + } else if (processInstance.getState().typeIsFailure()) { - ); - res = "[" + res + "]"; - }else if(processInstance.getState().typeIsFailure()){ - - List failedTaskList = new ArrayList<>(); - - for(TaskInstance task : taskInstances){ - if(task.getState().typeIsSuccess()){ + List failedTaskList = new ArrayList<>(); + for (TaskInstance task : taskInstances) { + if (task.getState().typeIsSuccess()) { continue; } - LinkedHashMap failedTaskMap = new LinkedHashMap(); - failedTaskMap.put("process instance id", String.valueOf(processInstance.getId())); - failedTaskMap.put("process instance name", processInstance.getName()); - failedTaskMap.put("task id", String.valueOf(task.getId())); - failedTaskMap.put("task name", task.getName()); - failedTaskMap.put("task type", task.getTaskType()); - failedTaskMap.put("task state", task.getState().toString()); - failedTaskMap.put("task start time", DateUtils.dateToString(task.getStartTime())); - failedTaskMap.put("task end time", DateUtils.dateToString(task.getEndTime())); - failedTaskMap.put("host", task.getHost()); - failedTaskMap.put("log path", task.getLogPath()); - failedTaskList.add(failedTaskMap); + ProcessAlertContent processAlertContent = ProcessAlertContent.newBuilder() + .processId(processInstance.getId()) + .processName(processInstance.getName()) + .taskId(task.getId()) + .taskName(task.getName()) + .taskType(task.getTaskType()) + .taskState(task.getState()) + .taskStartTime(task.getStartTime()) + .taskEndTime(task.getEndTime()) + .taskHost(task.getHost()) + .logPath(task.getLogPath()) + .build(); + failedTaskList.add(processAlertContent); } res = JSONUtils.toJsonString(failedTaskList); } @@ -154,21 +141,22 @@ public class AlertManager { /** * getting worker fault tolerant content * - * @param processInstance process instance + * @param processInstance process instance * @param toleranceTaskList tolerance task list * @return worker tolerance content */ - private String getWorkerToleranceContent(ProcessInstance processInstance, List toleranceTaskList){ + private String getWorkerToleranceContent(ProcessInstance processInstance, List toleranceTaskList) { - List> toleranceTaskInstanceList = new ArrayList<>(); + List toleranceTaskInstanceList = new ArrayList<>(); - for(TaskInstance taskInstance: toleranceTaskList){ - LinkedHashMap toleranceWorkerContentMap = new LinkedHashMap(); - toleranceWorkerContentMap.put("process name", processInstance.getName()); - toleranceWorkerContentMap.put("task name", taskInstance.getName()); - toleranceWorkerContentMap.put("host", taskInstance.getHost()); - toleranceWorkerContentMap.put("task retry times", String.valueOf(taskInstance.getRetryTimes())); - toleranceTaskInstanceList.add(toleranceWorkerContentMap); + for (TaskInstance taskInstance : toleranceTaskList) { + ProcessAlertContent processAlertContent = ProcessAlertContent.newBuilder() + .processName(processInstance.getName()) + .taskName(taskInstance.getName()) + .taskHost(taskInstance.getHost()) + .retryTimes(taskInstance.getRetryTimes()) + .build(); + toleranceTaskInstanceList.add(processAlertContent); } return JSONUtils.toJsonString(toleranceTaskInstanceList); } @@ -176,11 +164,11 @@ public class AlertManager { /** * send worker alert fault tolerance * - * @param processInstance process instance + * @param processInstance process instance * @param toleranceTaskList tolerance task list */ - public void sendAlertWorkerToleranceFault(ProcessInstance processInstance, List toleranceTaskList){ - try{ + public void sendAlertWorkerToleranceFault(ProcessInstance processInstance, List toleranceTaskList) { + try { Alert alert = new Alert(); alert.setTitle("worker fault tolerance"); alert.setShowType(ShowType.TABLE); @@ -188,13 +176,13 @@ public class AlertManager { alert.setContent(content); alert.setAlertType(AlertType.EMAIL); alert.setCreateTime(new Date()); - alert.setAlertGroupId(processInstance.getWarningGroupId() == null ? 1:processInstance.getWarningGroupId()); + alert.setAlertGroupId(processInstance.getWarningGroupId() == null ? 1 : processInstance.getWarningGroupId()); alert.setReceivers(processInstance.getProcessDefinition().getReceivers()); alert.setReceiversCc(processInstance.getProcessDefinition().getReceiversCc()); alertDao.addAlert(alert); logger.info("add alert to db , alert : {}", alert.toString()); - }catch (Exception e){ + } catch (Exception e) { logger.error("send alert failed:{} ", e.getMessage()); } @@ -202,40 +190,40 @@ public class AlertManager { /** * send process instance alert - * @param processInstance process instance - * @param taskInstances task instance list + * + * @param processInstance process instance + * @param taskInstances task instance list */ public void sendAlertProcessInstance(ProcessInstance processInstance, - List taskInstances){ + List taskInstances) { boolean sendWarnning = false; WarningType warningType = processInstance.getWarningType(); - switch (warningType){ + switch (warningType) { case ALL: - if(processInstance.getState().typeIsFinished()){ + if (processInstance.getState().typeIsFinished()) { sendWarnning = true; } break; case SUCCESS: - if(processInstance.getState().typeIsSuccess()){ + if (processInstance.getState().typeIsSuccess()) { sendWarnning = true; } break; case FAILURE: - if(processInstance.getState().typeIsFailure()){ + if (processInstance.getState().typeIsFailure()) { sendWarnning = true; } break; - default: + default: } - if(!sendWarnning){ + if (!sendWarnning) { return; } Alert alert = new Alert(); - String cmdName = getCommandCnName(processInstance.getCommandType()); - String success = processInstance.getState().typeIsSuccess() ? "success" :"failed"; + String success = processInstance.getState().typeIsSuccess() ? "success" : "failed"; alert.setTitle(cmdName + " " + success); ShowType showType = processInstance.getState().typeIsSuccess() ? ShowType.TEXT : ShowType.TABLE; alert.setShowType(showType); @@ -254,7 +242,7 @@ public class AlertManager { /** * send process timeout alert * - * @param processInstance process instance + * @param processInstance process instance * @param processDefinition process definition */ public void sendProcessTimeoutAlert(ProcessInstance processInstance, ProcessDefinition processDefinition) { diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/LogUtils.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/LogUtils.java new file mode 100644 index 0000000000..93008b9d64 --- /dev/null +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/LogUtils.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.server.utils; + +import org.apache.dolphinscheduler.dao.entity.TaskInstance; +import org.apache.dolphinscheduler.server.entity.TaskExecutionContext; +import org.apache.dolphinscheduler.server.log.TaskLogDiscriminator; + +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Optional; + +import javax.transaction.NotSupportedException; + +import org.slf4j.LoggerFactory; + +import ch.qos.logback.classic.sift.SiftingAppender; +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.core.spi.AppenderAttachable; + +public class LogUtils { + + private LogUtils() throws NotSupportedException { + throw new NotSupportedException(); + } + + /** + * get task log path + */ + @SuppressWarnings("unchecked") + private static String getTaskLogPath(int processDefinitionId, int processInstanceId, int taskInstanceId) { + // Optional.map will be skipped if null + return Optional.of(LoggerFactory.getILoggerFactory()) + .map(e -> (AppenderAttachable) (e.getLogger("ROOT"))) + .map(e -> (SiftingAppender) (e.getAppender("TASKLOGFILE"))) + .map(e -> ((TaskLogDiscriminator) (e.getDiscriminator()))) + .map(TaskLogDiscriminator::getLogBase) + .map(e -> Paths.get(e) + .toAbsolutePath() + .resolve(String.valueOf(processDefinitionId)) + .resolve(String.valueOf(processInstanceId)) + .resolve(taskInstanceId + ".log")) + .map(Path::toString) + .orElse(""); + } + + /** + * get task log path by TaskInstance + */ + public static String getTaskLogPath(TaskInstance taskInstance) { + return getTaskLogPath(taskInstance.getProcessDefinitionId(), taskInstance.getProcessInstanceId(), taskInstance.getId()); + } + + /** + * get task log path by TaskExecutionContext + */ + public static String getTaskLogPath(TaskExecutionContext taskExecutionContext) { + return getTaskLogPath(taskExecutionContext.getProcessId(), taskExecutionContext.getProcessInstanceId(), taskExecutionContext.getTaskInstanceId()); + } + +} diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ProcessUtils.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ProcessUtils.java index 310ab62bf1..cf49285b9f 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ProcessUtils.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ProcessUtils.java @@ -14,53 +14,52 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.server.utils; -import java.nio.charset.StandardCharsets; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.utils.CommonUtils; +import org.apache.dolphinscheduler.common.utils.FileUtils; import org.apache.dolphinscheduler.common.utils.LoggerUtils; import org.apache.dolphinscheduler.common.utils.OSUtils; import org.apache.dolphinscheduler.common.utils.StringUtils; -import org.apache.commons.io.FileUtils; import org.apache.dolphinscheduler.remote.utils.Host; import org.apache.dolphinscheduler.server.entity.TaskExecutionContext; import org.apache.dolphinscheduler.service.log.LogClientService; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; -import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; - /** - * mainly used to get the start command line of a process + * mainly used to get the start command line of a process. */ public class ProcessUtils { /** - * logger + * logger. */ - private final static Logger logger = LoggerFactory.getLogger(ProcessUtils.class); + private static final Logger logger = LoggerFactory.getLogger(ProcessUtils.class); /** * Initialization regularization, solve the problem of pre-compilation performance, - * avoid the thread safety problem of multi-thread operation + * avoid the thread safety problem of multi-thread operation. */ private static final Pattern MACPATTERN = Pattern.compile("-[+|-]-\\s(\\d+)"); private static final Pattern WINDOWSATTERN = Pattern.compile("(\\d+)"); /** - * build command line characters + * build command line characters. * @param commandList command list * @return command - * @throws IOException io exception */ - public static String buildCommandStr(List commandList) throws IOException { + public static String buildCommandStr(List commandList) { String cmdstr; String[] cmd = commandList.toArray(new String[commandList.size()]); SecurityManager security = System.getSecurityManager(); @@ -102,7 +101,6 @@ public class ProcessUtils { } } - cmdstr = createCommandLine( isShellFile(executablePath) ? VERIFICATION_CMD_BAT : VERIFICATION_WIN32, quoteString(executablePath), cmd); @@ -111,13 +109,12 @@ public class ProcessUtils { } /** - * get executable path + * get executable path. * * @param path path * @return executable path - * @throws IOException io exception */ - private static String getExecutablePath(String path) throws IOException { + private static String getExecutablePath(String path) { boolean pathIsQuoted = isQuoted(true, path, "Executable name has embedded quote, split the arguments"); File fileToRun = new File(pathIsQuoted ? path.substring(1, path.length() - 1) : path); @@ -125,7 +122,7 @@ public class ProcessUtils { } /** - * whether is shell file + * whether is shell file. * * @param executablePath executable path * @return true if endsWith .CMD or .BAT @@ -136,7 +133,7 @@ public class ProcessUtils { } /** - * quote string + * quote string. * * @param arg argument * @return format arg @@ -147,7 +144,7 @@ public class ProcessUtils { } /** - * get tokens from command + * get tokens from command. * * @param command command * @return token string array @@ -162,7 +159,7 @@ public class ProcessUtils { } /** - * Lazy Pattern + * Lazy Pattern. */ private static class LazyPattern { // Escape-support version: @@ -171,34 +168,29 @@ public class ProcessUtils { } /** - * verification cmd bat + * verification cmd bat. */ private static final int VERIFICATION_CMD_BAT = 0; /** - * verification win32 + * verification win32. */ private static final int VERIFICATION_WIN32 = 1; /** - * verification legacy + * verification legacy. */ private static final int VERIFICATION_LEGACY = 2; /** - * escape verification + * escape verification. */ private static final char[][] ESCAPE_VERIFICATION = {{' ', '\t', '<', '>', '&', '|', '^'}, - {' ', '\t', '<', '>'}, {' ', '\t'}}; + {' ', '\t', '<', '>'}, {' ', '\t'}}; /** - * matcher - */ - private static Matcher matcher; - - /** - * create command line + * create command line. * @param verificationType verification type * @param executablePath executable path * @param cmd cmd @@ -227,7 +219,7 @@ public class ProcessUtils { } /** - * whether is quoted + * whether is quoted. * @param noQuotesInside * @param arg * @param errorMessage @@ -255,7 +247,7 @@ public class ProcessUtils { } /** - * whether needs escaping + * whether needs escaping. * * @param verificationType verification type * @param arg arg @@ -277,16 +269,14 @@ public class ProcessUtils { } /** - * kill yarn application + * kill yarn application. * * @param appIds app id list * @param logger logger * @param tenantCode tenant code * @param executePath execute path - * @throws IOException io exception */ - public static void cancelApplication(List appIds, Logger logger, String tenantCode,String executePath) - throws IOException { + public static void cancelApplication(List appIds, Logger logger, String tenantCode, String executePath) { if (appIds.size() > 0) { String appid = appIds.get(appIds.size() - 1); String commandFile = String @@ -324,17 +314,17 @@ public class ProcessUtils { } /** - * kill tasks according to different task types + * kill tasks according to different task types. * * @param taskExecutionContext taskExecutionContext */ public static void kill(TaskExecutionContext taskExecutionContext) { try { int processId = taskExecutionContext.getProcessId(); - if(processId == 0 ){ + if (processId == 0) { logger.error("process kill failed, process id :{}, task id:{}", processId, taskExecutionContext.getTaskInstanceId()); - return ; + return; } String cmd = String.format("sudo kill -9 %s", getPidsStr(processId)); @@ -352,13 +342,13 @@ public class ProcessUtils { } /** - * get pids str + * get pids str. * * @param processId process id * @return pids * @throws Exception exception */ - public static String getPidsStr(int processId)throws Exception{ + public static String getPidsStr(int processId) throws Exception { StringBuilder sb = new StringBuilder(); Matcher mat; // pstree pid get sub pids @@ -370,14 +360,14 @@ public class ProcessUtils { mat = WINDOWSATTERN.matcher(pids); } - while (mat.find()){ + while (mat.find()) { sb.append(mat.group(1)).append(" "); } return sb.toString().trim(); } /** - * find logs and kill yarn tasks + * find logs and kill yarn tasks. * * @param taskExecutionContext taskExecutionContext */ @@ -392,7 +382,7 @@ public class ProcessUtils { Constants.RPC_PORT, taskExecutionContext.getLogPath()); } finally { - if(logClient != null){ + if (logClient != null) { logClient.close(); } } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/cache/impl/TaskExecutionContextCacheManagerImpl.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/cache/impl/TaskExecutionContextCacheManagerImpl.java index 009332f05c..9c92fb2d64 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/cache/impl/TaskExecutionContextCacheManagerImpl.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/cache/impl/TaskExecutionContextCacheManagerImpl.java @@ -32,7 +32,7 @@ public class TaskExecutionContextCacheManagerImpl implements TaskExecutionContex /** - * taskInstance caceh + * taskInstance cache */ private Map taskExecutionContextCache = new ConcurrentHashMap<>(); diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/config/WorkerConfig.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/config/WorkerConfig.java index 2dedaf8e1b..fa97403527 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/config/WorkerConfig.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/config/WorkerConfig.java @@ -49,6 +49,9 @@ public class WorkerConfig { @Value("${worker.listen.port: 1234}") private int listenPort; + @Value("${worker.weight:100}") + private int weight; + public int getListenPort() { return listenPort; } @@ -107,4 +110,13 @@ public class WorkerConfig { public void setWorkerMaxCpuloadAvg(int workerMaxCpuloadAvg) { this.workerMaxCpuloadAvg = workerMaxCpuloadAvg; } + + + public int getWeight() { + return weight; + } + + public void setWeight(int weight) { + this.weight = weight; + } } \ No newline at end of file diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java index 2ce4515279..3717ce37ae 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java @@ -17,16 +17,16 @@ package org.apache.dolphinscheduler.server.worker.processor; - -import ch.qos.logback.classic.LoggerContext; -import ch.qos.logback.classic.sift.SiftingAppender; -import com.github.rholder.retry.RetryException; -import io.netty.channel.Channel; -import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.enums.TaskType; import org.apache.dolphinscheduler.common.thread.ThreadUtils; -import org.apache.dolphinscheduler.common.utils.*; +import org.apache.dolphinscheduler.common.utils.DateUtils; +import org.apache.dolphinscheduler.common.utils.FileUtils; +import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.LoggerUtils; +import org.apache.dolphinscheduler.common.utils.NetUtils; +import org.apache.dolphinscheduler.common.utils.Preconditions; +import org.apache.dolphinscheduler.common.utils.RetryerUtils; import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.CommandType; import org.apache.dolphinscheduler.remote.command.TaskExecuteAckCommand; @@ -34,18 +34,23 @@ import org.apache.dolphinscheduler.remote.command.TaskExecuteRequestCommand; import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor; import org.apache.dolphinscheduler.remote.utils.JsonSerializer; import org.apache.dolphinscheduler.server.entity.TaskExecutionContext; -import org.apache.dolphinscheduler.server.log.TaskLogDiscriminator; +import org.apache.dolphinscheduler.server.utils.LogUtils; import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; import org.apache.dolphinscheduler.server.worker.runner.TaskExecuteThread; import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.util.Date; import java.util.Optional; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.github.rholder.retry.RetryException; + +import io.netty.channel.Channel; + /** * worker request processor */ @@ -97,7 +102,7 @@ public class TaskExecuteProcessor implements NettyRequestProcessor { return; } - taskExecutionContext.setHost(NetUtils.getHost() + ":" + workerConfig.getListenPort()); + taskExecutionContext.setHost(NetUtils.getHost() + ":" + workerConfig.getListenPort()); // custom logger Logger taskLogger = LoggerFactory.getLogger(LoggerUtils.buildTaskId(LoggerUtils.TASK_LOGGER_INFO_PREFIX, @@ -122,7 +127,15 @@ public class TaskExecuteProcessor implements NettyRequestProcessor { taskCallbackService.addRemoteChannel(taskExecutionContext.getTaskInstanceId(), new NettyRemoteChannel(channel, command.getOpaque())); - // tell master that task is in executing + if (DateUtils.getRemainTime(taskExecutionContext.getFirstSubmitTime(), taskExecutionContext.getDelayTime() * 60L) > 0) { + taskExecutionContext.setCurrentExecutionStatus(ExecutionStatus.DELAY_EXECUTION); + taskExecutionContext.setStartTime(null); + } else { + taskExecutionContext.setCurrentExecutionStatus(ExecutionStatus.RUNNING_EXECUTION); + taskExecutionContext.setStartTime(new Date()); + } + + // tell master the status of this task (RUNNING_EXECUTION or DELAY_EXECUTION) final Command ackCommand = buildAckCommand(taskExecutionContext).convert2Command(); try { @@ -137,28 +150,6 @@ public class TaskExecuteProcessor implements NettyRequestProcessor { } } - /** - * get task log path - * @return log path - */ - private String getTaskLogPath(TaskExecutionContext taskExecutionContext) { - String baseLog = ((TaskLogDiscriminator) ((SiftingAppender) ((LoggerContext) LoggerFactory.getILoggerFactory()) - .getLogger("ROOT") - .getAppender("TASKLOGFILE")) - .getDiscriminator()).getLogBase(); - if (baseLog.startsWith(Constants.SINGLE_SLASH)){ - return baseLog + Constants.SINGLE_SLASH + - taskExecutionContext.getProcessDefineId() + Constants.SINGLE_SLASH + - taskExecutionContext.getProcessInstanceId() + Constants.SINGLE_SLASH + - taskExecutionContext.getTaskInstanceId() + ".log"; - } - return System.getProperty("user.dir") + Constants.SINGLE_SLASH + - baseLog + Constants.SINGLE_SLASH + - taskExecutionContext.getProcessDefineId() + Constants.SINGLE_SLASH + - taskExecutionContext.getProcessInstanceId() + Constants.SINGLE_SLASH + - taskExecutionContext.getTaskInstanceId() + ".log"; - } - /** * build ack command * @param taskExecutionContext taskExecutionContext @@ -167,10 +158,10 @@ public class TaskExecuteProcessor implements NettyRequestProcessor { private TaskExecuteAckCommand buildAckCommand(TaskExecutionContext taskExecutionContext) { TaskExecuteAckCommand ackCommand = new TaskExecuteAckCommand(); ackCommand.setTaskInstanceId(taskExecutionContext.getTaskInstanceId()); - ackCommand.setStatus(ExecutionStatus.RUNNING_EXECUTION.getCode()); - ackCommand.setLogPath(getTaskLogPath(taskExecutionContext)); + ackCommand.setStatus(taskExecutionContext.getCurrentExecutionStatus().getCode()); + ackCommand.setLogPath(LogUtils.getTaskLogPath(taskExecutionContext)); ackCommand.setHost(taskExecutionContext.getHost()); - ackCommand.setStartTime(new Date()); + ackCommand.setStartTime(taskExecutionContext.getStartTime()); if(taskExecutionContext.getTaskType().equals(TaskType.SQL.name()) || taskExecutionContext.getTaskType().equals(TaskType.PROCEDURE.name())){ ackCommand.setExecutePath(null); }else{ diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/registry/WorkerRegistry.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/registry/WorkerRegistry.java index 5e400e1e1f..36998fad63 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/registry/WorkerRegistry.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/registry/WorkerRegistry.java @@ -16,9 +16,6 @@ */ package org.apache.dolphinscheduler.server.worker.registry; -import static org.apache.dolphinscheduler.common.Constants.DEFAULT_WORKER_GROUP; -import static org.apache.dolphinscheduler.common.Constants.SLASH; - import java.util.Date; import java.util.Set; import java.util.concurrent.Executors; @@ -44,9 +41,11 @@ import org.springframework.stereotype.Service; import com.google.common.collect.Sets; +import static org.apache.dolphinscheduler.common.Constants.*; + /** - * worker registry + * worker registry */ @Service public class WorkerRegistry { @@ -54,13 +53,13 @@ public class WorkerRegistry { private final Logger logger = LoggerFactory.getLogger(WorkerRegistry.class); /** - * zookeeper registry center + * zookeeper registry center */ @Autowired private ZookeeperRegistryCenter zookeeperRegistryCenter; /** - * worker config + * worker config */ @Autowired private WorkerConfig workerConfig; @@ -86,7 +85,7 @@ public class WorkerRegistry { } /** - * registry + * registry */ public void registry() { String address = NetUtils.getHost(); @@ -122,7 +121,7 @@ public class WorkerRegistry { } /** - * remove registry info + * remove registry info */ public void unRegistry() { String address = getLocalAddress(); @@ -135,13 +134,14 @@ public class WorkerRegistry { } /** - * get worker path + * get worker path */ private Set getWorkerZkPaths() { Set workerZkPaths = Sets.newHashSet(); String address = getLocalAddress(); String workerZkPathPrefix = this.zookeeperRegistryCenter.getWorkerPath(); + String weight = getWorkerWeight(); for (String workGroup : this.workerGroups) { StringBuilder workerZkPathBuilder = new StringBuilder(100); @@ -152,15 +152,23 @@ public class WorkerRegistry { // trim and lower case is need workerZkPathBuilder.append(workGroup.trim().toLowerCase()).append(SLASH); workerZkPathBuilder.append(address); + workerZkPathBuilder.append(weight); workerZkPaths.add(workerZkPathBuilder.toString()); } return workerZkPaths; } /** - * get local address + * get local address */ private String getLocalAddress() { return NetUtils.getHost() + ":" + workerConfig.getListenPort(); } + + /** + * get Worker Weight + */ + private String getWorkerWeight() { + return ":" + workerConfig.getWeight(); + } } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java index 26494bc77b..3ba49451c7 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java @@ -16,25 +16,20 @@ */ package org.apache.dolphinscheduler.server.worker.runner; -import java.io.File; -import java.util.Date; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; -import java.util.Set; - -import org.apache.commons.collections.MapUtils; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; +import org.apache.dolphinscheduler.common.enums.TaskType; import org.apache.dolphinscheduler.common.model.TaskNode; import org.apache.dolphinscheduler.common.process.Property; import org.apache.dolphinscheduler.common.task.TaskTimeoutParameter; import org.apache.dolphinscheduler.common.thread.ThreadUtils; -import org.apache.dolphinscheduler.common.utils.CollectionUtils; import org.apache.dolphinscheduler.common.utils.CommonUtils; +import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.common.utils.HadoopUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.RetryerUtils; +import org.apache.dolphinscheduler.remote.command.Command; +import org.apache.dolphinscheduler.remote.command.TaskExecuteAckCommand; import org.apache.dolphinscheduler.remote.command.TaskExecuteResponseCommand; import org.apache.dolphinscheduler.server.entity.TaskExecutionContext; import org.apache.dolphinscheduler.server.worker.cache.TaskExecutionContextCacheManager; @@ -43,9 +38,23 @@ import org.apache.dolphinscheduler.server.worker.processor.TaskCallbackService; import org.apache.dolphinscheduler.server.worker.task.AbstractTask; import org.apache.dolphinscheduler.server.worker.task.TaskManager; import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; + +import org.apache.commons.collections.MapUtils; + +import java.io.File; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.stream.Collectors; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.github.rholder.retry.RetryException; + /** * task scheduler thread @@ -105,6 +114,15 @@ public class TaskExecuteThread implements Runnable { // task node TaskNode taskNode = JSONUtils.parseObject(taskExecutionContext.getTaskJson(), TaskNode.class); + delayExecutionIfNeeded(); + if (taskExecutionContext.getStartTime() == null) { + taskExecutionContext.setStartTime(new Date()); + } + if (taskExecutionContext.getCurrentExecutionStatus() != ExecutionStatus.RUNNING_EXECUTION) { + changeTaskExecutionStatusToRunning(); + } + logger.info("the task begins to execute. task instance id: {}", taskExecutionContext.getTaskInstanceId()); + // copy hdfs/minio file to local downloadResource(taskExecutionContext.getExecutePath(), taskExecutionContext.getResources(), @@ -138,7 +156,7 @@ public class TaskExecuteThread implements Runnable { responseCommand.setProcessId(task.getProcessId()); responseCommand.setAppIds(task.getAppIds()); logger.info("task instance id : {},task final status : {}", taskExecutionContext.getTaskInstanceId(), task.getExitStatus()); - }catch (Exception e){ + } catch (Exception e) { logger.error("task scheduler failure", e); kill(); responseCommand.setStatus(ExecutionStatus.FAILURE.getCode()); @@ -147,9 +165,10 @@ public class TaskExecuteThread implements Runnable { responseCommand.setAppIds(task.getAppIds()); } finally { try { + taskExecutionContext.setCurrentExecutionStatus(ExecutionStatus.of(responseCommand.getStatus())); taskExecutionContextCacheManager.removeByTaskInstanceId(taskExecutionContext.getTaskInstanceId()); taskCallbackService.sendResult(taskExecutionContext.getTaskInstanceId(), responseCommand.convert2Command()); - }catch (Exception e){ + } catch (Exception e) { ThreadUtils.sleep(Constants.SLEEP_TIME_MILLIS); taskCallbackService.sendResult(taskExecutionContext.getTaskInstanceId(), responseCommand.convert2Command()); } @@ -256,4 +275,59 @@ public class TaskExecuteThread implements Runnable { } } } + + /** + * delay execution if needed. + */ + private void delayExecutionIfNeeded() { + long remainTime = DateUtils.getRemainTime(taskExecutionContext.getFirstSubmitTime(), + taskExecutionContext.getDelayTime() * 60L); + logger.info("delay execution time: {} s", remainTime < 0 ? 0 : remainTime); + if (remainTime > 0) { + try { + Thread.sleep(remainTime * Constants.SLEEP_TIME_MILLIS); + } catch (Exception e) { + logger.error("delay task execution failure, the task will be executed directly. process instance id:{}, task instance id:{}", + taskExecutionContext.getProcessInstanceId(), + taskExecutionContext.getTaskInstanceId()); + } + } + } + + /** + * send an ack to change the status of the task. + */ + private void changeTaskExecutionStatusToRunning() { + taskExecutionContext.setCurrentExecutionStatus(ExecutionStatus.RUNNING_EXECUTION); + Command ackCommand = buildAckCommand().convert2Command(); + try { + RetryerUtils.retryCall(() -> { + taskCallbackService.sendAck(taskExecutionContext.getTaskInstanceId(), ackCommand); + return Boolean.TRUE; + }); + } catch (ExecutionException | RetryException e) { + logger.error(e.getMessage(), e); + } + } + + /** + * build ack command. + * + * @return TaskExecuteAckCommand + */ + private TaskExecuteAckCommand buildAckCommand() { + TaskExecuteAckCommand ackCommand = new TaskExecuteAckCommand(); + ackCommand.setTaskInstanceId(taskExecutionContext.getTaskInstanceId()); + ackCommand.setStatus(taskExecutionContext.getCurrentExecutionStatus().getCode()); + ackCommand.setStartTime(taskExecutionContext.getStartTime()); + ackCommand.setLogPath(taskExecutionContext.getLogPath()); + ackCommand.setHost(taskExecutionContext.getHost()); + if (taskExecutionContext.getTaskType().equals(TaskType.SQL.name()) + || taskExecutionContext.getTaskType().equals(TaskType.PROCEDURE.name())) { + ackCommand.setExecutePath(null); + } else { + ackCommand.setExecutePath(taskExecutionContext.getExecutePath()); + } + return ackCommand; + } } \ No newline at end of file diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java index 3459e35b72..3dedeced06 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java @@ -16,12 +16,14 @@ */ package org.apache.dolphinscheduler.server.worker.task; -import com.sun.jna.platform.win32.Kernel32; -import com.sun.jna.platform.win32.WinNT; +import static org.apache.dolphinscheduler.common.Constants.EXIT_CODE_FAILURE; +import static org.apache.dolphinscheduler.common.Constants.EXIT_CODE_SUCCESS; + import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.thread.Stopper; import org.apache.dolphinscheduler.common.thread.ThreadUtils; +import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.common.utils.HadoopUtils; import org.apache.dolphinscheduler.common.utils.LoggerUtils; import org.apache.dolphinscheduler.common.utils.OSUtils; @@ -32,9 +34,12 @@ import org.apache.dolphinscheduler.server.utils.ProcessUtils; import org.apache.dolphinscheduler.server.worker.cache.TaskExecutionContextCacheManager; import org.apache.dolphinscheduler.server.worker.cache.impl.TaskExecutionContextCacheManagerImpl; import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; -import org.slf4j.Logger; -import java.io.*; +import java.io.BufferedReader; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStreamReader; import java.lang.reflect.Field; import java.nio.charset.StandardCharsets; import java.util.ArrayList; @@ -47,8 +52,10 @@ import java.util.function.Consumer; import java.util.regex.Matcher; import java.util.regex.Pattern; -import static org.apache.dolphinscheduler.common.Constants.EXIT_CODE_FAILURE; -import static org.apache.dolphinscheduler.common.Constants.EXIT_CODE_SUCCESS; +import org.slf4j.Logger; + +import com.sun.jna.platform.win32.Kernel32; +import com.sun.jna.platform.win32.WinNT; /** * abstract command executor @@ -305,14 +312,8 @@ public abstract class AbstractCommandExecutor { * @param commands process builder */ private void printCommand(List commands) { - String cmdStr; - - try { - cmdStr = ProcessUtils.buildCommandStr(commands); - logger.info("task run command:\n{}", cmdStr); - } catch (IOException e) { - logger.error(e.getMessage(), e); - } + String cmdStr = ProcessUtils.buildCommandStr(commands); + logger.info("task run command:\n{}", cmdStr); } /** @@ -474,8 +475,7 @@ public abstract class AbstractCommandExecutor { * @return remain time */ private long getRemaintime() { - long usedTime = (System.currentTimeMillis() - taskExecutionContext.getStartTime().getTime()) / 1000; - long remainTime = taskExecutionContext.getTaskTimeout() - usedTime; + long remainTime = DateUtils.getRemainTime(taskExecutionContext.getStartTime(), taskExecutionContext.getTaskTimeout()); if (remainTime < 0) { throw new RuntimeException("task execution time out"); diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java index acc75d70d2..f28f5804b0 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java @@ -147,8 +147,8 @@ public class SqlTask extends AbstractTask { } /** - * ready to execute SQL and parameter entity Map - * @return + * ready to execute SQL and parameter entity Map + * @return SqlBinds */ private SqlBinds getSqlAndSqlParamsMap(String sql) { Map sqlParamsMap = new HashMap<>(); @@ -250,7 +250,7 @@ public class SqlTask extends AbstractTask { * result process * * @param resultSet resultSet - * @throws Exception + * @throws Exception Exception */ private void resultProcess(ResultSet resultSet) throws Exception{ ArrayNode resultJSONArray = JSONUtils.createArrayNode(); @@ -262,7 +262,7 @@ public class SqlTask extends AbstractTask { while (rowCount < LIMIT && resultSet.next()) { ObjectNode mapOfColValues = JSONUtils.createObjectNode(); for (int i = 1; i <= num; i++) { - mapOfColValues.set(md.getColumnName(i), JSONUtils.toJsonNode(resultSet.getObject(i))); + mapOfColValues.set(md.getColumnLabel(i), JSONUtils.toJsonNode(resultSet.getObject(i))); } resultJSONArray.add(mapOfColValues); rowCount++; @@ -293,7 +293,7 @@ public class SqlTask extends AbstractTask { } /** - * post psql + * post sql * * @param connection connection * @param postStatementsBinds postStatementsBinds @@ -329,7 +329,7 @@ public class SqlTask extends AbstractTask { * create connection * * @return connection - * @throws Exception + * @throws Exception Exception */ private Connection createConnection() throws Exception{ // if hive , load connection params if exists @@ -367,7 +367,7 @@ public class SqlTask extends AbstractTask { try { resultSet.close(); } catch (SQLException e) { - + logger.error("close result set error : {}",e.getMessage(),e); } } @@ -375,7 +375,7 @@ public class SqlTask extends AbstractTask { try { pstmt.close(); } catch (SQLException e) { - + logger.error("close prepared statement error : {}",e.getMessage(),e); } } @@ -383,17 +383,17 @@ public class SqlTask extends AbstractTask { try { connection.close(); } catch (SQLException e) { - + logger.error("close connection error : {}",e.getMessage(),e); } } } /** * preparedStatement bind - * @param connection - * @param sqlBinds - * @return - * @throws Exception + * @param connection connection + * @param sqlBinds sqlBinds + * @return PreparedStatement + * @throws Exception Exception */ private PreparedStatement prepareStatementAndBind(Connection connection, SqlBinds sqlBinds) throws Exception { // is the timeout set diff --git a/dolphinscheduler-server/src/main/resources/worker.properties b/dolphinscheduler-server/src/main/resources/worker.properties index 0365c8a9c9..9fba30c147 100644 --- a/dolphinscheduler-server/src/main/resources/worker.properties +++ b/dolphinscheduler-server/src/main/resources/worker.properties @@ -32,3 +32,6 @@ # default worker group #worker.groups=default + +# default worker weight +#work.weight=100 diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/log/LoggerServerTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/log/LoggerServerTest.java index 0da88746f5..d3b1dcf84a 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/log/LoggerServerTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/log/LoggerServerTest.java @@ -17,46 +17,62 @@ package org.apache.dolphinscheduler.server.log; +import java.io.File; +import java.io.IOException; +import java.nio.charset.Charset; + import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.utils.FileUtils; +import org.apache.dolphinscheduler.common.utils.StringUtils; import org.apache.dolphinscheduler.service.log.LogClientService; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; import org.junit.Test; public class LoggerServerTest { + private LoggerServer loggerServer; - @Test - public void testRollViewLog(){ - LoggerServer loggerServer = new LoggerServer(); - loggerServer.start(); + private LogClientService logClientService; - LogClientService logClientService = new LogClientService(); - logClientService.rollViewLog("localhost", Constants.RPC_PORT,"/opt/demo.txt",0,1000); - - try { - Thread.sleep(5000); - } catch (InterruptedException e) { - - } - - loggerServer.stop(); - logClientService.close(); + @Before + public void startServerAndClient() { + this.loggerServer = new LoggerServer(); + this.loggerServer.start(); + this.logClientService = new LogClientService(); } @Test - public void testRemoveTaskLog(){ - LoggerServer loggerServer = new LoggerServer(); - loggerServer.start(); + public void testRollViewLog() throws IOException { + String expectedTmpDemoString = "testRolloViewLog"; + FileUtils.writeStringToFile(new File("/tmp/demo.txt"), expectedTmpDemoString, Charset.defaultCharset()); - LogClientService logClientService = new LogClientService(); - logClientService.removeTaskLog("localhost", Constants.RPC_PORT,"/opt/zhangsan"); + String resultTmpDemoString = this.logClientService.rollViewLog( + "localhost", Constants.RPC_PORT,"/tmp/demo.txt", 0, 1000); - try { - Thread.sleep(5000); - } catch (InterruptedException e) { + Assert.assertEquals(expectedTmpDemoString, resultTmpDemoString.replaceAll("[\r|\n|\t]", StringUtils.EMPTY)); - } + FileUtils.deleteFile("/tmp/demo.txt"); + } - loggerServer.stop(); - logClientService.close(); + @Test + public void testRemoveTaskLog() throws IOException { + String expectedTmpRemoveString = "testRemoveTaskLog"; + FileUtils.writeStringToFile(new File("/tmp/remove.txt"), expectedTmpRemoveString, Charset.defaultCharset()); + + Boolean b = this.logClientService.removeTaskLog("localhost", Constants.RPC_PORT,"/tmp/remove.txt"); + + Assert.assertTrue(b); + + String result = this.logClientService.viewLog("localhost", Constants.RPC_PORT,"/tmp/demo.txt"); + + Assert.assertEquals(StringUtils.EMPTY, result); + } + + @After + public void stopServerAndClient() { + this.loggerServer.stop(); + this.logClientService.close(); } } diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/ConditionsTaskTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/ConditionsTaskTest.java index f1ee8ccf11..61058de864 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/ConditionsTaskTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/ConditionsTaskTest.java @@ -16,14 +16,26 @@ */ package org.apache.dolphinscheduler.server.master; - +import org.apache.dolphinscheduler.common.enums.DependentRelation; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; +import org.apache.dolphinscheduler.common.enums.TaskType; +import org.apache.dolphinscheduler.common.model.DependentItem; +import org.apache.dolphinscheduler.common.model.DependentTaskModel; +import org.apache.dolphinscheduler.common.model.TaskNode; +import org.apache.dolphinscheduler.common.task.conditions.ConditionsParameters; +import org.apache.dolphinscheduler.common.task.dependent.DependentParameters; +import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.server.master.config.MasterConfig; import org.apache.dolphinscheduler.server.master.runner.ConditionsTaskExecThread; import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; import org.apache.dolphinscheduler.service.process.ProcessService; + +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -34,99 +46,144 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.context.ApplicationContext; -import java.util.ArrayList; -import java.util.List; - @RunWith(MockitoJUnitRunner.Silent.class) public class ConditionsTaskTest { - private static final Logger logger = LoggerFactory.getLogger(DependentTaskTest.class); + /** + * TaskNode.runFlag : task can be run normally + */ + public static final String FLOWNODE_RUN_FLAG_NORMAL = "NORMAL"; + private ProcessService processService; - private ApplicationContext applicationContext; - - private MasterConfig config; + private ProcessInstance processInstance; @Before public void before() { - config = new MasterConfig(); - config.setMasterTaskCommitRetryTimes(3); - config.setMasterTaskCommitInterval(1000); - processService = Mockito.mock(ProcessService.class); - applicationContext = Mockito.mock(ApplicationContext.class); + ApplicationContext applicationContext = Mockito.mock(ApplicationContext.class); SpringApplicationContext springApplicationContext = new SpringApplicationContext(); springApplicationContext.setApplicationContext(applicationContext); - Mockito.when(applicationContext.getBean(ProcessService.class)).thenReturn(processService); + + MasterConfig config = new MasterConfig(); Mockito.when(applicationContext.getBean(MasterConfig.class)).thenReturn(config); + config.setMasterTaskCommitRetryTimes(3); + config.setMasterTaskCommitInterval(1000); + processService = Mockito.mock(ProcessService.class); + Mockito.when(applicationContext.getBean(ProcessService.class)).thenReturn(processService); + + processInstance = getProcessInstance(); Mockito.when(processService - .findTaskInstanceById(252612)) - .thenReturn(getTaskInstance()); + .findProcessInstanceById(processInstance.getId())) + .thenReturn(processInstance); + } - Mockito.when(processService.saveTaskInstance(getTaskInstance())) + private TaskInstance testBasicInit(ExecutionStatus expectResult) { + TaskInstance taskInstance = getTaskInstance(getTaskNode(), processInstance); + + // for MasterBaseTaskExecThread.submit + Mockito.when(processService + .submitTask(taskInstance)) + .thenReturn(taskInstance); + // for MasterBaseTaskExecThread.call + Mockito.when(processService + .findTaskInstanceById(taskInstance.getId())) + .thenReturn(taskInstance); + // for ConditionsTaskExecThread.initTaskParameters + Mockito.when(processService + .saveTaskInstance(taskInstance)) + .thenReturn(true); + // for ConditionsTaskExecThread.updateTaskState + Mockito.when(processService + .updateTaskInstance(taskInstance)) .thenReturn(true); - Mockito.when(processService.findProcessInstanceById(10112)) - .thenReturn(getProcessInstance()); - + // for ConditionsTaskExecThread.waitTaskQuit + List conditions = Stream.of( + getTaskInstanceForValidTaskList(1001, "1", expectResult) + ).collect(Collectors.toList()); Mockito.when(processService - .findValidTaskListByProcessId(10112)) - .thenReturn(getTaskInstances()); - } + .findValidTaskListByProcessId(processInstance.getId())) + .thenReturn(conditions); - @Test - public void testCondition(){ - TaskInstance taskInstance = getTaskInstance(); - String dependString = "{\"dependTaskList\":[{\"dependItemList\":[{\"depTasks\":\"1\",\"status\":\"SUCCESS\"}],\"relation\":\"AND\"}],\"relation\":\"AND\"}"; - String conditionResult = "{\"successNode\":[\"2\"],\"failedNode\":[\"3\"]}"; - - taskInstance.setDependency(dependString); - Mockito.when(processService.submitTask(taskInstance)) - .thenReturn(taskInstance); - ConditionsTaskExecThread conditions = - new ConditionsTaskExecThread(taskInstance); - - try { - conditions.call(); - } catch (Exception e) { - e.printStackTrace(); - } - - Assert.assertEquals(ExecutionStatus.SUCCESS, conditions.getTaskInstance().getState()); - } - - - private TaskInstance getTaskInstance(){ - TaskInstance taskInstance = new TaskInstance(); - taskInstance.setId(252612); - taskInstance.setName("C"); - taskInstance.setTaskType("CONDITIONS"); - taskInstance.setProcessInstanceId(10112); - taskInstance.setProcessDefinitionId(100001); return taskInstance; } - - - private List getTaskInstances(){ - List list = new ArrayList<>(); - TaskInstance taskInstance = new TaskInstance(); - taskInstance.setId(199999); - taskInstance.setName("1"); - taskInstance.setState(ExecutionStatus.SUCCESS); - list.add(taskInstance); - return list; + @Test + public void testBasicSuccess() throws Exception { + TaskInstance taskInstance = testBasicInit(ExecutionStatus.SUCCESS); + ConditionsTaskExecThread taskExecThread = new ConditionsTaskExecThread(taskInstance); + taskExecThread.call(); + Assert.assertEquals(ExecutionStatus.SUCCESS, taskExecThread.getTaskInstance().getState()); } - private ProcessInstance getProcessInstance(){ + @Test + public void testBasicFailure() throws Exception { + TaskInstance taskInstance = testBasicInit(ExecutionStatus.FAILURE); + ConditionsTaskExecThread taskExecThread = new ConditionsTaskExecThread(taskInstance); + taskExecThread.call(); + Assert.assertEquals(ExecutionStatus.FAILURE, taskExecThread.getTaskInstance().getState()); + } + + private TaskNode getTaskNode() { + TaskNode taskNode = new TaskNode(); + taskNode.setId("tasks-1000"); + taskNode.setName("C"); + taskNode.setType(TaskType.CONDITIONS.toString()); + taskNode.setRunFlag(FLOWNODE_RUN_FLAG_NORMAL); + + DependentItem dependentItem = new DependentItem(); + dependentItem.setDepTasks("1"); + dependentItem.setStatus(ExecutionStatus.SUCCESS); + + DependentTaskModel dependentTaskModel = new DependentTaskModel(); + dependentTaskModel.setDependItemList(Stream.of(dependentItem).collect(Collectors.toList())); + dependentTaskModel.setRelation(DependentRelation.AND); + + DependentParameters dependentParameters = new DependentParameters(); + dependentParameters.setDependTaskList(Stream.of(dependentTaskModel).collect(Collectors.toList())); + dependentParameters.setRelation(DependentRelation.AND); + + // in: AND(AND(1 is SUCCESS)) + taskNode.setDependence(JSONUtils.toJsonString(dependentParameters)); + + ConditionsParameters conditionsParameters = new ConditionsParameters(); + conditionsParameters.setSuccessNode(Stream.of("2").collect(Collectors.toList())); + conditionsParameters.setFailedNode(Stream.of("3").collect(Collectors.toList())); + + // out: SUCCESS => 2, FAILED => 3 + taskNode.setConditionResult(JSONUtils.toJsonString(conditionsParameters)); + + return taskNode; + } + + private ProcessInstance getProcessInstance() { ProcessInstance processInstance = new ProcessInstance(); - processInstance.setId(10112); - processInstance.setProcessDefinitionId(100001); + processInstance.setId(1000); + processInstance.setProcessDefinitionId(1000); processInstance.setState(ExecutionStatus.RUNNING_EXECUTION); return processInstance; } + private TaskInstance getTaskInstance(TaskNode taskNode, ProcessInstance processInstance) { + TaskInstance taskInstance = new TaskInstance(); + taskInstance.setId(1000); + taskInstance.setTaskJson(JSONUtils.toJsonString(taskNode)); + taskInstance.setName(taskNode.getName()); + taskInstance.setTaskType(taskNode.getType()); + taskInstance.setProcessInstanceId(processInstance.getId()); + taskInstance.setProcessDefinitionId(processInstance.getProcessDefinitionId()); + return taskInstance; + } + + private TaskInstance getTaskInstanceForValidTaskList(int id, String name, ExecutionStatus state) { + TaskInstance taskInstance = new TaskInstance(); + taskInstance.setId(id); + taskInstance.setName(name); + taskInstance.setState(state); + return taskInstance; + } } diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/SubProcessTaskTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/SubProcessTaskTest.java new file mode 100644 index 0000000000..518b11631f --- /dev/null +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/SubProcessTaskTest.java @@ -0,0 +1,159 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.server.master; + +import org.apache.dolphinscheduler.common.enums.ExecutionStatus; +import org.apache.dolphinscheduler.common.enums.TaskType; +import org.apache.dolphinscheduler.common.model.TaskNode; +import org.apache.dolphinscheduler.common.thread.Stopper; +import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.dao.entity.ProcessInstance; +import org.apache.dolphinscheduler.dao.entity.TaskInstance; +import org.apache.dolphinscheduler.server.master.config.MasterConfig; +import org.apache.dolphinscheduler.server.master.runner.SubProcessTaskExecThread; +import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; +import org.apache.dolphinscheduler.service.process.ProcessService; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; +import org.springframework.context.ApplicationContext; + +@RunWith(PowerMockRunner.class) +@PrepareForTest({ + Stopper.class, +}) +public class SubProcessTaskTest { + + /** + * TaskNode.runFlag : task can be run normally + */ + public static final String FLOWNODE_RUN_FLAG_NORMAL = "NORMAL"; + + private ProcessService processService; + + private ProcessInstance processInstance; + + @Before + public void before() { + ApplicationContext applicationContext = Mockito.mock(ApplicationContext.class); + SpringApplicationContext springApplicationContext = new SpringApplicationContext(); + springApplicationContext.setApplicationContext(applicationContext); + + MasterConfig config = new MasterConfig(); + Mockito.when(applicationContext.getBean(MasterConfig.class)).thenReturn(config); + config.setMasterTaskCommitRetryTimes(3); + config.setMasterTaskCommitInterval(1000); + + PowerMockito.mockStatic(Stopper.class); + PowerMockito.when(Stopper.isRunning()).thenReturn(true); + + processService = Mockito.mock(ProcessService.class); + Mockito.when(applicationContext.getBean(ProcessService.class)).thenReturn(processService); + + processInstance = getProcessInstance(); + Mockito.when(processService + .findProcessInstanceById(processInstance.getId())) + .thenReturn(processInstance); + + // for SubProcessTaskExecThread.setTaskInstanceState + Mockito.when(processService + .updateTaskInstance(Mockito.any())) + .thenReturn(true); + + // for MasterBaseTaskExecThread.submit + Mockito.when(processService + .submitTask(Mockito.any())) + .thenAnswer(t -> t.getArgument(0)); + } + + private TaskInstance testBasicInit(ExecutionStatus expectResult) { + TaskInstance taskInstance = getTaskInstance(getTaskNode(), processInstance); + + ProcessInstance subProcessInstance = getSubProcessInstance(expectResult); + // for SubProcessTaskExecThread.waitTaskQuit + Mockito.when(processService + .findProcessInstanceById(subProcessInstance.getId())) + .thenReturn(subProcessInstance); + Mockito.when(processService + .findSubProcessInstance(processInstance.getId(), taskInstance.getId())) + .thenReturn(subProcessInstance); + + return taskInstance; + } + + @Test + public void testBasicSuccess() throws Exception { + TaskInstance taskInstance = testBasicInit(ExecutionStatus.SUCCESS); + SubProcessTaskExecThread taskExecThread = new SubProcessTaskExecThread(taskInstance); + taskExecThread.call(); + Assert.assertEquals(ExecutionStatus.SUCCESS, taskExecThread.getTaskInstance().getState()); + } + + @Test + public void testBasicFailure() throws Exception { + TaskInstance taskInstance = testBasicInit(ExecutionStatus.FAILURE); + SubProcessTaskExecThread taskExecThread = new SubProcessTaskExecThread(taskInstance); + taskExecThread.call(); + Assert.assertEquals(ExecutionStatus.FAILURE, taskExecThread.getTaskInstance().getState()); + } + + private TaskNode getTaskNode() { + TaskNode taskNode = new TaskNode(); + taskNode.setId("tasks-10"); + taskNode.setName("S"); + taskNode.setType(TaskType.SUB_PROCESS.toString()); + taskNode.setRunFlag(FLOWNODE_RUN_FLAG_NORMAL); + return taskNode; + } + + private ProcessInstance getProcessInstance() { + ProcessInstance processInstance = new ProcessInstance(); + processInstance.setId(100); + processInstance.setProcessDefinitionId(1); + processInstance.setState(ExecutionStatus.RUNNING_EXECUTION); + + return processInstance; + } + + private ProcessInstance getSubProcessInstance(ExecutionStatus executionStatus) { + ProcessInstance processInstance = new ProcessInstance(); + processInstance.setId(102); + processInstance.setProcessDefinitionId(2); + processInstance.setState(executionStatus); + + return processInstance; + } + + private TaskInstance getTaskInstance(TaskNode taskNode, ProcessInstance processInstance) { + TaskInstance taskInstance = new TaskInstance(); + taskInstance.setId(1000); + taskInstance.setTaskJson(JSONUtils.toJsonString(taskNode)); + taskInstance.setName(taskNode.getName()); + taskInstance.setTaskType(taskNode.getType()); + taskInstance.setProcessInstanceId(processInstance.getId()); + taskInstance.setProcessDefinitionId(processInstance.getProcessDefinitionId()); + taskInstance.setState(ExecutionStatus.SUBMITTED_SUCCESS); + return taskInstance; + } +} diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumerTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumerTest.java index 997c129573..32881b5681 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumerTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumerTest.java @@ -55,7 +55,8 @@ import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; @RunWith(SpringJUnit4ClassRunner.class) @ContextConfiguration(classes={DependencyConfig.class, SpringApplicationContext.class, SpringZKServer.class, CuratorZookeeperClient.class, NettyExecutorManager.class, ExecutorDispatcher.class, ZookeeperRegistryCenter.class, TaskPriorityQueueConsumer.class, - ZookeeperNodeManager.class, ZookeeperCachedOperator.class, ZookeeperConfig.class, MasterConfig.class}) + ZookeeperNodeManager.class, ZookeeperCachedOperator.class, ZookeeperConfig.class, MasterConfig.class, + CuratorZookeeperClient.class}) public class TaskPriorityQueueConsumerTest { diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/ExecutorDispatcherTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/ExecutorDispatcherTest.java index 98231bee06..de1f0517bd 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/ExecutorDispatcherTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/ExecutorDispatcherTest.java @@ -31,6 +31,7 @@ import org.apache.dolphinscheduler.server.worker.processor.TaskExecuteProcessor; import org.apache.dolphinscheduler.server.worker.registry.WorkerRegistry; import org.apache.dolphinscheduler.server.zk.SpringZKServer; import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; +import org.apache.dolphinscheduler.service.zk.CuratorZookeeperClient; import org.apache.dolphinscheduler.service.zk.ZookeeperCachedOperator; import org.apache.dolphinscheduler.service.zk.ZookeeperConfig; import org.junit.Test; @@ -46,7 +47,7 @@ import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; @RunWith(SpringJUnit4ClassRunner.class) @ContextConfiguration(classes={DependencyConfig.class, SpringApplicationContext.class, SpringZKServer.class, WorkerRegistry.class, NettyExecutorManager.class, ExecutorDispatcher.class, ZookeeperRegistryCenter.class, WorkerConfig.class, - ZookeeperNodeManager.class, ZookeeperCachedOperator.class, ZookeeperConfig.class}) + ZookeeperNodeManager.class, ZookeeperCachedOperator.class, ZookeeperConfig.class, CuratorZookeeperClient.class}) public class ExecutorDispatcherTest { @Autowired diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/executor/NettyExecutorManagerTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/executor/NettyExecutorManagerTest.java index a1c6b71437..f7d98baed1 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/executor/NettyExecutorManagerTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/executor/NettyExecutorManagerTest.java @@ -37,6 +37,7 @@ import org.apache.dolphinscheduler.server.worker.processor.TaskExecuteProcessor; import org.apache.dolphinscheduler.server.worker.registry.WorkerRegistry; import org.apache.dolphinscheduler.server.zk.SpringZKServer; import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; +import org.apache.dolphinscheduler.service.zk.CuratorZookeeperClient; import org.apache.dolphinscheduler.service.zk.ZookeeperCachedOperator; import org.apache.dolphinscheduler.service.zk.ZookeeperConfig; import org.junit.Assert; @@ -52,7 +53,7 @@ import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; */ @RunWith(SpringJUnit4ClassRunner.class) @ContextConfiguration(classes={DependencyConfig.class, SpringZKServer.class, WorkerRegistry.class, - ZookeeperNodeManager.class, ZookeeperRegistryCenter.class, WorkerConfig.class, + ZookeeperNodeManager.class, ZookeeperRegistryCenter.class, WorkerConfig.class, CuratorZookeeperClient.class, ZookeeperCachedOperator.class, ZookeeperConfig.class, SpringApplicationContext.class, NettyExecutorManager.class}) public class NettyExecutorManagerTest { diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/RoundRobinHostManagerTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/RoundRobinHostManagerTest.java index c38c9d4ae8..ab2cc12e20 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/RoundRobinHostManagerTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/RoundRobinHostManagerTest.java @@ -14,66 +14,53 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.server.master.dispatch.host; +import com.google.common.collect.Sets; -import org.apache.dolphinscheduler.common.utils.NetUtils; import org.apache.dolphinscheduler.common.utils.StringUtils; import org.apache.dolphinscheduler.remote.utils.Host; import org.apache.dolphinscheduler.server.master.dispatch.context.ExecutionContext; -import org.apache.dolphinscheduler.server.registry.DependencyConfig; import org.apache.dolphinscheduler.server.registry.ZookeeperNodeManager; -import org.apache.dolphinscheduler.server.registry.ZookeeperRegistryCenter; import org.apache.dolphinscheduler.server.utils.ExecutionContextTestUtils; -import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; -import org.apache.dolphinscheduler.server.worker.registry.WorkerRegistry; -import org.apache.dolphinscheduler.server.zk.SpringZKServer; -import org.apache.dolphinscheduler.service.zk.ZookeeperCachedOperator; -import org.apache.dolphinscheduler.service.zk.ZookeeperConfig; + import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit4.SpringRunner; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; /** * round robin host manager test */ -@RunWith(SpringRunner.class) -@ContextConfiguration(classes={DependencyConfig.class, SpringZKServer.class, WorkerRegistry.class, ZookeeperRegistryCenter.class, WorkerConfig.class, - ZookeeperNodeManager.class, ZookeeperCachedOperator.class, ZookeeperConfig.class}) +@RunWith(MockitoJUnitRunner.class) public class RoundRobinHostManagerTest { - @Autowired + @Mock private ZookeeperNodeManager zookeeperNodeManager; - @Autowired - private WorkerRegistry workerRegistry; - - @Autowired - private WorkerConfig workerConfig; + @InjectMocks + RoundRobinHostManager roundRobinHostManager; @Test - public void testSelectWithEmptyResult(){ - RoundRobinHostManager roundRobinHostManager = new RoundRobinHostManager(); - roundRobinHostManager.setZookeeperNodeManager(zookeeperNodeManager); + public void testSelectWithEmptyResult() { + Mockito.when(zookeeperNodeManager.getWorkerGroupNodes("default")).thenReturn(null); ExecutionContext context = ExecutionContextTestUtils.getExecutionContext(10000); Host emptyHost = roundRobinHostManager.select(context); Assert.assertTrue(StringUtils.isEmpty(emptyHost.getAddress())); } @Test - public void testSelectWithResult(){ - workerRegistry.registry(); - RoundRobinHostManager roundRobinHostManager = new RoundRobinHostManager(); - roundRobinHostManager.setZookeeperNodeManager(zookeeperNodeManager); + public void testSelectWithResult() { + Mockito.when(zookeeperNodeManager.getWorkerGroupNodes("default")).thenReturn(Sets.newHashSet("192.168.1.1:22:100")); ExecutionContext context = ExecutionContextTestUtils.getExecutionContext(10000); Host host = roundRobinHostManager.select(context); Assert.assertTrue(StringUtils.isNotEmpty(host.getAddress())); - Assert.assertTrue(host.getAddress().equalsIgnoreCase(NetUtils.getHost() + ":" + workerConfig.getListenPort())); - workerRegistry.unRegistry(); + Assert.assertTrue(host.getAddress().equalsIgnoreCase("192.168.1.1:22")); } } diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/RandomSelectorTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/RandomSelectorTest.java index a14ea32e4e..f25a227947 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/RandomSelectorTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/RandomSelectorTest.java @@ -16,7 +16,9 @@ */ package org.apache.dolphinscheduler.server.master.dispatch.host.assign; +import org.apache.commons.lang.ObjectUtils; import org.apache.dolphinscheduler.common.utils.StringUtils; +import org.apache.dolphinscheduler.remote.utils.Host; import org.junit.Assert; import org.junit.Test; @@ -36,16 +38,16 @@ public class RandomSelectorTest { @Test public void testSelect1(){ - RandomSelector selector = new RandomSelector(); - String result = selector.select(Arrays.asList("1")); - Assert.assertTrue(StringUtils.isNotEmpty(result)); - Assert.assertTrue(result.equalsIgnoreCase("1")); + RandomSelector selector = new RandomSelector(); + Host result = selector.select(Arrays.asList(new Host("192.168.1.1",80,100),new Host("192.168.1.2",80,20))); + Assert.assertNotNull(result); } @Test public void testSelect(){ - RandomSelector selector = new RandomSelector(); - int result = selector.select(Arrays.asList(1,2,3,4,5,6,7)); - Assert.assertTrue(result >= 1 && result <= 7); + RandomSelector selector = new RandomSelector(); + Host result = selector.select(Arrays.asList(new Host("192.168.1.1",80,100),new Host("192.168.1.1",80,20))); + Assert.assertNotNull(result); + } } diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/RoundRobinSelectorTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/RoundRobinSelectorTest.java index adc55a4774..ed62caaa2c 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/RoundRobinSelectorTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/RoundRobinSelectorTest.java @@ -17,6 +17,7 @@ package org.apache.dolphinscheduler.server.master.dispatch.host.assign; import org.apache.dolphinscheduler.common.utils.StringUtils; +import org.apache.dolphinscheduler.remote.utils.Host; import org.junit.Assert; import org.junit.Test; @@ -30,26 +31,46 @@ import java.util.List; public class RoundRobinSelectorTest { @Test(expected = IllegalArgumentException.class) - public void testSelectWithIllegalArgumentException(){ + public void testSelectWithIllegalArgumentException() { RoundRobinSelector selector = new RoundRobinSelector(); selector.select(Collections.EMPTY_LIST); } @Test - public void testSelect1(){ - RoundRobinSelector selector = new RoundRobinSelector(); - String result = selector.select(Arrays.asList("1")); - Assert.assertTrue(StringUtils.isNotEmpty(result)); - Assert.assertTrue(result.equalsIgnoreCase("1")); + public void testSelect1() { + RoundRobinSelector selector = new RoundRobinSelector(); + Host result = null; + result = selector.select(Arrays.asList(new Host("192.168.1.1", 80, 20, "kris"), new Host("192.168.1.2", 80, 10, "kris"))); + Assert.assertEquals("192.168.1.1", result.getIp()); + result = selector.select(Arrays.asList(new Host("192.168.1.1", 80, 20, "kris"), new Host("192.168.1.2", 80, 10, "kris"))); + Assert.assertEquals("192.168.1.2", result.getIp()); + result = selector.select(Arrays.asList(new Host("192.168.1.1", 80, 20, "kris"), new Host("192.168.1.2", 80, 10, "kris"))); + Assert.assertEquals("192.168.1.1", result.getIp()); + // add new host + result = selector.select(Arrays.asList(new Host("192.168.1.1", 80, 20, "kris"), new Host("192.168.1.2", 80, 10, "kris"))); + Assert.assertEquals("192.168.1.1", result.getIp()); + result = selector.select(Arrays.asList(new Host("192.168.1.1", 80, 20, "kris"), new Host("192.168.1.2", 80, 10, "kris"))); + Assert.assertEquals("192.168.1.2", result.getIp()); + result = selector.select(Arrays.asList(new Host("192.168.1.1", 80, 20, "kris"), new Host("192.168.1.2", 80, 10, "kris"), new Host("192.168.1.3", 80, 10, "kris"))); + Assert.assertEquals("192.168.1.1",result.getIp()); + result = selector.select(Arrays.asList(new Host("192.168.1.1", 80, 20, "kris"), new Host("192.168.1.2", 80, 10, "kris"), new Host("192.168.1.3", 80, 10, "kris"))); + Assert.assertEquals("192.168.1.3",result.getIp()); + result = selector.select(Arrays.asList(new Host("192.168.1.1", 80, 20, "kris"), new Host("192.168.1.2", 80, 10, "kris"), new Host("192.168.1.3", 80, 10, "kris"))); + Assert.assertEquals("192.168.1.1",result.getIp()); + result = selector.select(Arrays.asList(new Host("192.168.1.1", 80, 20, "kris"), new Host("192.168.1.2", 80, 10, "kris"), new Host("192.168.1.3", 80, 10, "kris"))); + Assert.assertEquals("192.168.1.2",result.getIp()); + result = selector.select(Arrays.asList(new Host("192.168.1.1", 80, 20, "kris"), new Host("192.168.1.2", 80, 10, "kris"), new Host("192.168.1.3", 80, 10, "kris"))); + Assert.assertEquals("192.168.1.1",result.getIp()); + result = selector.select(Arrays.asList(new Host("192.168.1.1", 80, 20, "kris"), new Host("192.168.1.2", 80, 10, "kris"), new Host("192.168.1.3", 80, 10, "kris"))); + Assert.assertEquals("192.168.1.3",result.getIp()); + // remove host3 + result = selector.select(Arrays.asList(new Host("192.168.1.1", 80, 20, "kris"), new Host("192.168.1.2", 80, 10, "kris"))); + Assert.assertEquals("192.168.1.1",result.getIp()); + result = selector.select(Arrays.asList(new Host("192.168.1.1", 80, 20, "kris"), new Host("192.168.1.2", 80, 10, "kris"))); + Assert.assertEquals("192.168.1.2",result.getIp()); + result = selector.select(Arrays.asList(new Host("192.168.1.1", 80, 20, "kris"), new Host("192.168.1.2", 80, 10, "kris"))); + Assert.assertEquals("192.168.1.1",result.getIp()); + } - @Test - public void testSelect(){ - RoundRobinSelector selector = new RoundRobinSelector(); - List sources = Arrays.asList(1, 2, 3, 4, 5, 6, 7); - int result = selector.select(sources); - Assert.assertTrue(result == 1); - int result2 = selector.select(Arrays.asList(1,2,3,4,5,6,7)); - Assert.assertTrue(result2 == 2); - } } diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskResponseServiceTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskResponseServiceTest.java index f19e2b4b64..a2b1b4ecc2 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskResponseServiceTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskResponseServiceTest.java @@ -22,6 +22,7 @@ import org.apache.dolphinscheduler.server.registry.DependencyConfig; import org.apache.dolphinscheduler.server.registry.ZookeeperNodeManager; import org.apache.dolphinscheduler.server.registry.ZookeeperRegistryCenter; import org.apache.dolphinscheduler.server.zk.SpringZKServer; +import org.apache.dolphinscheduler.service.zk.CuratorZookeeperClient; import org.apache.dolphinscheduler.service.zk.ZookeeperCachedOperator; import org.apache.dolphinscheduler.service.zk.ZookeeperConfig; import org.junit.Assert; @@ -35,7 +36,8 @@ import java.util.Date; @RunWith(SpringJUnit4ClassRunner.class) @ContextConfiguration(classes={DependencyConfig.class, SpringZKServer.class, TaskResponseService.class, ZookeeperRegistryCenter.class, - ZookeeperCachedOperator.class, ZookeeperConfig.class, ZookeeperNodeManager.class, TaskResponseService.class}) + ZookeeperCachedOperator.class, ZookeeperConfig.class, ZookeeperNodeManager.class, TaskResponseService.class, + CuratorZookeeperClient.class}) public class TaskResponseServiceTest { @Autowired diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryTest.java index 18c8b496d7..7763e07314 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryTest.java @@ -17,12 +17,19 @@ package org.apache.dolphinscheduler.server.master.registry; +import static org.apache.dolphinscheduler.common.Constants.HEARTBEAT_FOR_ZOOKEEPER_INFO_LENGTH; + import org.apache.dolphinscheduler.remote.utils.Constants; import org.apache.dolphinscheduler.server.master.config.MasterConfig; import org.apache.dolphinscheduler.server.registry.ZookeeperRegistryCenter; import org.apache.dolphinscheduler.server.zk.SpringZKServer; +import org.apache.dolphinscheduler.service.zk.CuratorZookeeperClient; import org.apache.dolphinscheduler.service.zk.ZookeeperCachedOperator; import org.apache.dolphinscheduler.service.zk.ZookeeperConfig; + +import java.util.List; +import java.util.concurrent.TimeUnit; + import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; @@ -30,15 +37,12 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringRunner; -import java.util.List; -import java.util.concurrent.TimeUnit; - -import static org.apache.dolphinscheduler.common.Constants.HEARTBEAT_FOR_ZOOKEEPER_INFO_LENGTH; /** * master registry test */ @RunWith(SpringRunner.class) -@ContextConfiguration(classes={SpringZKServer.class, MasterRegistry.class,ZookeeperRegistryCenter.class, MasterConfig.class, ZookeeperCachedOperator.class, ZookeeperConfig.class}) +@ContextConfiguration(classes = {SpringZKServer.class, MasterRegistry.class, ZookeeperRegistryCenter.class, + MasterConfig.class, ZookeeperCachedOperator.class, ZookeeperConfig.class, CuratorZookeeperClient.class}) public class MasterRegistryTest { @Autowired @@ -63,6 +67,7 @@ public class MasterRegistryTest { @Test public void testUnRegistry() throws InterruptedException { + masterRegistry.init(); masterRegistry.registry(); TimeUnit.SECONDS.sleep(masterConfig.getMasterHeartbeatInterval() + 2); //wait heartbeat info write into zk node masterRegistry.unRegistry(); diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThreadTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThreadTest.java index f29691e9bb..f6fdfaab63 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThreadTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThreadTest.java @@ -17,14 +17,15 @@ package org.apache.dolphinscheduler.server.master.runner; -import java.util.HashSet; -import java.util.Set; - import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.server.registry.ZookeeperRegistryCenter; import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; import org.apache.dolphinscheduler.service.process.ProcessService; + +import java.util.HashSet; +import java.util.Set; + import org.junit.Assert; import org.junit.Before; import org.junit.Test; diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/LogUtilsTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/LogUtilsTest.java new file mode 100644 index 0000000000..02cca399ed --- /dev/null +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/LogUtilsTest.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.server.utils; + +import org.apache.dolphinscheduler.dao.entity.TaskInstance; +import org.apache.dolphinscheduler.server.log.TaskLogDiscriminator; + +import java.nio.file.Path; +import java.nio.file.Paths; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; +import org.slf4j.LoggerFactory; + +import ch.qos.logback.classic.Logger; +import ch.qos.logback.classic.sift.SiftingAppender; + +@RunWith(MockitoJUnitRunner.class) +public class LogUtilsTest { + + @Test + public void testGetTaskLogPath() { + TaskInstance taskInstance = new TaskInstance(); + taskInstance.setProcessDefinitionId(1); + taskInstance.setProcessInstanceId(100); + taskInstance.setId(1000); + + Logger rootLogger = (Logger) LoggerFactory.getILoggerFactory().getLogger("ROOT"); + Assert.assertNotNull(rootLogger); + + SiftingAppender appender = Mockito.mock(SiftingAppender.class); + // it's a trick to mock logger.getAppend("TASKLOGFILE") + Mockito.when(appender.getName()).thenReturn("TASKLOGFILE"); + rootLogger.addAppender(appender); + + Path logBase = Paths.get("path").resolve("to").resolve("test"); + + TaskLogDiscriminator taskLogDiscriminator = Mockito.mock(TaskLogDiscriminator.class); + Mockito.when(taskLogDiscriminator.getLogBase()).thenReturn(logBase.toString()); + Mockito.when(appender.getDiscriminator()).thenReturn(taskLogDiscriminator); + + Path logPath = Paths.get(".").toAbsolutePath().getParent() + .resolve(logBase) + .resolve("1").resolve("100").resolve("1000.log"); + Assert.assertEquals(logPath.toString(), LogUtils.getTaskLogPath(taskInstance)); + } + +} diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/ProcessUtilsTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/ProcessUtilsTest.java index 1e0adaad9b..ace5cd8471 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/ProcessUtilsTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/ProcessUtilsTest.java @@ -40,11 +40,8 @@ public class ProcessUtilsTest { public void testBuildCommandStr() { List commands = new ArrayList<>(); commands.add("sudo"); - try { - Assert.assertEquals(ProcessUtils.buildCommandStr(commands), "sudo"); - } catch (IOException e) { - Assert.fail(e.getMessage()); - } + Assert.assertEquals(ProcessUtils.buildCommandStr(commands), "sudo"); + } } diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThreadTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThreadTest.java new file mode 100644 index 0000000000..2e7e531f30 --- /dev/null +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThreadTest.java @@ -0,0 +1,171 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.server.worker.runner; + +import org.apache.dolphinscheduler.common.enums.ExecutionStatus; +import org.apache.dolphinscheduler.common.model.TaskNode; +import org.apache.dolphinscheduler.common.task.AbstractParameters; +import org.apache.dolphinscheduler.common.utils.CommonUtils; +import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.LoggerUtils; +import org.apache.dolphinscheduler.remote.command.Command; +import org.apache.dolphinscheduler.remote.command.TaskExecuteAckCommand; +import org.apache.dolphinscheduler.remote.command.TaskExecuteResponseCommand; +import org.apache.dolphinscheduler.server.entity.TaskExecutionContext; +import org.apache.dolphinscheduler.server.worker.cache.impl.TaskExecutionContextCacheManagerImpl; +import org.apache.dolphinscheduler.server.worker.processor.TaskCallbackService; +import org.apache.dolphinscheduler.server.worker.task.AbstractTask; +import org.apache.dolphinscheduler.server.worker.task.TaskManager; +import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; + +import java.util.Date; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * test task execute thread. + */ +@RunWith(PowerMockRunner.class) +@PrepareForTest({TaskManager.class, JSONUtils.class, CommonUtils.class, SpringApplicationContext.class}) +public class TaskExecuteThreadTest { + + private TaskExecutionContext taskExecutionContext; + + private TaskCallbackService taskCallbackService; + + private Command ackCommand; + + private Command responseCommand; + + private Logger taskLogger; + + private TaskExecutionContextCacheManagerImpl taskExecutionContextCacheManager; + + @Before + public void before() { + // init task execution context, logger + taskExecutionContext = new TaskExecutionContext(); + taskExecutionContext.setProcessId(12345); + taskExecutionContext.setProcessDefineId(1); + taskExecutionContext.setProcessInstanceId(1); + taskExecutionContext.setTaskInstanceId(1); + taskExecutionContext.setTaskType(""); + taskExecutionContext.setFirstSubmitTime(new Date()); + taskExecutionContext.setDelayTime(0); + taskExecutionContext.setLogPath("/tmp/test.log"); + taskExecutionContext.setHost("localhost"); + taskExecutionContext.setExecutePath("/tmp/dolphinscheduler/exec/process/1/2/3/4"); + + ackCommand = new TaskExecuteAckCommand().convert2Command(); + responseCommand = new TaskExecuteResponseCommand(taskExecutionContext.getTaskInstanceId()).convert2Command(); + + taskLogger = LoggerFactory.getLogger(LoggerUtils.buildTaskId( + LoggerUtils.TASK_LOGGER_INFO_PREFIX, + taskExecutionContext.getProcessDefineId(), + taskExecutionContext.getProcessInstanceId(), + taskExecutionContext.getTaskInstanceId() + )); + + taskExecutionContextCacheManager = new TaskExecutionContextCacheManagerImpl(); + taskExecutionContextCacheManager.cacheTaskExecutionContext(taskExecutionContext); + + taskCallbackService = PowerMockito.mock(TaskCallbackService.class); + PowerMockito.doNothing().when(taskCallbackService).sendAck(taskExecutionContext.getTaskInstanceId(), ackCommand); + PowerMockito.doNothing().when(taskCallbackService).sendResult(taskExecutionContext.getTaskInstanceId(), responseCommand); + + PowerMockito.mockStatic(SpringApplicationContext.class); + PowerMockito.when(SpringApplicationContext.getBean(TaskExecutionContextCacheManagerImpl.class)) + .thenReturn(taskExecutionContextCacheManager); + + PowerMockito.mockStatic(TaskManager.class); + PowerMockito.when(TaskManager.newTask(taskExecutionContext, taskLogger)) + .thenReturn(new SimpleTask(taskExecutionContext, taskLogger)); + + PowerMockito.mockStatic(JSONUtils.class); + PowerMockito.when(JSONUtils.parseObject(taskExecutionContext.getTaskJson(), TaskNode.class)) + .thenReturn(new TaskNode()); + + PowerMockito.mockStatic(CommonUtils.class); + PowerMockito.when(CommonUtils.getSystemEnvPath()).thenReturn("/user_home/.bash_profile"); + } + + @Test + public void testNormalExecution() { + taskExecutionContext.setTaskType("SQL"); + taskExecutionContext.setStartTime(new Date()); + taskExecutionContext.setCurrentExecutionStatus(ExecutionStatus.RUNNING_EXECUTION); + TaskExecuteThread taskExecuteThread = new TaskExecuteThread(taskExecutionContext, taskCallbackService, taskLogger); + taskExecuteThread.run(); + + Assert.assertEquals(ExecutionStatus.SUCCESS, taskExecutionContext.getCurrentExecutionStatus()); + } + + @Test + public void testDelayExecution() { + taskExecutionContext.setTaskType("PYTHON"); + taskExecutionContext.setStartTime(null); + taskExecutionContext.setDelayTime(1); + taskExecutionContext.setCurrentExecutionStatus(ExecutionStatus.DELAY_EXECUTION); + TaskExecuteThread taskExecuteThread = new TaskExecuteThread(taskExecutionContext, taskCallbackService, taskLogger); + taskExecuteThread.run(); + + Assert.assertEquals(ExecutionStatus.SUCCESS, taskExecutionContext.getCurrentExecutionStatus()); + } + + private class SimpleTask extends AbstractTask { + + protected SimpleTask(TaskExecutionContext taskExecutionContext, Logger logger) { + super(taskExecutionContext, logger); + // pid + this.processId = taskExecutionContext.getProcessId(); + } + + @Override + public AbstractParameters getParameters() { + return null; + } + + @Override + public void init() { + + } + + @Override + public void handle() throws Exception { + + } + + @Override + public void after() { + + } + + @Override + public ExecutionStatus getExitStatus() { + return ExecutionStatus.SUCCESS; + } + } +} diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/log/LogClientService.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/log/LogClientService.java index 92d38d470a..474bf12c77 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/log/LogClientService.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/log/LogClientService.java @@ -16,13 +16,20 @@ */ package org.apache.dolphinscheduler.service.log; -import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.remote.NettyRemotingClient; import org.apache.dolphinscheduler.remote.command.Command; -import org.apache.dolphinscheduler.remote.command.log.*; +import org.apache.dolphinscheduler.remote.command.log.GetLogBytesRequestCommand; +import org.apache.dolphinscheduler.remote.command.log.GetLogBytesResponseCommand; +import org.apache.dolphinscheduler.remote.command.log.RemoveTaskLogRequestCommand; +import org.apache.dolphinscheduler.remote.command.log.RemoveTaskLogResponseCommand; +import org.apache.dolphinscheduler.remote.command.log.RollViewLogRequestCommand; +import org.apache.dolphinscheduler.remote.command.log.RollViewLogResponseCommand; +import org.apache.dolphinscheduler.remote.command.log.ViewLogRequestCommand; +import org.apache.dolphinscheduler.remote.command.log.ViewLogResponseCommand; import org.apache.dolphinscheduler.remote.config.NettyClientConfig; import org.apache.dolphinscheduler.remote.utils.Host; import org.apache.dolphinscheduler.remote.utils.JsonSerializer; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -38,8 +45,10 @@ public class LogClientService { private final NettyRemotingClient client; + private volatile boolean isRunning; + /** - * request time out + * request time out */ private static final long LOG_REQUEST_TIMEOUT = 10 * 1000L; @@ -50,18 +59,21 @@ public class LogClientService { this.clientConfig = new NettyClientConfig(); this.clientConfig.setWorkerThreads(4); this.client = new NettyRemotingClient(clientConfig); + this.isRunning = true; } /** * close */ - public void close() { + public void close() { this.client.close(); + this.isRunning = false; logger.info("logger client closed"); } /** * roll view log + * * @param host host * @param port port * @param path path @@ -69,7 +81,7 @@ public class LogClientService { * @param limit limit * @return log content */ - public String rollViewLog(String host, int port, String path,int skipLineNum,int limit) { + public String rollViewLog(String host, int port, String path, int skipLineNum, int limit) { logger.info("roll view log, host : {}, port : {}, path {}, skipLineNum {} ,limit {}", host, port, path, skipLineNum, limit); RollViewLogRequestCommand request = new RollViewLogRequestCommand(path, skipLineNum, limit); String result = ""; @@ -77,7 +89,7 @@ public class LogClientService { try { Command command = request.convert2Command(); Command response = this.client.sendSync(address, command, LOG_REQUEST_TIMEOUT); - if(response != null){ + if (response != null) { RollViewLogResponseCommand rollReviewLog = JsonSerializer.deserialize( response.getBody(), RollViewLogResponseCommand.class); return rollReviewLog.getMsg(); @@ -92,6 +104,7 @@ public class LogClientService { /** * view log + * * @param host host * @param port port * @param path path @@ -105,7 +118,7 @@ public class LogClientService { try { Command command = request.convert2Command(); Command response = this.client.sendSync(address, command, LOG_REQUEST_TIMEOUT); - if(response != null){ + if (response != null) { ViewLogResponseCommand viewLog = JsonSerializer.deserialize( response.getBody(), ViewLogResponseCommand.class); return viewLog.getMsg(); @@ -120,6 +133,7 @@ public class LogClientService { /** * get log size + * * @param host host * @param port port * @param path log path @@ -133,7 +147,7 @@ public class LogClientService { try { Command command = request.convert2Command(); Command response = this.client.sendSync(address, command, LOG_REQUEST_TIMEOUT); - if(response != null){ + if (response != null) { GetLogBytesResponseCommand getLog = JsonSerializer.deserialize( response.getBody(), GetLogBytesResponseCommand.class); return getLog.getData(); @@ -149,6 +163,7 @@ public class LogClientService { /** * remove task log + * * @param host host * @param port port * @param path path @@ -162,7 +177,7 @@ public class LogClientService { try { Command command = request.convert2Command(); Command response = this.client.sendSync(address, command, LOG_REQUEST_TIMEOUT); - if(response != null){ + if (response != null) { RemoveTaskLogResponseCommand taskLogResponse = JsonSerializer.deserialize( response.getBody(), RemoveTaskLogResponseCommand.class); return taskLogResponse.getStatus(); @@ -174,4 +189,8 @@ public class LogClientService { } return result; } + + public boolean isRunning() { + return isRunning; + } } \ No newline at end of file diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java index 7df56c6cdd..6f642672cd 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java @@ -16,31 +16,89 @@ */ package org.apache.dolphinscheduler.service.process; -import com.cronutils.model.Cron; -import com.fasterxml.jackson.databind.node.ObjectNode; -import org.apache.commons.lang.ArrayUtils; +import static org.apache.dolphinscheduler.common.Constants.CMDPARAM_COMPLEMENT_DATA_END_DATE; +import static org.apache.dolphinscheduler.common.Constants.CMDPARAM_COMPLEMENT_DATA_START_DATE; +import static org.apache.dolphinscheduler.common.Constants.CMDPARAM_EMPTY_SUB_PROCESS; +import static org.apache.dolphinscheduler.common.Constants.CMDPARAM_RECOVER_PROCESS_ID_STRING; +import static org.apache.dolphinscheduler.common.Constants.CMDPARAM_SUB_PROCESS; +import static org.apache.dolphinscheduler.common.Constants.CMDPARAM_SUB_PROCESS_DEFINE_ID; +import static org.apache.dolphinscheduler.common.Constants.CMDPARAM_SUB_PROCESS_PARENT_INSTANCE_ID; +import static org.apache.dolphinscheduler.common.Constants.YYYY_MM_DD_HH_MM_SS; + +import static java.util.stream.Collectors.toSet; + import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.common.enums.*; +import org.apache.dolphinscheduler.common.enums.AuthorizationType; +import org.apache.dolphinscheduler.common.enums.CommandType; +import org.apache.dolphinscheduler.common.enums.CycleEnum; +import org.apache.dolphinscheduler.common.enums.ExecutionStatus; +import org.apache.dolphinscheduler.common.enums.FailureStrategy; +import org.apache.dolphinscheduler.common.enums.Flag; +import org.apache.dolphinscheduler.common.enums.ResourceType; +import org.apache.dolphinscheduler.common.enums.TaskDependType; +import org.apache.dolphinscheduler.common.enums.WarningType; import org.apache.dolphinscheduler.common.model.DateInterval; import org.apache.dolphinscheduler.common.model.TaskNode; import org.apache.dolphinscheduler.common.process.Property; import org.apache.dolphinscheduler.common.task.subprocess.SubProcessParameters; -import org.apache.dolphinscheduler.common.utils.*; -import org.apache.dolphinscheduler.dao.entity.*; -import org.apache.dolphinscheduler.dao.mapper.*; +import org.apache.dolphinscheduler.common.utils.CollectionUtils; +import org.apache.dolphinscheduler.common.utils.DateUtils; +import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.ParameterUtils; +import org.apache.dolphinscheduler.common.utils.StringUtils; +import org.apache.dolphinscheduler.dao.entity.Command; +import org.apache.dolphinscheduler.dao.entity.CycleDependency; +import org.apache.dolphinscheduler.dao.entity.DataSource; +import org.apache.dolphinscheduler.dao.entity.ErrorCommand; +import org.apache.dolphinscheduler.dao.entity.ProcessData; +import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; +import org.apache.dolphinscheduler.dao.entity.ProcessInstance; +import org.apache.dolphinscheduler.dao.entity.ProcessInstanceMap; +import org.apache.dolphinscheduler.dao.entity.Project; +import org.apache.dolphinscheduler.dao.entity.Resource; +import org.apache.dolphinscheduler.dao.entity.Schedule; +import org.apache.dolphinscheduler.dao.entity.TaskInstance; +import org.apache.dolphinscheduler.dao.entity.Tenant; +import org.apache.dolphinscheduler.dao.entity.UdfFunc; +import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.dao.mapper.CommandMapper; +import org.apache.dolphinscheduler.dao.mapper.DataSourceMapper; +import org.apache.dolphinscheduler.dao.mapper.ErrorCommandMapper; +import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper; +import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapMapper; +import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper; +import org.apache.dolphinscheduler.dao.mapper.ProjectMapper; +import org.apache.dolphinscheduler.dao.mapper.ResourceMapper; +import org.apache.dolphinscheduler.dao.mapper.ScheduleMapper; +import org.apache.dolphinscheduler.dao.mapper.TaskInstanceMapper; +import org.apache.dolphinscheduler.dao.mapper.TenantMapper; +import org.apache.dolphinscheduler.dao.mapper.UdfFuncMapper; +import org.apache.dolphinscheduler.dao.mapper.UserMapper; import org.apache.dolphinscheduler.remote.utils.Host; import org.apache.dolphinscheduler.service.log.LogClientService; import org.apache.dolphinscheduler.service.quartz.cron.CronUtils; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Calendar; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; + import org.quartz.CronExpression; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import org.springframework.transaction.annotation.Transactional; -import java.util.*; -import java.util.stream.Collectors; -import static java.util.stream.Collectors.toSet; -import static org.apache.dolphinscheduler.common.Constants.*; + +import com.cronutils.model.Cron; +import com.fasterxml.jackson.databind.node.ObjectNode; /** * process relative dao that some mappers in this. @@ -52,6 +110,7 @@ public class ProcessService { private final int[] stateArray = new int[]{ExecutionStatus.SUBMITTED_SUCCESS.ordinal(), ExecutionStatus.RUNNING_EXECUTION.ordinal(), + ExecutionStatus.DELAY_EXECUTION.ordinal(), ExecutionStatus.READY_PAUSE.ordinal(), ExecutionStatus.READY_STOP.ordinal()}; @@ -1001,8 +1060,9 @@ public class ProcessService { if(taskInstance.getState() != ExecutionStatus.NEED_FAULT_TOLERANCE){ taskInstance.setRetryTimes(taskInstance.getRetryTimes() + 1 ); } + taskInstance.setSubmitTime(null); + taskInstance.setStartTime(null); taskInstance.setEndTime(null); - taskInstance.setStartTime(new Date()); taskInstance.setFlag(Flag.YES); taskInstance.setHost(null); taskInstance.setId(0); @@ -1012,7 +1072,12 @@ public class ProcessService { taskInstance.setExecutorId(processInstance.getExecutorId()); taskInstance.setProcessInstancePriority(processInstance.getProcessInstancePriority()); taskInstance.setState(getSubmitTaskState(taskInstance, processInstanceState)); - taskInstance.setSubmitTime(new Date()); + if (taskInstance.getSubmitTime() == null) { + taskInstance.setSubmitTime(new Date()); + } + if (taskInstance.getFirstSubmitTime() == null) { + taskInstance.setFirstSubmitTime(taskInstance.getSubmitTime()); + } boolean saveResult = saveTaskInstance(taskInstance); if(!saveResult){ return null; @@ -1062,10 +1127,11 @@ public class ProcessService { public ExecutionStatus getSubmitTaskState(TaskInstance taskInstance, ExecutionStatus processInstanceState){ ExecutionStatus state = taskInstance.getState(); if( - // running or killed + // running, delayed or killed // the task already exists in task queue // return state state == ExecutionStatus.RUNNING_EXECUTION + || state == ExecutionStatus.DELAY_EXECUTION || state == ExecutionStatus.KILL || checkTaskExistsInTaskQueue(taskInstance) ){ @@ -1588,7 +1654,7 @@ public class ProcessService { */ public List getCycleDependencies(int masterId,int[] ids,Date scheduledFireTime) throws Exception { List cycleDependencyList = new ArrayList(); - if(ArrayUtils.isEmpty(ids)){ + if (ids == null || ids.length == 0) { logger.warn("ids[] is empty!is invalid!"); return cycleDependencyList; } @@ -1772,7 +1838,7 @@ public class ProcessService { public List listUnauthorized(int userId,T[] needChecks,AuthorizationType authorizationType){ List resultList = new ArrayList(); - if (!ArrayUtils.isEmpty(needChecks)) { + if (Objects.nonNull(needChecks) && needChecks.length > 0) { Set originResSet = new HashSet(Arrays.asList(needChecks)); switch (authorizationType){ diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/QuartzExecutors.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/QuartzExecutors.java index 01d7b2f1e5..3b15810e05 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/QuartzExecutors.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/QuartzExecutors.java @@ -61,20 +61,20 @@ public class QuartzExecutors { */ private static Scheduler scheduler; - /** - * instance of QuartzExecutors - */ - private static volatile QuartzExecutors INSTANCE = null; - /** * load conf */ private static Configuration conf; + private static final class Holder { + private static final QuartzExecutors instance = new QuartzExecutors(); + } + private QuartzExecutors() { try { conf = new PropertiesConfiguration(QUARTZ_PROPERTIES_PATH); + init(); }catch (ConfigurationException e){ logger.warn("not loaded quartz configuration file, will used default value",e); } @@ -85,18 +85,7 @@ public class QuartzExecutors { * @return instance of Quartz Executors */ public static QuartzExecutors getInstance() { - if (INSTANCE == null) { - synchronized (QuartzExecutors.class) { - // when more than two threads run into the first null check same time, to avoid instanced more than one time, it needs to be checked again. - if (INSTANCE == null) { - QuartzExecutors quartzExecutors = new QuartzExecutors(); - //finish QuartzExecutors init - quartzExecutors.init(); - INSTANCE = quartzExecutors; - } - } - } - return INSTANCE; + return Holder.instance; } diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/config.js b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/config.js index a4960f7ac5..2e60929577 100755 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/config.js +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/config.js @@ -229,6 +229,13 @@ const tasksState = { color: '#5101be', icoUnicode: 'ans-icon-dependence', isSpin: false + }, + DELAY_EXECUTION: { + id: 12, + desc: `${i18n.$t('Delay execution')}`, + color: '#5102ce', + icoUnicode: 'ans-icon-coin', + isSpin: false } } diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue index d12d0f7d57..1880adc51b 100755 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue @@ -103,7 +103,7 @@ {{spinnerLoading ? 'Loading...' : $t('Save')}} + + {{spinnerLoading ? 'Loading...' : $t('Version Info')}} +

@@ -147,6 +158,7 @@ import { findComponentDownward } from '@/module/util/' import disabledState from '@/module/mixin/disabledState' import { mapActions, mapState, mapMutations } from 'vuex' + import mVersions from '../../projects/pages/definition/pages/list/_source/versions' let eventModel @@ -176,7 +188,7 @@ releaseState: String }, methods: { - ...mapActions('dag', ['saveDAGchart', 'updateInstance', 'updateDefinition', 'getTaskState']), + ...mapActions('dag', ['saveDAGchart', 'updateInstance', 'updateDefinition', 'getTaskState', 'switchProcessDefinitionVersion', 'getProcessDefinitionVersionsPage', 'deleteProcessDefinitionVersion']), ...mapMutations('dag', ['addTasks', 'cacheTasks', 'resetParams', 'setIsEditDag', 'setName', 'addConnects']), // DAG automatic layout @@ -196,7 +208,6 @@ ], Connector: 'Bezier', PaintStyle: { lineWidth: 2, stroke: '#456' }, // Connection style - HoverPaintStyle: {stroke: '#ccc', strokeWidth: 3}, ConnectionOverlays: [ [ 'Arrow', @@ -370,6 +381,12 @@ this[this.type === 'instance' ? 'updateInstance' : 'updateDefinition'](this.urlParam.id).then(res => { this.$message.success(res.msg) this.spinnerLoading = false + // Jump process definition + if (this.type === 'instance') { + this.$router.push({ path: `/projects/instance/list/${this.urlParam.id}?_t=${new Date().getTime()}` }) + } else { + this.$router.push({ path: `/projects/definition/list/${this.urlParam.id}?_t=${new Date().getTime()}` }) + } resolve() }).catch(e => { this.$message.error(e.msg || '') @@ -657,6 +674,136 @@ if(eventModel && this.taskId == $id){ eventModel.remove() } + }, + + /** + * query the process definition pagination version + */ + _version (item) { + let self = this + this.getProcessDefinitionVersionsPage({ + pageNo: 1, + pageSize: 10, + processDefinitionId: this.urlParam.id + }).then(res => { + let processDefinitionVersions = res.data.lists + let total = res.data.totalCount + let pageSize = res.data.pageSize + let pageNo = res.data.currentPage + if (this.versionsModel) { + this.versionsModel.remove() + } + this.versionsModel = this.$drawer({ + direction: 'right', + closable: true, + showMask: true, + escClose: true, + render (h) { + return h(mVersions, { + on: { + /** + * switch version in process definition version list + * + * @param version the version user want to change + * @param processDefinitionId the process definition id + * @param fromThis fromThis + */ + mVersionSwitchProcessDefinitionVersion ({ version, processDefinitionId, fromThis }) { + + self.$store.state.dag.isSwitchVersion = true + + self.switchProcessDefinitionVersion({ + version: version, + processDefinitionId: processDefinitionId + }).then(res => { + self.$message.success($t('Switch Version Successfully')) + setTimeout(() => { + fromThis.$destroy() + self.versionsModel.remove() + }, 0) + self.$router.push({ path: `/projects/definition/list/${processDefinitionId}?_t=${new Date().getTime()}` }) + }).catch(e => { + self.$store.state.dag.isSwitchVersion = false + self.$message.error(e.msg || '') + }) + }, + + /** + * Paging event of process definition versions + * + * @param pageNo page number + * @param pageSize page size + * @param processDefinitionId the process definition id of page version + * @param fromThis fromThis + */ + mVersionGetProcessDefinitionVersionsPage ({ pageNo, pageSize, processDefinitionId, fromThis }) { + self.getProcessDefinitionVersionsPage({ + pageNo: pageNo, + pageSize: pageSize, + processDefinitionId: processDefinitionId + }).then(res => { + fromThis.processDefinitionVersions = res.data.lists + fromThis.total = res.data.totalCount + fromThis.pageSize = res.data.pageSize + fromThis.pageNo = res.data.currentPage + }).catch(e => { + self.$message.error(e.msg || '') + }) + }, + + /** + * delete one version of process definition + * + * @param version the version need to delete + * @param processDefinitionId the process definition id user want to delete + * @param fromThis fromThis + */ + mVersionDeleteProcessDefinitionVersion ({ version, processDefinitionId, fromThis }) { + self.deleteProcessDefinitionVersion({ + version: version, + processDefinitionId: processDefinitionId + }).then(res => { + self.$message.success(res.msg || '') + fromThis.$emit('mVersionGetProcessDefinitionVersionsPage', { + pageNo: 1, + pageSize: 10, + processDefinitionId: processDefinitionId, + fromThis: fromThis + }) + }).catch(e => { + self.$message.error(e.msg || '') + }) + }, + + /** + * remove this drawer + * + * @param fromThis + */ + close ({ fromThis }) { + setTimeout(() => { + fromThis.$destroy() + self.versionsModel.remove() + }, 0) + } + }, + props: { + processDefinition: { + id: self.urlParam.id, + version: self.$store.state.dag.version, + state: self.releaseState + }, + processDefinitionVersions: processDefinitionVersions, + total: total, + pageNo: pageNo, + pageSize: pageSize + } + }) + } + }) + }).catch(e => { + this.$message.error(e.msg || '') + }) } }, watch: { @@ -685,7 +832,6 @@ ], Connector: 'Bezier', PaintStyle: { lineWidth: 2, stroke: '#456' }, // Connection style - HoverPaintStyle: {stroke: '#ccc', strokeWidth: 3}, ConnectionOverlays: [ [ 'Arrow', diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue index 6f07f97f02..8444863aea 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue @@ -109,6 +109,20 @@ ({{$t('Minute')}})
+ + +
+
+ {{$t('Delay execution time')}} +
+
+ + + ({{$t('Minute')}}) +
+
+ +
{{$t('State')}} @@ -127,7 +141,6 @@
-
{{$t('State')}} @@ -339,6 +352,8 @@ maxRetryTimes: '0', // Failure retry interval retryInterval: '1', + // Delay execution time + delayTime: '0', // Task timeout alarm timeout: {}, // Task priority @@ -466,6 +481,7 @@ dependence: this.cacheDependence, maxRetryTimes: this.maxRetryTimes, retryInterval: this.retryInterval, + delayTime: this.delayTime, timeout: this.timeout, taskInstancePriority: this.taskInstancePriority, workerGroup: this.workerGroup, @@ -544,6 +560,7 @@ dependence: this.dependence, maxRetryTimes: this.maxRetryTimes, retryInterval: this.retryInterval, + delayTime: this.delayTime, timeout: this.timeout, taskInstancePriority: this.taskInstancePriority, workerGroup: this.workerGroup, @@ -634,6 +651,7 @@ this.description = o.description this.maxRetryTimes = o.maxRetryTimes this.retryInterval = o.retryInterval + this.delayTime = o.delayTime if(o.conditionResult) { this.successBranch = o.conditionResult.successNode[0] this.failedBranch = o.conditionResult.failedNode[0] @@ -699,6 +717,7 @@ dependence: this.cacheDependence, maxRetryTimes: this.maxRetryTimes, retryInterval: this.retryInterval, + delayTime: this.delayTime, timeout: this.timeout, taskInstancePriority: this.taskInstancePriority, workerGroup: this.workerGroup, diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/flink.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/flink.vue index d1488df6d0..87d9146fa8 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/flink.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/flink.vue @@ -48,7 +48,7 @@
{{$t('Main jar package')}}
- +
{{ node.raw.fullName }}
@@ -158,7 +158,7 @@
{{$t('Resources')}}
- +
{{ node.raw.fullName }}
diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/mr.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/mr.vue index d05c6983d6..a53501d2b5 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/mr.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/mr.vue @@ -44,7 +44,7 @@
{{$t('Main jar package')}}
- +
{{ node.raw.fullName }}
@@ -78,7 +78,7 @@
{{$t('Resources')}}
- +
{{ node.raw.fullName }}
diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/python.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/python.vue index 30b9f7f79c..c6c2a22a62 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/python.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/python.vue @@ -31,7 +31,7 @@
{{$t('Resources')}}
- +
{{ node.raw.fullName }}
 + @@ -138,6 +139,8 @@
@@ -146,8 +149,10 @@ import _ from 'lodash' import mStart from './start' import mTiming from './timing' + import mRelatedItems from './relatedItems' import { mapActions } from 'vuex' import { publishStatus } from '@/conf/home/pages/dag/_source/config' + import mVersions from './versions' export default { name: 'definition-list', @@ -164,7 +169,7 @@ pageSize: Number }, methods: { - ...mapActions('dag', ['editProcessState', 'getStartCheck', 'getReceiver', 'deleteDefinition', 'batchDeleteDefinition','exportDefinition','copyProcess']), + ...mapActions('dag', ['editProcessState', 'getStartCheck', 'getReceiver', 'deleteDefinition', 'batchDeleteDefinition', 'exportDefinition', 'getProcessDefinitionVersionsPage', 'copyProcess', 'switchProcessDefinitionVersion', 'deleteProcessDefinitionVersion', 'moveProcess']), ...mapActions('security', ['getWorkerGroupsAll']), _rtPublishStatus (code) { return _.filter(publishStatus, v => v.code === code)[0].desc @@ -315,8 +320,27 @@ */ _copyProcess (item) { this.copyProcess({ - processId: item.id + processDefinitionIds: item.id, + targetProjectId: item.projectId }).then(res => { + this.strSelectIds = '' + this.$message.success(res.msg) + $('body').find('.tooltip.fade.top.in').remove() + this._onUpdate() + }).catch(e => { + this.$message.error(e.msg || '') + }) + }, + + /** + * move + */ + _moveProcess (item) { + this.moveProcess({ + processDefinitionIds: item.id, + targetProjectId: item.projectId + }).then(res => { + this.strSelectIds = '' this.$message.success(res.msg) $('body').find('.tooltip.fade.top.in').remove() this._onUpdate() @@ -334,6 +358,125 @@ }) }, + _version (item) { + let self = this + this.getProcessDefinitionVersionsPage({ + pageNo: 1, + pageSize: 10, + processDefinitionId: item.id + }).then(res => { + let processDefinitionVersions = res.data.lists + let total = res.data.totalCount + let pageSize = res.data.pageSize + let pageNo = res.data.currentPage + if (this.versionsModel) { + this.versionsModel.remove() + } + this.versionsModel = this.$drawer({ + direction: 'right', + closable: true, + showMask: true, + escClose: true, + render (h) { + return h(mVersions, { + on: { + /** + * switch version in process definition version list + * + * @param version the version user want to change + * @param processDefinitionId the process definition id + * @param fromThis fromThis + */ + mVersionSwitchProcessDefinitionVersion ({ version, processDefinitionId, fromThis }) { + self.switchProcessDefinitionVersion({ + version: version, + processDefinitionId: processDefinitionId + }).then(res => { + self.$message.success($t('Switch Version Successfully')) + setTimeout(() => { + fromThis.$destroy() + self.versionsModel.remove() + }, 0) + self.$router.push({ path: `/projects/definition/list/${processDefinitionId}` }) + }).catch(e => { + self.$message.error(e.msg || '') + }) + }, + + /** + * Paging event of process definition versions + * + * @param pageNo page number + * @param pageSize page size + * @param processDefinitionId the process definition id of page version + * @param fromThis fromThis + */ + mVersionGetProcessDefinitionVersionsPage ({ pageNo, pageSize, processDefinitionId, fromThis }) { + self.getProcessDefinitionVersionsPage({ + pageNo: pageNo, + pageSize: pageSize, + processDefinitionId: processDefinitionId + }).then(res => { + fromThis.processDefinitionVersions = res.data.lists + fromThis.total = res.data.totalCount + fromThis.pageSize = res.data.pageSize + fromThis.pageNo = res.data.currentPage + }).catch(e => { + self.$message.error(e.msg || '') + }) + }, + + /** + * delete one version of process definition + * + * @param version the version need to delete + * @param processDefinitionId the process definition id user want to delete + * @param fromThis fromThis + */ + mVersionDeleteProcessDefinitionVersion ({ version, processDefinitionId, fromThis }) { + self.deleteProcessDefinitionVersion({ + version: version, + processDefinitionId: processDefinitionId + }).then(res => { + self.$message.success(res.msg || '') + fromThis.$emit('mVersionGetProcessDefinitionVersionsPage', { + pageNo: 1, + pageSize: 10, + processDefinitionId: processDefinitionId, + fromThis: fromThis + }) + }).catch(e => { + self.$message.error(e.msg || '') + }) + }, + + /** + * remove this drawer + * + * @param fromThis + */ + close ({ fromThis }) { + setTimeout(() => { + fromThis.$destroy() + self.versionsModel.remove() + }, 0) + } + }, + props: { + processDefinition: item, + processDefinitionVersions: processDefinitionVersions, + total: total, + pageNo: pageNo, + pageSize: pageSize + } + }) + } + }) + }).catch(e => { + this.$message.error(e.msg || '') + }) + }, + _batchExport () { this.exportDefinition({ processDefinitionIds: this.strSelectIds, @@ -348,7 +491,64 @@ this.$message.error(e.msg) }) }, - + /** + * Batch Copy + */ + _batchCopy () { + let self = this + let modal = this.$modal.dialog({ + closable: false, + showMask: true, + escClose: true, + className: 'v-modal-custom', + transitionName: 'opacityp', + render (h) { + return h(mRelatedItems, { + on: { + onBatchCopy (item) { + self._copyProcess({id: self.strSelectIds,projectId: item}) + modal.remove() + }, + close () { + modal.remove() + } + }, + props: { + tmp: false + } + }) + } + }) + }, + /** + * _batchMove + */ + _batchMove() { + let self = this + let modal = this.$modal.dialog({ + closable: false, + showMask: true, + escClose: true, + className: 'v-modal-custom', + transitionName: 'opacityp', + render (h) { + return h(mRelatedItems, { + on: { + onBatchMove (item) { + self._moveProcess({id: self.strSelectIds,projectId: item}) + modal.remove() + }, + close () { + modal.remove() + } + }, + props: { + tmp: true + } + }) + } + }) + }, /** * Edit state */ @@ -423,6 +623,6 @@ }, mounted () { }, - components: { } + components: { mVersions } } diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/relatedItems.vue b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/relatedItems.vue new file mode 100644 index 0000000000..9090bbd771 --- /dev/null +++ b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/relatedItems.vue @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + \ No newline at end of file diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/versions.vue b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/versions.vue new file mode 100644 index 0000000000..fcc3410080 --- /dev/null +++ b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/versions.vue @@ -0,0 +1,260 @@ +/* +* Licensed to the Apache Software Foundation (ASF) under one or more +* contributor license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright ownership. +* The ASF licenses this file to You under the Apache License, Version 2.0 +* (the "License"); you may not use this file except in compliance with +* the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + + + + + diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/index/_source/taskStatusCount.vue b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/index/_source/taskStatusCount.vue new file mode 100644 index 0000000000..90ae53f4c9 --- /dev/null +++ b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/index/_source/taskStatusCount.vue @@ -0,0 +1,147 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + + + diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/index/index.vue b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/index/index.vue index 2b5cbbc017..7ca6e3a0f6 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/index/index.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/index/index.vue @@ -35,8 +35,8 @@ {{$t('Task status statistics')}}
- - + +
@@ -68,7 +68,7 @@ import dayjs from 'dayjs' import mDefineUserCount from './_source/defineUserCount' import mCommandStateCount from './_source/commandStateCount' - import mTaskCtatusCount from './_source/taskCtatusCount' + import mTaskStatusCount from './_source/taskStatusCount' import mProcessStateCount from './_source/processStateCount' import mQueueCount from './_source/queueCount' import localStore from '@/module/util/localStorage' @@ -105,7 +105,7 @@ mListConstruction, mDefineUserCount, mCommandStateCount, - mTaskCtatusCount, + mTaskStatusCount, mProcessStateCount, mQueueCount } diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/udf/pages/function/_source/createUdf.vue b/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/udf/pages/function/_source/createUdf.vue index beaa7d7139..7aa2711e54 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/udf/pages/function/_source/createUdf.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/udf/pages/function/_source/createUdf.vue @@ -72,7 +72,7 @@