From d284e44d52f75bd6c5f5756d9b7ae4a5af24fadc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=99=BE=E5=B2=81?= Date: Fri, 1 Oct 2021 14:54:04 +0800 Subject: [PATCH] Rename tis to pigeon from backend layer (#6437) * add mock getTaskLogName() for UT test pass * make task-tis rename to task-pigeon * rename module name 'tis' to pigeon * rename module name 'tis' to pigeon,make sanity check pass * remove execute console.log() --- .../common/enums/TaskType.java | 4 +- ...eters.java => PigeonCommonParameters.java} | 4 +- .../common/utils/TaskParametersUtils.java | 6 +- .../common/utils/TaskParametersUtilsTest.java | 2 +- .../src/main/provisio/dolphinscheduler.xml | 4 +- .../consumer/TaskPriorityQueueConsumer.java | 2 +- .../pom.xml | 4 +- .../readme.md | 0 .../plugin/task/pigeon/PigeonConfig.java} | 26 +++---- .../plugin/task/pigeon/PigeonParameters.java} | 8 +-- .../task/pigeon/PigeonParamsConstants.java} | 6 +- .../plugin/task/pigeon/PigeonTask.java} | 66 +++++++++--------- .../task/pigeon/PigeonTaskChannel.java} | 13 ++-- .../pigeon/PigeonTaskChannelFactory.java} | 14 ++-- .../plugin/task/pigeon/PigeonTaskPlugin.java} | 6 +- .../plugin/task/pigeon}/config.properties | 0 .../plugin/task/pigeon/PigeonTaskTest.java} | 27 ++++--- .../plugin/task/pigeon/PigeonTaskTest.json} | 4 +- dolphinscheduler-task-plugin/pom.xml | 2 +- .../dag/_source/images/task-icos/tis.png | Bin 1361 -> 0 bytes .../_source/images/task-icos/tis_hover.png | Bin 1445 -> 0 bytes pom.xml | 2 +- 22 files changed, 101 insertions(+), 99 deletions(-) rename dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/tis/{TISCommonParameters.java => PigeonCommonParameters.java} (94%) rename dolphinscheduler-task-plugin/{dolphinscheduler-task-tis => dolphinscheduler-task-pigeon}/pom.xml (96%) rename dolphinscheduler-task-plugin/{dolphinscheduler-task-tis => dolphinscheduler-task-pigeon}/readme.md (100%) rename dolphinscheduler-task-plugin/{dolphinscheduler-task-tis/src/main/java/org/apache/dolphinscheduler/plugin/task/tis/TISConfig.java => dolphinscheduler-task-pigeon/src/main/java/org/apache/dolphinscheduler/plugin/task/pigeon/PigeonConfig.java} (78%) rename dolphinscheduler-task-plugin/{dolphinscheduler-task-tis/src/main/java/org/apache/dolphinscheduler/plugin/task/tis/TISParameters.java => dolphinscheduler-task-pigeon/src/main/java/org/apache/dolphinscheduler/plugin/task/pigeon/PigeonParameters.java} (91%) rename dolphinscheduler-task-plugin/{dolphinscheduler-task-tis/src/main/java/org/apache/dolphinscheduler/plugin/task/tis/TISParamsConstants.java => dolphinscheduler-task-pigeon/src/main/java/org/apache/dolphinscheduler/plugin/task/pigeon/PigeonParamsConstants.java} (87%) rename dolphinscheduler-task-plugin/{dolphinscheduler-task-tis/src/main/java/org/apache/dolphinscheduler/plugin/task/tis/TISTask.java => dolphinscheduler-task-pigeon/src/main/java/org/apache/dolphinscheduler/plugin/task/pigeon/PigeonTask.java} (84%) rename dolphinscheduler-task-plugin/{dolphinscheduler-task-tis/src/main/java/org/apache/dolphinscheduler/plugin/task/tis/TISTaskChannel.java => dolphinscheduler-task-pigeon/src/main/java/org/apache/dolphinscheduler/plugin/task/pigeon/PigeonTaskChannel.java} (76%) rename dolphinscheduler-task-plugin/{dolphinscheduler-task-tis/src/main/java/org/apache/dolphinscheduler/plugin/task/tis/TISTaskChannelFactory.java => dolphinscheduler-task-pigeon/src/main/java/org/apache/dolphinscheduler/plugin/task/pigeon/PigeonTaskChannelFactory.java} (81%) rename dolphinscheduler-task-plugin/{dolphinscheduler-task-tis/src/main/java/org/apache/dolphinscheduler/plugin/task/tis/TISTaskPlugin.java => dolphinscheduler-task-pigeon/src/main/java/org/apache/dolphinscheduler/plugin/task/pigeon/PigeonTaskPlugin.java} (85%) rename dolphinscheduler-task-plugin/{dolphinscheduler-task-tis/src/main/resources/org/apache/dolphinscheduler/plugin/task/tis => dolphinscheduler-task-pigeon/src/main/resources/org/apache/dolphinscheduler/plugin/task/pigeon}/config.properties (100%) rename dolphinscheduler-task-plugin/{dolphinscheduler-task-tis/src/test/java/org/apache/dolphinscheduler/plugin/task/tis/TISTaskTest.java => dolphinscheduler-task-pigeon/src/test/java/org/apache/dolphinscheduler/plugin/task/pigeon/PigeonTaskTest.java} (88%) rename dolphinscheduler-task-plugin/{dolphinscheduler-task-tis/src/test/resources/org/apache/dolphinscheduler/plugin/task/tis/TISTaskTest.json => dolphinscheduler-task-pigeon/src/test/resources/org/apache/dolphinscheduler/plugin/task/pigeon/PigeonTaskTest.json} (94%) delete mode 100755 dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/images/task-icos/tis.png delete mode 100755 dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/images/task-icos/tis_hover.png diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/TaskType.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/TaskType.java index 7882bac404..d5338cad4c 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/TaskType.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/TaskType.java @@ -37,6 +37,7 @@ public enum TaskType { * 11 CONDITIONS * 12 SQOOP * 13 WATERDROP + * 15 PIGEON */ SHELL(0, "SHELL"), SQL(1, "SQL"), @@ -53,8 +54,7 @@ public enum TaskType { SQOOP(12, "SQOOP"), WATERDROP(13, "WATERDROP"), SWITCH(14, "SWITCH"), - TIS(15, "TIS"), - ; + PIGEON(15, "PIGEON"); TaskType(int code, String desc) { this.code = code; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/tis/TISCommonParameters.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/tis/PigeonCommonParameters.java similarity index 94% rename from dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/tis/TISCommonParameters.java rename to dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/tis/PigeonCommonParameters.java index aebbdd0746..6606cc13e9 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/tis/TISCommonParameters.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/tis/PigeonCommonParameters.java @@ -30,9 +30,9 @@ import org.slf4j.LoggerFactory; /** * TIS parameter */ -public class TISCommonParameters extends AbstractParameters { +public class PigeonCommonParameters extends AbstractParameters { - private static final Logger logger = LoggerFactory.getLogger(TISCommonParameters.class); + private static final Logger logger = LoggerFactory.getLogger(PigeonCommonParameters.class); /** * TIS target job name */ diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/TaskParametersUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/TaskParametersUtils.java index e4f990d052..781b83b828 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/TaskParametersUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/TaskParametersUtils.java @@ -32,7 +32,7 @@ import org.apache.dolphinscheduler.common.task.sql.SqlParameters; import org.apache.dolphinscheduler.common.task.sqoop.SqoopParameters; import org.apache.dolphinscheduler.common.task.subprocess.SubProcessParameters; import org.apache.dolphinscheduler.common.task.switchtask.SwitchParameters; -import org.apache.dolphinscheduler.common.task.tis.TISCommonParameters; +import org.apache.dolphinscheduler.common.task.tis.PigeonCommonParameters; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -86,8 +86,8 @@ public class TaskParametersUtils { return JSONUtils.parseObject(parameter, SqoopParameters.class); case "SWITCH": return JSONUtils.parseObject(parameter, SwitchParameters.class); - case "TIS": - return JSONUtils.parseObject(parameter, TISCommonParameters.class); + case "PIGEON": + return JSONUtils.parseObject(parameter, PigeonCommonParameters.class); default: logger.error("not support task type: {}", taskType); return null; diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/TaskParametersUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/TaskParametersUtilsTest.java index 47fe1acfa5..ad9f5f2342 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/TaskParametersUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/TaskParametersUtilsTest.java @@ -41,6 +41,6 @@ public class TaskParametersUtilsTest { Assert.assertNotNull(TaskParametersUtils.getParameters(TaskType.DEPENDENT.getDesc(), "{}")); Assert.assertNotNull(TaskParametersUtils.getParameters(TaskType.FLINK.getDesc(), "{}")); Assert.assertNotNull(TaskParametersUtils.getParameters(TaskType.HTTP.getDesc(), "{}")); - Assert.assertNotNull(TaskParametersUtils.getParameters(TaskType.TIS.getDesc(), "{}")); + Assert.assertNotNull(TaskParametersUtils.getParameters(TaskType.PIGEON.getDesc(), "{}")); } } diff --git a/dolphinscheduler-dist/src/main/provisio/dolphinscheduler.xml b/dolphinscheduler-dist/src/main/provisio/dolphinscheduler.xml index fc480acef1..2ed6a3fc58 100644 --- a/dolphinscheduler-dist/src/main/provisio/dolphinscheduler.xml +++ b/dolphinscheduler-dist/src/main/provisio/dolphinscheduler.xml @@ -106,8 +106,8 @@ - - + + diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java index 3ccb827b5c..7b18e2bc71 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java @@ -137,7 +137,7 @@ public class TaskPriorityQueueConsumer extends Thread { result = dispatcher.dispatch(executionContext); } } catch (ExecuteException e) { - logger.error("dispatch error: {}", e.getMessage()); + logger.error("dispatch error: {}", e.getMessage(),e); } return result; } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-tis/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-pigeon/pom.xml similarity index 96% rename from dolphinscheduler-task-plugin/dolphinscheduler-task-tis/pom.xml rename to dolphinscheduler-task-plugin/dolphinscheduler-task-pigeon/pom.xml index 71fc55294d..4a77c5a91e 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-tis/pom.xml +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-pigeon/pom.xml @@ -26,7 +26,7 @@ 4.0.0 - dolphinscheduler-task-tis + dolphinscheduler-task-pigeon dolphinscheduler-plugin @@ -107,7 +107,7 @@ - dolphinscheduler-task-tis-${project.version} + dolphinscheduler-task-pigeon-${project.version} \ No newline at end of file diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-tis/readme.md b/dolphinscheduler-task-plugin/dolphinscheduler-task-pigeon/readme.md similarity index 100% rename from dolphinscheduler-task-plugin/dolphinscheduler-task-tis/readme.md rename to dolphinscheduler-task-plugin/dolphinscheduler-task-pigeon/readme.md diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-tis/src/main/java/org/apache/dolphinscheduler/plugin/task/tis/TISConfig.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-pigeon/src/main/java/org/apache/dolphinscheduler/plugin/task/pigeon/PigeonConfig.java similarity index 78% rename from dolphinscheduler-task-plugin/dolphinscheduler-task-tis/src/main/java/org/apache/dolphinscheduler/plugin/task/tis/TISConfig.java rename to dolphinscheduler-task-plugin/dolphinscheduler-task-pigeon/src/main/java/org/apache/dolphinscheduler/plugin/task/pigeon/PigeonConfig.java index e6a00f0453..a8a70ce0a5 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-tis/src/main/java/org/apache/dolphinscheduler/plugin/task/tis/TISConfig.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-pigeon/src/main/java/org/apache/dolphinscheduler/plugin/task/pigeon/PigeonConfig.java @@ -15,15 +15,15 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.plugin.task.tis; +package org.apache.dolphinscheduler.plugin.task.pigeon; import org.apache.dolphinscheduler.spi.utils.StringUtils; import java.util.ResourceBundle; -public class TISConfig { +public class PigeonConfig { - private static TISConfig cfg; + private static PigeonConfig cfg; private final String jobTriggerUrl; private final String jobTriggerPostBody; @@ -33,15 +33,15 @@ public class TISConfig { private final String jobLogsFetchUrl; private final String jobCancelPostBody; - public static synchronized TISConfig getInstance() { + public static synchronized PigeonConfig getInstance() { if (cfg == null) { - cfg = new TISConfig(); + cfg = new PigeonConfig(); } return cfg; } - private TISConfig() { - ResourceBundle bundle = ResourceBundle.getBundle(TISConfig.class.getPackage().getName().replace(".", "/") + "/config"); + private PigeonConfig() { + ResourceBundle bundle = ResourceBundle.getBundle(PigeonConfig.class.getPackage().getName().replace(".", "/") + "/config"); this.jobTriggerUrl = bundle.getString("job.trigger.url"); this.jobStatusUrl = bundle.getString("job.status.url"); this.jobTriggerPostBody = bundle.getString("job.trigger.post.body"); @@ -55,7 +55,7 @@ public class TISConfig { } public String getJobTriggerUrl(String tisHost) { - checkTisHost(tisHost); + checkHost(tisHost); return String.format(this.jobTriggerUrl, tisHost); } @@ -67,17 +67,17 @@ public class TISConfig { return String.format(jobStatusPostBody, taskId); } - public String getJobLogsFetchUrl(String tisHost, String jobName, int taskId) { - checkTisHost(tisHost); - return String.format(jobLogsFetchUrl, tisHost, jobName, taskId); + public String getJobLogsFetchUrl(String host, String jobName, int taskId) { + checkHost(host); + return String.format(jobLogsFetchUrl, host, jobName, taskId); } public String getJobStatusUrl(String tisHost) { - checkTisHost(tisHost); + checkHost(tisHost); return String.format(this.jobStatusUrl, tisHost); } - private static void checkTisHost(String tisHost) { + private static void checkHost(String tisHost) { if (StringUtils.isBlank(tisHost)) { throw new IllegalArgumentException("param tisHost can not be null"); } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-tis/src/main/java/org/apache/dolphinscheduler/plugin/task/tis/TISParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-pigeon/src/main/java/org/apache/dolphinscheduler/plugin/task/pigeon/PigeonParameters.java similarity index 91% rename from dolphinscheduler-task-plugin/dolphinscheduler-task-tis/src/main/java/org/apache/dolphinscheduler/plugin/task/tis/TISParameters.java rename to dolphinscheduler-task-plugin/dolphinscheduler-task-pigeon/src/main/java/org/apache/dolphinscheduler/plugin/task/pigeon/PigeonParameters.java index f19c33ed43..94a544f138 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-tis/src/main/java/org/apache/dolphinscheduler/plugin/task/tis/TISParameters.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-pigeon/src/main/java/org/apache/dolphinscheduler/plugin/task/pigeon/PigeonParameters.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.plugin.task.tis; +package org.apache.dolphinscheduler.plugin.task.pigeon; import org.apache.dolphinscheduler.spi.task.AbstractParameters; import org.apache.dolphinscheduler.spi.task.ResourceInfo; @@ -30,11 +30,11 @@ import org.slf4j.LoggerFactory; /** * TIS parameter */ -public class TISParameters extends AbstractParameters { +public class PigeonParameters extends AbstractParameters { - private static final Logger logger = LoggerFactory.getLogger(TISParameters.class); + private static final Logger logger = LoggerFactory.getLogger(PigeonParameters.class); /** - * TIS target job name + * Pigeon target job name */ private String targetJobName; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-tis/src/main/java/org/apache/dolphinscheduler/plugin/task/tis/TISParamsConstants.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-pigeon/src/main/java/org/apache/dolphinscheduler/plugin/task/pigeon/PigeonParamsConstants.java similarity index 87% rename from dolphinscheduler-task-plugin/dolphinscheduler-task-tis/src/main/java/org/apache/dolphinscheduler/plugin/task/tis/TISParamsConstants.java rename to dolphinscheduler-task-plugin/dolphinscheduler-task-pigeon/src/main/java/org/apache/dolphinscheduler/plugin/task/pigeon/PigeonParamsConstants.java index 54dfcebb7a..e50755a728 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-tis/src/main/java/org/apache/dolphinscheduler/plugin/task/tis/TISParamsConstants.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-pigeon/src/main/java/org/apache/dolphinscheduler/plugin/task/pigeon/PigeonParamsConstants.java @@ -15,13 +15,13 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.plugin.task.tis; +package org.apache.dolphinscheduler.plugin.task.pigeon; -public class TISParamsConstants { +public class PigeonParamsConstants { public static String NAME_TARGET_JOB_NAME = "targetJobName"; public static String TARGET_JOB_NAME = NAME_TARGET_JOB_NAME; - private TISParamsConstants() { + private PigeonParamsConstants() { } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-tis/src/main/java/org/apache/dolphinscheduler/plugin/task/tis/TISTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-pigeon/src/main/java/org/apache/dolphinscheduler/plugin/task/pigeon/PigeonTask.java similarity index 84% rename from dolphinscheduler-task-plugin/dolphinscheduler-task-tis/src/main/java/org/apache/dolphinscheduler/plugin/task/tis/TISTask.java rename to dolphinscheduler-task-plugin/dolphinscheduler-task-pigeon/src/main/java/org/apache/dolphinscheduler/plugin/task/pigeon/PigeonTask.java index aca7a5b04b..2ed7deef68 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-tis/src/main/java/org/apache/dolphinscheduler/plugin/task/tis/TISTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-pigeon/src/main/java/org/apache/dolphinscheduler/plugin/task/pigeon/PigeonTask.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.plugin.task.tis; +package org.apache.dolphinscheduler.plugin.task.pigeon; import org.apache.dolphinscheduler.plugin.task.api.AbstractTaskExecutor; import org.apache.dolphinscheduler.spi.task.AbstractParameters; @@ -48,51 +48,51 @@ import org.java_websocket.handshake.ServerHandshake; /** * TIS DataX Task **/ -public class TISTask extends AbstractTaskExecutor { +public class PigeonTask extends AbstractTaskExecutor { - public static final String KEY_POOL_VAR_TIS_HOST = "tisHost"; + public static final String KEY_POOL_VAR_PIGEON_HOST = "p_host"; private final TaskRequest taskExecutionContext; - private TISParameters tisParameters; + private PigeonParameters parameters; private BizResult triggerResult; - private final TISConfig tisConfig; + private final PigeonConfig config; - public TISTask(TaskRequest taskExecutionContext) { + public PigeonTask(TaskRequest taskExecutionContext) { super(taskExecutionContext); this.taskExecutionContext = taskExecutionContext; - this.tisConfig = TISConfig.getInstance(); + this.config = PigeonConfig.getInstance(); } @Override public void init() { super.init(); - logger.info("tis task params {}", taskExecutionContext.getTaskParams()); - tisParameters = JSONUtils.parseObject(taskExecutionContext.getTaskParams(), TISParameters.class); - if (!tisParameters.checkParameters()) { + logger.info("PIGEON task params {}", taskExecutionContext.getTaskParams()); + parameters = JSONUtils.parseObject(taskExecutionContext.getTaskParams(), PigeonParameters.class); + if (!parameters.checkParameters()) { throw new RuntimeException("datax task params is not valid"); } } @Override public void handle() throws Exception { - // Trigger TIS DataX pipeline - logger.info("start execute TIS task"); + // Trigger PIGEON DataX pipeline + logger.info("start execute PIGEON task"); long startTime = System.currentTimeMillis(); - String targetJobName = this.tisParameters.getTargetJobName(); - String tisHost = getTisHost(); + String targetJobName = this.parameters.getTargetJobName(); + String host = getHost(); try { final String triggerUrl = getTriggerUrl(); - final String getStatusUrl = tisConfig.getJobStatusUrl(tisHost); + final String getStatusUrl = config.getJobStatusUrl(host); HttpPost post = new HttpPost(triggerUrl); post.addHeader("appname", targetJobName); addFormUrlencoded(post); - StringEntity entity = new StringEntity(tisConfig.getJobTriggerPostBody(), StandardCharsets.UTF_8); + StringEntity entity = new StringEntity(config.getJobTriggerPostBody(), StandardCharsets.UTF_8); post.setEntity(entity); ExecResult execState = null; int taskId; WebSocketClient webSocket = null; try (CloseableHttpClient client = HttpClients.createDefault(); - // trigger to start TIS dataX task + // trigger to start PIGEON dataX task CloseableHttpResponse response = client.execute(post)) { triggerResult = processResponse(triggerUrl, response, BizResult.class); if (!triggerResult.isSuccess()) { @@ -101,11 +101,11 @@ public class TISTask extends AbstractTaskExecutor { if (CollectionUtils.isNotEmpty(errormsg)) { errs.append(",errs:").append(errormsg.stream().collect(Collectors.joining(","))); } - throw new Exception("trigger TIS job faild taskName:" + targetJobName + errs.toString()); + throw new Exception("trigger PIGEON job faild taskName:" + targetJobName + errs.toString()); } taskId = triggerResult.getBizresult().getTaskid(); - webSocket = receiveRealtimeLog(tisHost, targetJobName, taskId); + webSocket = receiveRealtimeLog(host, targetJobName, taskId); setAppIds(String.valueOf(taskId)); @@ -141,11 +141,11 @@ public class TISTask extends AbstractTaskExecutor { } long costTime = System.currentTimeMillis() - startTime; - logger.info("TIS task: {},taskId:{} costTime : {} milliseconds, statusCode : {}", + logger.info("PIGEON task: {},taskId:{} costTime : {} milliseconds, statusCode : {}", targetJobName, taskId, costTime, (execState == ExecResult.SUCCESS) ? "'success'" : "'failure'"); setExitStatusCode((execState == ExecResult.SUCCESS) ? TaskConstants.EXIT_CODE_SUCCESS : TaskConstants.EXIT_CODE_FAILURE); } catch (Exception e) { - logger.error("execute TIS dataX faild,TIS task name:" + targetJobName, e); + logger.error("execute PIGEON dataX faild,PIGEON task name:" + targetJobName, e); setExitStatusCode(TaskConstants.EXIT_CODE_FAILURE); if (e instanceof InterruptedException) { Thread.currentThread().interrupt(); @@ -165,7 +165,7 @@ public class TISTask extends AbstractTaskExecutor { logger.info("start to cancelApplication taskId:{}", triggerResult.getTaskId()); final String triggerUrl = getTriggerUrl(); - StringEntity entity = new StringEntity(tisConfig.getJobCancelPostBody(triggerResult.getTaskId()), StandardCharsets.UTF_8); + StringEntity entity = new StringEntity(config.getJobCancelPostBody(triggerResult.getTaskId()), StandardCharsets.UTF_8); CancelResult cancelResult = null; HttpPost post = new HttpPost(triggerUrl); @@ -181,26 +181,26 @@ public class TISTask extends AbstractTaskExecutor { if (org.apache.dolphinscheduler.spi.utils.CollectionUtils.isNotEmpty(errormsg)) { errs.append(",errs:").append(errormsg.stream().collect(Collectors.joining(","))); } - throw new Exception("cancel TIS job faild taskId:" + triggerResult.getTaskId() + errs.toString()); + throw new Exception("cancel PIGEON job faild taskId:" + triggerResult.getTaskId() + errs.toString()); } } } private String getTriggerUrl() { - final String tisHost = getTisHost(); - return tisConfig.getJobTriggerUrl(tisHost); + final String tisHost = getHost(); + return config.getJobTriggerUrl(tisHost); } - private String getTisHost() { - final String tisHost = taskExecutionContext.getDefinedParams().get(KEY_POOL_VAR_TIS_HOST); - if (StringUtils.isEmpty(tisHost)) { - throw new IllegalStateException("global var '" + KEY_POOL_VAR_TIS_HOST + "' can not be empty"); + private String getHost() { + final String host = taskExecutionContext.getDefinedParams().get(KEY_POOL_VAR_PIGEON_HOST); + if (StringUtils.isEmpty(host)) { + throw new IllegalStateException("global var '" + KEY_POOL_VAR_PIGEON_HOST + "' can not be empty"); } - return tisHost; + return host; } private WebSocketClient receiveRealtimeLog(final String tisHost, String dataXName, int taskId) throws Exception { - final String applyURI = tisConfig.getJobLogsFetchUrl(tisHost, dataXName, taskId); + final String applyURI = config.getJobLogsFetchUrl(tisHost, dataXName, taskId); logger.info("apply ws connection,uri:{}", applyURI); WebSocketClient webSocketClient = new WebSocketClient(new URI(applyURI)) { @Override @@ -241,8 +241,8 @@ public class TISTask extends AbstractTaskExecutor { @Override public AbstractParameters getParameters() { - Objects.requireNonNull(this.tisParameters, "tisParameters can not be null"); - return this.tisParameters; + Objects.requireNonNull(this.parameters, "tisParameters can not be null"); + return this.parameters; } private static class CancelResult extends AjaxResult { diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-tis/src/main/java/org/apache/dolphinscheduler/plugin/task/tis/TISTaskChannel.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-pigeon/src/main/java/org/apache/dolphinscheduler/plugin/task/pigeon/PigeonTaskChannel.java similarity index 76% rename from dolphinscheduler-task-plugin/dolphinscheduler-task-tis/src/main/java/org/apache/dolphinscheduler/plugin/task/tis/TISTaskChannel.java rename to dolphinscheduler-task-plugin/dolphinscheduler-task-pigeon/src/main/java/org/apache/dolphinscheduler/plugin/task/pigeon/PigeonTaskChannel.java index 6ae45d6218..78b087a57e 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-tis/src/main/java/org/apache/dolphinscheduler/plugin/task/tis/TISTaskChannel.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-pigeon/src/main/java/org/apache/dolphinscheduler/plugin/task/pigeon/PigeonTaskChannel.java @@ -15,20 +15,25 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.plugin.task.tis; +package org.apache.dolphinscheduler.plugin.task.pigeon; import org.apache.dolphinscheduler.spi.task.AbstractTask; import org.apache.dolphinscheduler.spi.task.TaskChannel; import org.apache.dolphinscheduler.spi.task.request.TaskRequest; -public class TISTaskChannel implements TaskChannel { +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class PigeonTaskChannel implements TaskChannel { + private static final Logger logger = LoggerFactory.getLogger(PigeonTaskChannel.class); + @Override public void cancelApplication(boolean status) { - + logger.info("pigeon task cancel"); } @Override public AbstractTask createTask(TaskRequest taskRequest) { - return new TISTask(taskRequest); + return new PigeonTask(taskRequest); } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-tis/src/main/java/org/apache/dolphinscheduler/plugin/task/tis/TISTaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-pigeon/src/main/java/org/apache/dolphinscheduler/plugin/task/pigeon/PigeonTaskChannelFactory.java similarity index 81% rename from dolphinscheduler-task-plugin/dolphinscheduler-task-tis/src/main/java/org/apache/dolphinscheduler/plugin/task/tis/TISTaskChannelFactory.java rename to dolphinscheduler-task-plugin/dolphinscheduler-task-pigeon/src/main/java/org/apache/dolphinscheduler/plugin/task/pigeon/PigeonTaskChannelFactory.java index 8419198ec8..bc0c6f2b9b 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-tis/src/main/java/org/apache/dolphinscheduler/plugin/task/tis/TISTaskChannelFactory.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-pigeon/src/main/java/org/apache/dolphinscheduler/plugin/task/pigeon/PigeonTaskChannelFactory.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.plugin.task.tis; +package org.apache.dolphinscheduler.plugin.task.pigeon; import org.apache.dolphinscheduler.spi.params.base.PluginParams; import org.apache.dolphinscheduler.spi.params.base.Validate; @@ -26,24 +26,22 @@ import org.apache.dolphinscheduler.spi.task.TaskChannelFactory; import java.util.Arrays; import java.util.List; -/** - * TIS endpoint - **/ -public class TISTaskChannelFactory implements TaskChannelFactory { +public class PigeonTaskChannelFactory implements TaskChannelFactory { @Override public TaskChannel create() { - return new TISTaskChannel(); + return new PigeonTaskChannel(); } @Override public String getName() { - return "TIS"; + return "PIGEON"; } @Override public List getParams() { - InputParam webHookParam = InputParam.newBuilder(TISParamsConstants.NAME_TARGET_JOB_NAME, TISParamsConstants.TARGET_JOB_NAME) + InputParam webHookParam + = InputParam.newBuilder(PigeonParamsConstants.NAME_TARGET_JOB_NAME, PigeonParamsConstants.TARGET_JOB_NAME) .addValidate(Validate.newBuilder().setRequired(true).build()) .build(); return Arrays.asList(webHookParam); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-tis/src/main/java/org/apache/dolphinscheduler/plugin/task/tis/TISTaskPlugin.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-pigeon/src/main/java/org/apache/dolphinscheduler/plugin/task/pigeon/PigeonTaskPlugin.java similarity index 85% rename from dolphinscheduler-task-plugin/dolphinscheduler-task-tis/src/main/java/org/apache/dolphinscheduler/plugin/task/tis/TISTaskPlugin.java rename to dolphinscheduler-task-plugin/dolphinscheduler-task-pigeon/src/main/java/org/apache/dolphinscheduler/plugin/task/pigeon/PigeonTaskPlugin.java index a84b04d0e9..b731d2ab24 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-tis/src/main/java/org/apache/dolphinscheduler/plugin/task/tis/TISTaskPlugin.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-pigeon/src/main/java/org/apache/dolphinscheduler/plugin/task/pigeon/PigeonTaskPlugin.java @@ -15,17 +15,17 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.plugin.task.tis; +package org.apache.dolphinscheduler.plugin.task.pigeon; import org.apache.dolphinscheduler.spi.DolphinSchedulerPlugin; import org.apache.dolphinscheduler.spi.task.TaskChannelFactory; import com.google.common.collect.ImmutableList; -public class TISTaskPlugin implements DolphinSchedulerPlugin { +public class PigeonTaskPlugin implements DolphinSchedulerPlugin { @Override public Iterable getTaskChannelFactorys() { - return ImmutableList.of(new TISTaskChannelFactory()); + return ImmutableList.of(new PigeonTaskChannelFactory()); } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-tis/src/main/resources/org/apache/dolphinscheduler/plugin/task/tis/config.properties b/dolphinscheduler-task-plugin/dolphinscheduler-task-pigeon/src/main/resources/org/apache/dolphinscheduler/plugin/task/pigeon/config.properties similarity index 100% rename from dolphinscheduler-task-plugin/dolphinscheduler-task-tis/src/main/resources/org/apache/dolphinscheduler/plugin/task/tis/config.properties rename to dolphinscheduler-task-plugin/dolphinscheduler-task-pigeon/src/main/resources/org/apache/dolphinscheduler/plugin/task/pigeon/config.properties diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-tis/src/test/java/org/apache/dolphinscheduler/plugin/task/tis/TISTaskTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-pigeon/src/test/java/org/apache/dolphinscheduler/plugin/task/pigeon/PigeonTaskTest.java similarity index 88% rename from dolphinscheduler-task-plugin/dolphinscheduler-task-tis/src/test/java/org/apache/dolphinscheduler/plugin/task/tis/TISTaskTest.java rename to dolphinscheduler-task-plugin/dolphinscheduler-task-pigeon/src/test/java/org/apache/dolphinscheduler/plugin/task/pigeon/PigeonTaskTest.java index adbeb36042..4d64598ce0 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-tis/src/test/java/org/apache/dolphinscheduler/plugin/task/tis/TISTaskTest.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-pigeon/src/test/java/org/apache/dolphinscheduler/plugin/task/pigeon/PigeonTaskTest.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.plugin.task.tis; +package org.apache.dolphinscheduler.plugin.task.pigeon; import static com.github.dreamhead.moco.Moco.file; import static com.github.dreamhead.moco.MocoJsonRunner.jsonHttpServer; @@ -38,15 +38,14 @@ import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.mockito.Mockito; -import org.powermock.api.mockito.PowerMockito; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.github.dreamhead.moco.HttpServer; -public class TISTaskTest { - private static final Logger logger = LoggerFactory.getLogger(TISTaskTest.class); - private TISTask tisTask; +public class PigeonTaskTest { + private static final Logger logger = LoggerFactory.getLogger(PigeonTaskTest.class); + private PigeonTask pigeonTask; private TaskRequest taskExecutionContext; @@ -56,7 +55,7 @@ public class TISTaskTest { String taskParams = "{\"targetJobName\":\"mysql_elastic\"}"; taskExecutionContext = Mockito.mock(TaskRequest.class); - Mockito.when(taskExecutionContext.getTaskLogName()).thenReturn("tislogger"); + Mockito.when(taskExecutionContext.getTaskLogName()).thenReturn("pigeonlogger"); Mockito.when(taskExecutionContext.getTaskParams()).thenReturn(taskParams); Mockito.when(taskExecutionContext.getExecutePath()).thenReturn("/tmp"); Mockito.when(taskExecutionContext.getTaskAppId()).thenReturn(UUID.randomUUID().toString()); @@ -66,17 +65,17 @@ public class TISTaskTest { Mockito.when(taskExecutionContext.getLogPath()).thenReturn("/tmp/dx"); // Mockito.when(taskExecutionContext.getVarPool()) // .thenReturn("[{\"direct\":\"IN\",\"prop\":\"" + TISTask.KEY_POOL_VAR_TIS_HOST + "\",\"type\":\"VARCHAR\",\"value\":\"127.0.0.1:8080\"}]"); - Map gloabParams = Collections.singletonMap(TISTask.KEY_POOL_VAR_TIS_HOST, "127.0.0.1:8080"); + Map gloabParams = Collections.singletonMap(PigeonTask.KEY_POOL_VAR_PIGEON_HOST, "127.0.0.1:8080"); Mockito.when(taskExecutionContext.getDefinedParams()).thenReturn(gloabParams); - tisTask = new TISTask(taskExecutionContext); - tisTask.init(); + pigeonTask = new PigeonTask(taskExecutionContext); + pigeonTask.init(); } @Test public void testGetTISConfigParams() { - TISConfig cfg = TISConfig.getInstance(); + PigeonConfig cfg = PigeonConfig.getInstance(); String tisHost = "127.0.0.1:8080"; Assert.assertEquals("http://127.0.0.1:8080/tjs/coredefine/coredefine.ajax", cfg.getJobTriggerUrl(tisHost)); String jobName = "mysql_elastic"; @@ -97,7 +96,7 @@ public class TISTaskTest { public void testInit() throws Exception { try { - tisTask.init(); + pigeonTask.init(); } catch (Exception e) { Assert.fail(e.getMessage()); } @@ -106,12 +105,12 @@ public class TISTaskTest { @Test public void testHandle() throws Exception { - HttpServer server = jsonHttpServer(8080, file("src/test/resources/org/apache/dolphinscheduler/plugin/task/tis/TISTaskTest.json")); + HttpServer server = jsonHttpServer(8080, file("src/test/resources/org/apache/dolphinscheduler/plugin/task/pigeon/PigeonTaskTest.json")); running(server, () -> { - tisTask.handle(); + pigeonTask.handle(); - Assert.assertEquals("TIS execute be success", ExecutionStatus.SUCCESS, tisTask.getExitStatus()); + Assert.assertEquals("PIGEON execute be success", ExecutionStatus.SUCCESS, pigeonTask.getExitStatus()); }); } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-tis/src/test/resources/org/apache/dolphinscheduler/plugin/task/tis/TISTaskTest.json b/dolphinscheduler-task-plugin/dolphinscheduler-task-pigeon/src/test/resources/org/apache/dolphinscheduler/plugin/task/pigeon/PigeonTaskTest.json similarity index 94% rename from dolphinscheduler-task-plugin/dolphinscheduler-task-tis/src/test/resources/org/apache/dolphinscheduler/plugin/task/tis/TISTaskTest.json rename to dolphinscheduler-task-plugin/dolphinscheduler-task-pigeon/src/test/resources/org/apache/dolphinscheduler/plugin/task/pigeon/PigeonTaskTest.json index fe11b39fda..5aef31fd21 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-tis/src/test/resources/org/apache/dolphinscheduler/plugin/task/tis/TISTaskTest.json +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-pigeon/src/test/resources/org/apache/dolphinscheduler/plugin/task/pigeon/PigeonTaskTest.json @@ -1,6 +1,6 @@ [ { - "description": "trigger TIS task execute", + "description": "trigger task execute", "request": { "uri": "/tjs/coredefine/coredefine.ajax", "method": "post", @@ -15,7 +15,7 @@ } }, { - "description": "Get TIS task execute status", + "description": "Get task execute status", "request": { "uri": "/tjs/config/config.ajax", "method": "post", diff --git a/dolphinscheduler-task-plugin/pom.xml b/dolphinscheduler-task-plugin/pom.xml index cffc6d1cb9..bd223db8e1 100644 --- a/dolphinscheduler-task-plugin/pom.xml +++ b/dolphinscheduler-task-plugin/pom.xml @@ -40,7 +40,7 @@ dolphinscheduler-task-sql dolphinscheduler-task-sqoop dolphinscheduler-task-procedure - dolphinscheduler-task-tis + dolphinscheduler-task-pigeon diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/images/task-icos/tis.png b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/images/task-icos/tis.png deleted file mode 100755 index 652f5c4172e182cdeefc326de735941c4fce8fb3..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1361 zcmeAS@N?(olHy`uVBq!ia0vp^Y9P$P3?%12mYf5m7>k44ofy`glX=O&z@!o26XFUK z{Qv*|s#UA@@83Uj=1d@C+qP}<=g$W+fZRok76BP^=gwWSWC>7k>C&Zh=FC~Ra3Mr$ z_Uzd}K_D9_0%XsdHxI}LDgX+?xj^=;S+jr&fQHPNF$1UoC>Qk2+&sK|0)j%qB4XkalF~A=a`Fm_ zO3JEgYU*0Ldiq8tre@}r)^-k#PA+clUfw?bLBXM6;gQiXv2pQ@9$8t2YmuxRm;Wy@EtTDx}Lx(ypQZ`rne$Id-__a8cZ z^w{x}r_P+cchA3uNn z{{83A-+%wY*Zg7wCVLT27sn6@$-4(%M|(3eus^teZP~2Tms$ z#9uMh)CmHuL+@`Yg}P2HE1(|DP3Dv_-a9 zrvBR0_~KqR$N5V|2P;;K9=K`#SupGi^QU`XvKBa;a*KFYHtAlP!ar}(L$j_sUj4$g z;O$1iT`N*&pFVTpY3RJ2KR29zy4-6G`>8vb2M(=$`rInc`$_1F9_RDYuluIF(b?*4 zRI7PHB%j&hZDrNss&c=Z5ke*3SYq6A?E?v%$dAt0N zh^E@YFqO!*=7` zXUiT;7ghYu-stW(X>$9*u3E)O(-&s$>zhz}N-t;13dtWYZV6pgog`CgHTSph(yu>O zdlX7uv^~SHQvbu!v4iVYKQ=d+#od(nsxM+LxC1Sl|sj~Iy@KoI~E7ns6=ht_;Rb-_DliJVHg(jpiERO-e1ySxq<}L(Y)gS- zn4k(4h2(6tfEcpSEn>+>wk9Y5?Fi4=^kT8_xibj|X|X*O7PaF=k1&24Lt0=i%m^Ey zEieE}Ml4(29e!)z9Xl3o@;SPQHcLlvFBx|6z9a>+S=Os`o>0{UljZ% z8GVMcBU4*v?6>9cYj~XEhnd0ZqLWkQ%^f;AH@WV~!Tu8W{68{X7`zfvvX$HWTP5m! z1?e?I|4A0se{P)_Y`8obx+?5d1Z{r9)G2MxZhB?kP`irC!Jaal+)FwknkD>oz|99Y zG#f|EAk-WBcnB^r!(}p_l^G02;P1@%K;)KHe(YE~3TAqQ+20Z|Xp5 z;_yd)^jXK0Mf#jYdx@HO;S^rs`SmVWyoIt>kxQ7NuMz)gninzVbs7f3^IUtsL#o z{nsq!6g4@;hsDaPsg^su6x}3uKyXW^GT0>1lxO0QfgUgRf@evtyY7X-Va{VQFZHakHWH5@v(w= zopy}@n)_hKJF;dTD@kKyxhA3~IhZ~(WDkE99K*%iS>hCv^*^7x-fsW^ diff --git a/pom.xml b/pom.xml index 0cc4f5ac1c..5d49a974b6 100644 --- a/pom.xml +++ b/pom.xml @@ -1103,7 +1103,7 @@ **/alert/processor/AlertRequestProcessorTest.java **/alert/runner/AlertSenderTest.java **/alert/AlertServerTest.java - **/plugin/task/tis/TISTaskTest.java + **/plugin/task/pigeon/PigeonTaskTest.java