From 46b8b1526e9aeb3362bd874e2e84afb0085fd4c6 Mon Sep 17 00:00:00 2001 From: wuxiaofei Date: Fri, 22 Jan 2021 17:22:33 +0800 Subject: [PATCH 001/142] [Improvement-#4503][dolphinscheduler-dist]Jackson version upgraded to 2.10.5 --- dolphinscheduler-common/sql/soft_version | 1 + .../org/apache/dolphinscheduler/common/utils/JSONUtils.java | 6 +++++- dolphinscheduler-dist/release-docs/LICENSE | 6 +++--- pom.xml | 2 +- tools/dependencies/known-dependencies.txt | 6 +++--- 5 files changed, 13 insertions(+), 8 deletions(-) create mode 100644 dolphinscheduler-common/sql/soft_version diff --git a/dolphinscheduler-common/sql/soft_version b/dolphinscheduler-common/sql/soft_version new file mode 100644 index 0000000000..5409e995b5 --- /dev/null +++ b/dolphinscheduler-common/sql/soft_version @@ -0,0 +1 @@ +32432423 \ No newline at end of file diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java index fc11a2add2..4ab04997d8 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java @@ -294,7 +294,11 @@ public class JSONUtils { public static ObjectNode parseObject(String text) { try { - return (ObjectNode) objectMapper.readTree(text); + if (text.isEmpty()) { + return parseObject(text, ObjectNode.class); + } else { + return (ObjectNode) objectMapper.readTree(text); + } } catch (Exception e) { throw new RuntimeException("String json deserialization exception.", e); } diff --git a/dolphinscheduler-dist/release-docs/LICENSE b/dolphinscheduler-dist/release-docs/LICENSE index 4f8afe6741..44892a242e 100644 --- a/dolphinscheduler-dist/release-docs/LICENSE +++ b/dolphinscheduler-dist/release-docs/LICENSE @@ -286,10 +286,10 @@ The text of each license is also included at licenses/LICENSE-[project].txt. httpclient 4.4.1: https://mvnrepository.com/artifact/org.apache.httpcomponents/httpclient/4.4.1, Apache 2.0 httpcore 4.4.1: https://mvnrepository.com/artifact/org.apache.httpcomponents/httpcore/4.4.1, Apache 2.0 httpmime 4.5.7: https://mvnrepository.com/artifact/org.apache.httpcomponents/httpmime/4.5.7, Apache 2.0 - jackson-annotations 2.9.10: https://mvnrepository.com/artifact/com.fasterxml.jackson.core/jackson-annotations/2.9.10, Apache 2.0 - jackson-core 2.9.10: https://github.com/FasterXML/jackson-core, Apache 2.0 + jackson-annotations 2.10.5: https://mvnrepository.com/artifact/com.fasterxml.jackson.core/jackson-annotations/2.10.5, Apache 2.0 + jackson-core 2.10.5: https://github.com/FasterXML/jackson-core, Apache 2.0 jackson-core-asl 1.9.13: https://mvnrepository.com/artifact/org.codehaus.jackson/jackson-core-asl/1.9.13, Apache 2.0 - jackson-databind 2.9.10: https://github.com/FasterXML/jackson-databind, Apache 2.0 + jackson-databind 2.10.5: https://github.com/FasterXML/jackson-databind, Apache 2.0 jackson-datatype-jdk8 2.9.10: https://mvnrepository.com/artifact/com.fasterxml.jackson.datatype/jackson-datatype-jdk8/2.9.10, Apache 2.0 jackson-datatype-jsr310 2.9.10: https://mvnrepository.com/artifact/com.fasterxml.jackson.datatype/jackson-datatype-jsr310/2.9.10, Apache 2.0 jackson-jaxrs 1.9.13: https://mvnrepository.com/artifact/org.codehaus.jackson/jackson-jaxrs/1.9.13, Apache 2.0 and LGPL 2.1 diff --git a/pom.xml b/pom.xml index 5c8dd67613..63776bdb03 100644 --- a/pom.xml +++ b/pom.xml @@ -65,7 +65,7 @@ 1.2.3 2.7.3 2.3.0 - 2.9.10 + 2.10.5 3.2.0 2.0.1 5.0.5 diff --git a/tools/dependencies/known-dependencies.txt b/tools/dependencies/known-dependencies.txt index ebec45992f..188db804f1 100755 --- a/tools/dependencies/known-dependencies.txt +++ b/tools/dependencies/known-dependencies.txt @@ -79,10 +79,10 @@ httpclient-4.4.1.jar httpcore-4.4.1.jar httpmime-4.5.12.jar j2objc-annotations-1.1.jar -jackson-annotations-2.9.10.jar -jackson-core-2.9.10.jar +jackson-annotations-2.10.5.jar +jackson-core-2.10.5.jar jackson-core-asl-1.9.13.jar -jackson-databind-2.9.10.jar +jackson-databind-2.10.5.jar jackson-datatype-jdk8-2.9.10.jar jackson-datatype-jsr310-2.9.10.jar jackson-jaxrs-1.9.13.jar From bd2d0ba1709c254d4b8f5b466755a1de76b1bcc7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E6=B8=85?= <62982788+597365581@users.noreply.github.com> Date: Fri, 29 Jan 2021 10:12:16 +0800 Subject: [PATCH 002/142] =?UTF-8?q?fix=20[Bug][LoggerServer]=20task=20log?= =?UTF-8?q?=20can=20not=20refresh=20in=20time=C2=A0#4617?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fix [Bug][LoggerServer] task log can not refresh in time #4617 --- .../server/worker/task/AbstractCommandExecutor.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java index da5c0e6980..bf306e51bc 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java @@ -348,12 +348,14 @@ public abstract class AbstractCommandExecutor { long lastFlushTime = System.currentTimeMillis(); - while ((line = inReader.readLine()) != null) { + while ((line = inReader.readLine()) != null || logBuffer.size()>0) { if (line.startsWith("${setValue(")) { varPool.append(line.substring("${setValue(".length(), line.length() - 2)); varPool.append("$VarPool$"); } else { + if(null != line){ logBuffer.add(line); + } lastFlushTime = flush(lastFlushTime); } } @@ -560,4 +562,4 @@ public abstract class AbstractCommandExecutor { protected abstract String commandInterpreter(); protected abstract void createCommandFileIfNotExists(String execCommand, String commandFile) throws IOException; -} \ No newline at end of file +} From 7ec5c94dfe08b25d80916a4c6aaf62a28a0a2d2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E6=B8=85?= <62982788+597365581@users.noreply.github.com> Date: Fri, 29 Jan 2021 10:16:10 +0800 Subject: [PATCH 003/142] Update AbstractCommandExecutor.java --- .../server/worker/task/AbstractCommandExecutor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java index bf306e51bc..2235ef731b 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java @@ -353,7 +353,7 @@ public abstract class AbstractCommandExecutor { varPool.append(line.substring("${setValue(".length(), line.length() - 2)); varPool.append("$VarPool$"); } else { - if(null != line){ + if (null != line) { logBuffer.add(line); } lastFlushTime = flush(lastFlushTime); From 0b602bcaf4b8acdf7dca1d1ad00e5b2d73d62b69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E6=B8=85?= <62982788+597365581@users.noreply.github.com> Date: Fri, 29 Jan 2021 10:20:49 +0800 Subject: [PATCH 004/142] Update AbstractCommandExecutor.java --- .../server/worker/task/AbstractCommandExecutor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java index 2235ef731b..c63af2cb61 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java @@ -348,7 +348,7 @@ public abstract class AbstractCommandExecutor { long lastFlushTime = System.currentTimeMillis(); - while ((line = inReader.readLine()) != null || logBuffer.size()>0) { + while ((line = inReader.readLine()) != null || logBuffer.size() > 0) { if (line.startsWith("${setValue(")) { varPool.append(line.substring("${setValue(".length(), line.length() - 2)); varPool.append("$VarPool$"); From 866f650e57df61690cb8b0e7ba215cdbaac94327 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E6=B8=85?= <62982788+597365581@users.noreply.github.com> Date: Fri, 29 Jan 2021 10:41:03 +0800 Subject: [PATCH 005/142] Update AbstractCommandExecutor.java --- .../worker/task/AbstractCommandExecutor.java | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java index c63af2cb61..731cc20fbc 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java @@ -349,15 +349,15 @@ public abstract class AbstractCommandExecutor { long lastFlushTime = System.currentTimeMillis(); while ((line = inReader.readLine()) != null || logBuffer.size() > 0) { - if (line.startsWith("${setValue(")) { - varPool.append(line.substring("${setValue(".length(), line.length() - 2)); - varPool.append("$VarPool$"); - } else { - if (null != line) { - logBuffer.add(line); - } - lastFlushTime = flush(lastFlushTime); - } + if (null != line) { + if (line.startsWith("${setValue(")) { + varPool.append(line.substring("${setValue(".length(), line.length() - 2)); + varPool.append("$VarPool$"); + } else { + logBuffer.add(line); + } + } + lastFlushTime = flush(lastFlushTime); } } catch (Exception e) { logger.error(e.getMessage(), e); From 25305a53f12a3bca83cd1f93af4cdd4ca22b6720 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E6=B8=85?= <62982788+597365581@users.noreply.github.com> Date: Fri, 29 Jan 2021 10:47:50 +0800 Subject: [PATCH 006/142] Update AbstractCommandExecutor.java --- .../worker/task/AbstractCommandExecutor.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java index 731cc20fbc..9a2d257a63 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java @@ -350,14 +350,14 @@ public abstract class AbstractCommandExecutor { while ((line = inReader.readLine()) != null || logBuffer.size() > 0) { if (null != line) { - if (line.startsWith("${setValue(")) { - varPool.append(line.substring("${setValue(".length(), line.length() - 2)); - varPool.append("$VarPool$"); + if (line.startsWith("${setValue(")) { + varPool.append(line.substring("${setValue(".length(), line.length() - 2)); + varPool.append("$VarPool$"); } else { - logBuffer.add(line); - } - } - lastFlushTime = flush(lastFlushTime); + logBuffer.add(line); + } + } + lastFlushTime = flush(lastFlushTime); } } catch (Exception e) { logger.error(e.getMessage(), e); From 13f0fcf5ff0480aa17874ae38b149235c19c22e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E6=B8=85?= <62982788+597365581@users.noreply.github.com> Date: Fri, 29 Jan 2021 11:56:19 +0800 Subject: [PATCH 007/142] Update AbstractCommandExecutor.java --- .../server/worker/task/AbstractCommandExecutor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java index 9a2d257a63..023eacb69b 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java @@ -347,7 +347,7 @@ public abstract class AbstractCommandExecutor { String line; long lastFlushTime = System.currentTimeMillis(); - + logBuffer.add("welcome to use dolphinscheduler scheduling system..."); while ((line = inReader.readLine()) != null || logBuffer.size() > 0) { if (null != line) { if (line.startsWith("${setValue(")) { From d5c4c290d32bd68f10eade84cb3660f90b39bdba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E6=B8=85?= <62982788+597365581@users.noreply.github.com> Date: Fri, 29 Jan 2021 12:29:42 +0800 Subject: [PATCH 008/142] Update AbstractCommandExecutor.java --- .../server/worker/task/AbstractCommandExecutor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java index 023eacb69b..9d7584de72 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java @@ -347,7 +347,7 @@ public abstract class AbstractCommandExecutor { String line; long lastFlushTime = System.currentTimeMillis(); - logBuffer.add("welcome to use dolphinscheduler scheduling system..."); + //logBuffer.add("welcome to use dolphinscheduler scheduling system..."); while ((line = inReader.readLine()) != null || logBuffer.size() > 0) { if (null != line) { if (line.startsWith("${setValue(")) { From 7b345f697af66ed978fb93e071a8c1ff89ec97bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E6=B8=85?= <62982788+597365581@users.noreply.github.com> Date: Fri, 29 Jan 2021 13:47:10 +0800 Subject: [PATCH 009/142] Update AbstractCommandExecutor.java --- .../server/worker/task/AbstractCommandExecutor.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java index 9d7584de72..1854aa9ea5 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java @@ -344,11 +344,10 @@ public abstract class AbstractCommandExecutor { try { inReader = new BufferedReader(new InputStreamReader(process.getInputStream())); - String line; + String line = null; long lastFlushTime = System.currentTimeMillis(); - //logBuffer.add("welcome to use dolphinscheduler scheduling system..."); - while ((line = inReader.readLine()) != null || logBuffer.size() > 0) { + while (logBuffer.size()>0 || (line = inReader.readLine()) != null) { if (null != line) { if (line.startsWith("${setValue(")) { varPool.append(line.substring("${setValue(".length(), line.length() - 2)); From cd89aed360e1e28d1772f9b97b03a9724ccba1fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E6=B8=85?= <62982788+597365581@users.noreply.github.com> Date: Fri, 29 Jan 2021 13:49:35 +0800 Subject: [PATCH 010/142] Update AbstractCommandExecutor.java --- .../server/worker/task/AbstractCommandExecutor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java index 1854aa9ea5..30950a75df 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java @@ -347,7 +347,7 @@ public abstract class AbstractCommandExecutor { String line = null; long lastFlushTime = System.currentTimeMillis(); - while (logBuffer.size()>0 || (line = inReader.readLine()) != null) { + while (logBuffer.size() > 0 || (line = inReader.readLine()) != null) { if (null != line) { if (line.startsWith("${setValue(")) { varPool.append(line.substring("${setValue(".length(), line.length() - 2)); From c61d0a717437f0a42cb3be5008e0f24e7ca819b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E6=B8=85?= <62982788+597365581@users.noreply.github.com> Date: Fri, 29 Jan 2021 14:30:44 +0800 Subject: [PATCH 011/142] Update AbstractCommandExecutor.java --- .../server/worker/task/AbstractCommandExecutor.java | 1 + 1 file changed, 1 insertion(+) diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java index 30950a75df..8dbf7220b0 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java @@ -355,6 +355,7 @@ public abstract class AbstractCommandExecutor { } else { logBuffer.add(line); } + line = null; } lastFlushTime = flush(lastFlushTime); } From f30f4cb3d4a330318ba0964b70225ed709fa286d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E6=B8=85?= <62982788+597365581@users.noreply.github.com> Date: Sat, 30 Jan 2021 09:48:34 +0800 Subject: [PATCH 012/142] Update AbstractCommandExecutor.java MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 将inReader.readLine()和logBuffer 分开，改成两个线程来处理。 --- .../worker/task/AbstractCommandExecutor.java | 49 +++++++++++++------ 1 file changed, 33 insertions(+), 16 deletions(-) diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java index 8dbf7220b0..cd57be9181 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java @@ -82,6 +82,8 @@ public abstract class AbstractCommandExecutor { * log list */ protected final List logBuffer; + + protected boolean logOutputIsScuccess = false; /** * taskExecutionContext @@ -336,34 +338,49 @@ public abstract class AbstractCommandExecutor { */ private void parseProcessOutput(Process process) { String threadLoggerInfoName = String.format(LoggerUtils.TASK_LOGGER_THREAD_NAME + "-%s", taskExecutionContext.getTaskAppId()); - ExecutorService parseProcessOutputExecutorService = ThreadUtils.newDaemonSingleThreadExecutor(threadLoggerInfoName); - parseProcessOutputExecutorService.submit(new Runnable() { + ExecutorService getOutputLogService = ThreadUtils.newDaemonSingleThreadExecutor(threadLoggerInfoName + "-" + "getOutputLogService"); + getOutputLogService.submit(new Runnable() { @Override public void run() { BufferedReader inReader = null; try { inReader = new BufferedReader(new InputStreamReader(process.getInputStream())); - String line = null; - - long lastFlushTime = System.currentTimeMillis(); - while (logBuffer.size() > 0 || (line = inReader.readLine()) != null) { - if (null != line) { - if (line.startsWith("${setValue(")) { - varPool.append(line.substring("${setValue(".length(), line.length() - 2)); - varPool.append("$VarPool$"); - } else { - logBuffer.add(line); - } - line = null; + String line; + while ((line = inReader.readLine()) != null) { + if (line.startsWith("${setValue(")) { + varPool.append(line.substring("${setValue(".length(), line.length() - 2)); + varPool.append("$VarPool$"); + } else { + logBuffer.add(line); + } + } + } catch (Exception e) { + logger.error(e.getMessage(), e); + } finally { + logOutputIsScuccess = true; + close(inReader); + } + } + }); + + ExecutorService parseProcessOutputExecutorService = ThreadUtils.newDaemonSingleThreadExecutor(threadLoggerInfoName); + parseProcessOutputExecutorService.submit(new Runnable() { + @Override + public void run() { + try { + long lastFlushTime = System.currentTimeMillis(); + while (logBuffer.size() > 0 || !logOutputIsScuccess) { + if (logBuffer.size() > 0) { + lastFlushTime = flush(lastFlushTime); + } else { + Thread.sleep(Constants.DEFAULT_LOG_FLUSH_INTERVAL); } - lastFlushTime = flush(lastFlushTime); } } catch (Exception e) { logger.error(e.getMessage(), e); } finally { clear(); - close(inReader); } } }); From 4597e87970521e916364fef71cb955c5a28330d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E6=B8=85?= <62982788+597365581@users.noreply.github.com> Date: Sat, 30 Jan 2021 09:53:25 +0800 Subject: [PATCH 013/142] Update AbstractCommandExecutor.java --- .../server/worker/task/AbstractCommandExecutor.java | 1 + 1 file changed, 1 insertion(+) diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java index cd57be9181..5b09be85ae 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java @@ -363,6 +363,7 @@ public abstract class AbstractCommandExecutor { } } }); + getOutputLogService.shutdown(); ExecutorService parseProcessOutputExecutorService = ThreadUtils.newDaemonSingleThreadExecutor(threadLoggerInfoName); parseProcessOutputExecutorService.submit(new Runnable() { From c086b03b8dfba9f61106ad89b08b47fbd5593421 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E6=B8=85?= <62982788+597365581@users.noreply.github.com> Date: Sat, 30 Jan 2021 12:53:19 +0800 Subject: [PATCH 014/142] Update ShellTaskTest.java --- .../server/worker/task/shell/ShellTaskTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTaskTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTaskTest.java index c5f2de82ea..e08b00feda 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTaskTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTaskTest.java @@ -99,6 +99,8 @@ public class ShellTaskTest { public void testComplementData() throws Exception { shellTask = new ShellTask(taskExecutionContext, logger); shellTask.init(); + shellCommandExecutor.isSuccessOfYarnState(new ArrayList<>()); + shellCommandExecutor.isSuccessOfYarnState(null); PowerMockito.when(shellCommandExecutor.run(anyString())).thenReturn(commandExecuteResult); shellTask.handle(); } From b3adb4d10abe1cd561294669959bdb5351e14e6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E6=B8=85?= <62982788+597365581@users.noreply.github.com> Date: Sat, 30 Jan 2021 13:12:21 +0800 Subject: [PATCH 015/142] Update ShellTaskTest.java --- .../server/worker/task/shell/ShellTaskTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTaskTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTaskTest.java index e08b00feda..b0abbb8934 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTaskTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTaskTest.java @@ -28,7 +28,7 @@ import java.nio.file.Paths; import java.util.Date; import java.util.HashMap; import java.util.Map; - +import java.util.ArrayList; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; From f40d4136eda12867bcd5984da39ce9bc25eddd26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E6=B8=85?= <62982788+597365581@users.noreply.github.com> Date: Sat, 30 Jan 2021 13:19:33 +0800 Subject: [PATCH 016/142] Update ShellTaskTest.java --- .../server/worker/task/shell/ShellTaskTest.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTaskTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTaskTest.java index b0abbb8934..ab39378397 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTaskTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTaskTest.java @@ -22,13 +22,12 @@ import static org.mockito.ArgumentMatchers.anyString; import org.apache.dolphinscheduler.server.entity.TaskExecutionContext; import org.apache.dolphinscheduler.server.worker.task.CommandExecuteResult; import org.apache.dolphinscheduler.server.worker.task.ShellCommandExecutor; - import java.nio.file.Files; import java.nio.file.Paths; +import java.util.ArrayList; import java.util.Date; import java.util.HashMap; import java.util.Map; -import java.util.ArrayList; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; From 3b14dfb928e355f9b276bcfed4b66f967eb4077b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E6=B8=85?= <62982788+597365581@users.noreply.github.com> Date: Sat, 30 Jan 2021 14:01:48 +0800 Subject: [PATCH 017/142] Update ShellCommandExecutorTest.java --- .../shell/ShellCommandExecutorTest.java | 52 ++++++++++++++----- 1 file changed, 39 insertions(+), 13 deletions(-) diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java index ed3479ee0d..887023cadd 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java @@ -16,39 +16,52 @@ */ package org.apache.dolphinscheduler.server.worker.shell; +import com.alibaba.fastjson.JSON; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.model.TaskNode; -import org.apache.dolphinscheduler.common.utils.LoggerUtils; +import org.apache.dolphinscheduler.common.utils.OSUtils; import org.apache.dolphinscheduler.dao.entity.TaskInstance; +import org.apache.dolphinscheduler.common.utils.LoggerUtils; import org.apache.dolphinscheduler.server.worker.task.AbstractTask; +import org.apache.dolphinscheduler.server.worker.task.ShellCommandExecutor; import org.apache.dolphinscheduler.server.worker.task.TaskProps; -import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; import org.apache.dolphinscheduler.service.process.ProcessService; import org.junit.Before; import org.junit.Ignore; import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PowerMockIgnore; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.dolphinscheduler.common.utils.*; +import org.springframework.context.ApplicationContext; +import java.lang.reflect.Method; import java.util.Date; /** - * python shell command executor test + * python shell command executor test */ -@Ignore +@RunWith(PowerMockRunner.class) +@PrepareForTest(OSUtils.class) +@PowerMockIgnore({"javax.management.*"}) public class ShellCommandExecutorTest { private static final Logger logger = LoggerFactory.getLogger(ShellCommandExecutorTest.class); private ProcessService processService = null; + private ApplicationContext applicationContext; @Before - public void before(){ - processService = SpringApplicationContext.getBean(ProcessService.class); + public void before() { + processService = PowerMockito.mock(ProcessService.class); + applicationContext = PowerMockito.mock(ApplicationContext.class); } + @Ignore @Test public void test() throws Exception { @@ -63,11 +76,10 @@ public class ShellCommandExecutorTest { taskProps.setTaskInstanceId(7657); - TaskInstance taskInstance = processService.findTaskInstanceById(7657); String taskJson = taskInstance.getTaskJson(); - TaskNode taskNode = JSONUtils.parseObject(taskJson, TaskNode.class); + TaskNode taskNode = JSON.parseObject(taskJson, TaskNode.class); taskProps.setTaskParams(taskNode.getParams()); @@ -91,14 +103,28 @@ public class ShellCommandExecutorTest { task.handle(); ExecutionStatus status = ExecutionStatus.SUCCESS; - if (task.getExitStatusCode() == Constants.EXIT_CODE_SUCCESS){ + if (task.getExitStatusCode() == Constants.EXIT_CODE_SUCCESS) { status = ExecutionStatus.SUCCESS; - }else if (task.getExitStatusCode() == Constants.EXIT_CODE_KILL){ + } else if (task.getExitStatusCode() == Constants.EXIT_CODE_KILL) { status = ExecutionStatus.KILL; - }else { + } else { status = ExecutionStatus.FAILURE; } logger.info(status.toString()); } -} \ No newline at end of file + + @Test + public void testParseProcessOutput() { + Class shellCommandExecutorClass = ShellCommandExecutor.class; + try { + Object instance = shellCommandExecutorClass.newInstance(); + + Method method = shellCommandExecutorClass.getDeclaredMethod("parseProcessOutput", new Class[]{}); + method.setAccessible(true); + ShellCommandExecutor result = (ShellCommandExecutor) method.invoke(instance, new Object[]{}); + } catch (Exception e) { + logger.error(e.getMessage()); + } + } +} From d24238c1fe242d4f0f10193bf028f1c710f598bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E6=B8=85?= <62982788+597365581@users.noreply.github.com> Date: Sat, 30 Jan 2021 14:10:11 +0800 Subject: [PATCH 018/142] Update ShellCommandExecutorTest.java --- .../server/worker/shell/ShellCommandExecutorTest.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java index 887023cadd..f2e241878a 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java @@ -20,9 +20,9 @@ import com.alibaba.fastjson.JSON; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.model.TaskNode; +import org.apache.dolphinscheduler.common.utils.LoggerUtils; import org.apache.dolphinscheduler.common.utils.OSUtils; import org.apache.dolphinscheduler.dao.entity.TaskInstance; -import org.apache.dolphinscheduler.common.utils.LoggerUtils; import org.apache.dolphinscheduler.server.worker.task.AbstractTask; import org.apache.dolphinscheduler.server.worker.task.ShellCommandExecutor; import org.apache.dolphinscheduler.server.worker.task.TaskProps; @@ -38,9 +38,8 @@ import org.powermock.modules.junit4.PowerMockRunner; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.context.ApplicationContext; - -import java.lang.reflect.Method; import java.util.Date; +import java.lang.reflect.Method; /** * python shell command executor test From f5f3f8972d718467fec7e78d17d4d6a8c4d53553 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E6=B8=85?= <62982788+597365581@users.noreply.github.com> Date: Sat, 30 Jan 2021 14:12:14 +0800 Subject: [PATCH 019/142] Update ShellCommandExecutorTest.java --- .../server/worker/shell/ShellCommandExecutorTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java index f2e241878a..037d48e790 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java @@ -38,8 +38,8 @@ import org.powermock.modules.junit4.PowerMockRunner; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.context.ApplicationContext; -import java.util.Date; import java.lang.reflect.Method; +import java.util.Date; /** * python shell command executor test From af996be47b53bce5e55a118e5a4b83d0f51fb27c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E6=B8=85?= <62982788+597365581@users.noreply.github.com> Date: Sat, 30 Jan 2021 14:15:37 +0800 Subject: [PATCH 020/142] Update ShellCommandExecutorTest.java --- .../server/worker/shell/ShellCommandExecutorTest.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java index 037d48e790..5aaeac866b 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java @@ -15,7 +15,7 @@ * limitations under the License. */ package org.apache.dolphinscheduler.server.worker.shell; - +import java.lang.reflect.Method; import com.alibaba.fastjson.JSON; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; @@ -38,7 +38,6 @@ import org.powermock.modules.junit4.PowerMockRunner; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.context.ApplicationContext; -import java.lang.reflect.Method; import java.util.Date; /** From a168e577134f0933ff904a61dfdb8ce64414b5d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E6=B8=85?= <62982788+597365581@users.noreply.github.com> Date: Sat, 30 Jan 2021 14:19:06 +0800 Subject: [PATCH 021/142] Update ShellCommandExecutorTest.java --- .../server/worker/shell/ShellCommandExecutorTest.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java index 5aaeac866b..de1ebf5948 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java @@ -15,7 +15,9 @@ * limitations under the License. */ package org.apache.dolphinscheduler.server.worker.shell; + import java.lang.reflect.Method; + import com.alibaba.fastjson.JSON; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; @@ -38,6 +40,7 @@ import org.powermock.modules.junit4.PowerMockRunner; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.context.ApplicationContext; + import java.util.Date; /** From 6f02ff1aad0c2d52eebb92984801bef05297f0c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E6=B8=85?= <62982788+597365581@users.noreply.github.com> Date: Sat, 30 Jan 2021 14:29:19 +0800 Subject: [PATCH 022/142] Update ShellCommandExecutorTest.java --- .../server/worker/shell/ShellCommandExecutorTest.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java index de1ebf5948..f258e63ad0 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java @@ -18,10 +18,10 @@ package org.apache.dolphinscheduler.server.worker.shell; import java.lang.reflect.Method; -import com.alibaba.fastjson.JSON; + import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; -import org.apache.dolphinscheduler.common.model.TaskNode; + import org.apache.dolphinscheduler.common.utils.LoggerUtils; import org.apache.dolphinscheduler.common.utils.OSUtils; import org.apache.dolphinscheduler.dao.entity.TaskInstance; @@ -80,8 +80,8 @@ public class ShellCommandExecutorTest { TaskInstance taskInstance = processService.findTaskInstanceById(7657); String taskJson = taskInstance.getTaskJson(); - TaskNode taskNode = JSON.parseObject(taskJson, TaskNode.class); - taskProps.setTaskParams(taskNode.getParams()); +// TaskNode taskNode = JSON.parseObject(taskJson, TaskNode.class); +// taskProps.setTaskParams(taskNode.getParams()); // custom logger From 756dcb9002422512960abe77dc271365d31c88c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E6=B8=85?= <62982788+597365581@users.noreply.github.com> Date: Sat, 30 Jan 2021 14:31:27 +0800 Subject: [PATCH 023/142] Update ShellCommandExecutorTest.java --- .../server/worker/shell/ShellCommandExecutorTest.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java index f258e63ad0..a59fdf8d03 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java @@ -85,10 +85,10 @@ public class ShellCommandExecutorTest { // custom logger - Logger taskLogger = LoggerFactory.getLogger(LoggerUtils.buildTaskId(LoggerUtils.TASK_LOGGER_INFO_PREFIX, - taskInstance.getProcessDefinitionId(), - taskInstance.getProcessInstanceId(), - taskInstance.getId())); +// Logger taskLogger = LoggerFactory.getLogger(LoggerUtils.buildTaskId(LoggerUtils.TASK_LOGGER_INFO_PREFIX, +// taskInstance.getProcessDefinitionId(), +// taskInstance.getProcessInstanceId(), +// taskInstance.getId())); // AbstractTask task = TaskManager.newTask(taskInstance.getTaskType(), taskProps, taskLogger); From dfc40c064543be6eea99940a61b3764c8b5a41da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E6=B8=85?= <62982788+597365581@users.noreply.github.com> Date: Sat, 30 Jan 2021 14:54:00 +0800 Subject: [PATCH 024/142] Update ShellCommandExecutorTest.java add unit test --- .../server/worker/shell/ShellCommandExecutorTest.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java index a59fdf8d03..1e93a0df39 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java @@ -121,9 +121,10 @@ public class ShellCommandExecutorTest { try { Object instance = shellCommandExecutorClass.newInstance(); - Method method = shellCommandExecutorClass.getDeclaredMethod("parseProcessOutput", new Class[]{}); + Method method = shellCommandExecutorClass.getDeclaredMethod("parseProcessOutput", new Class[]{Process.class}); method.setAccessible(true); - ShellCommandExecutor result = (ShellCommandExecutor) method.invoke(instance, new Object[]{}); + Object arg1s[] = {PowerMockito.mock(Process.class)}; + ShellCommandExecutor result = (ShellCommandExecutor) method.invoke(instance, arg1s); } catch (Exception e) { logger.error(e.getMessage()); } From 720b36f2847492ff1ff842f9b245ab5a45b9f4e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E6=B8=85?= <62982788+597365581@users.noreply.github.com> Date: Sat, 30 Jan 2021 14:55:41 +0800 Subject: [PATCH 025/142] Update ShellCommandExecutorTest.java add unit test --- .../server/worker/shell/ShellCommandExecutorTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java index 1e93a0df39..62df0dc11e 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java @@ -123,7 +123,7 @@ public class ShellCommandExecutorTest { Method method = shellCommandExecutorClass.getDeclaredMethod("parseProcessOutput", new Class[]{Process.class}); method.setAccessible(true); - Object arg1s[] = {PowerMockito.mock(Process.class)}; + Object[] arg1s = {PowerMockito.mock(Process.class)}; ShellCommandExecutor result = (ShellCommandExecutor) method.invoke(instance, arg1s); } catch (Exception e) { logger.error(e.getMessage()); From 658016731fc3a04e628da18f79e53501168e0972 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E6=B8=85?= <62982788+597365581@users.noreply.github.com> Date: Sat, 30 Jan 2021 15:19:00 +0800 Subject: [PATCH 026/142] Update ShellCommandExecutorTest.java add unit test --- .../shell/ShellCommandExecutorTest.java | 42 ++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java index 62df0dc11e..aa3d624fe7 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java @@ -123,7 +123,47 @@ public class ShellCommandExecutorTest { Method method = shellCommandExecutorClass.getDeclaredMethod("parseProcessOutput", new Class[]{Process.class}); method.setAccessible(true); - Object[] arg1s = {PowerMockito.mock(Process.class)}; + Object[] arg1s = {new Process() { + @Override + public OutputStream getOutputStream() { + return new OutputStream() { + @Override + public void write(int b) throws IOException { + + } + }; + } + + @Override + public InputStream getInputStream() { + return new InputStream() { + @Override + public int read() throws IOException { + return 0; + } + }; + } + + @Override + public InputStream getErrorStream() { + return null; + } + + @Override + public int waitFor() throws InterruptedException { + return 0; + } + + @Override + public int exitValue() { + return 0; + } + + @Override + public void destroy() { + + } + }}; ShellCommandExecutor result = (ShellCommandExecutor) method.invoke(instance, arg1s); } catch (Exception e) { logger.error(e.getMessage()); From daae917320eb82a7483014424c60af535d3e2a09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E6=B8=85?= <62982788+597365581@users.noreply.github.com> Date: Sat, 30 Jan 2021 15:22:09 +0800 Subject: [PATCH 027/142] Update ShellCommandExecutorTest.java add unit test --- .../server/worker/shell/ShellCommandExecutorTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java index aa3d624fe7..b562ee71bc 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java @@ -161,7 +161,7 @@ public class ShellCommandExecutorTest { @Override public void destroy() { - + logger.info("unit test"); } }}; ShellCommandExecutor result = (ShellCommandExecutor) method.invoke(instance, arg1s); From 6819def6e296011b2c7b02f51c9a7e3e0290c4a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E6=B8=85?= <62982788+597365581@users.noreply.github.com> Date: Sat, 30 Jan 2021 15:25:31 +0800 Subject: [PATCH 028/142] Update ShellCommandExecutorTest.java add unit test --- .../server/worker/shell/ShellCommandExecutorTest.java | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java index b562ee71bc..cdb76940a4 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java @@ -133,7 +133,6 @@ public class ShellCommandExecutorTest { } }; } - @Override public InputStream getInputStream() { return new InputStream() { @@ -143,25 +142,21 @@ public class ShellCommandExecutorTest { } }; } - @Override public InputStream getErrorStream() { return null; } - @Override public int waitFor() throws InterruptedException { return 0; } - @Override public int exitValue() { return 0; } - @Override public void destroy() { - logger.info("unit test"); + logger.info("unit test"); } }}; ShellCommandExecutor result = (ShellCommandExecutor) method.invoke(instance, arg1s); From d94b75624af70ba8d262a6687bcc9e3e6e8506f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E6=B8=85?= <62982788+597365581@users.noreply.github.com> Date: Sat, 30 Jan 2021 15:28:12 +0800 Subject: [PATCH 029/142] Update ShellCommandExecutorTest.java update unit test --- .../server/worker/shell/ShellCommandExecutorTest.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java index cdb76940a4..10778b85e0 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java @@ -133,6 +133,7 @@ public class ShellCommandExecutorTest { } }; } + @Override public InputStream getInputStream() { return new InputStream() { @@ -142,18 +143,22 @@ public class ShellCommandExecutorTest { } }; } + @Override public InputStream getErrorStream() { return null; } + @Override public int waitFor() throws InterruptedException { return 0; } + @Override public int exitValue() { return 0; } + @Override public void destroy() { logger.info("unit test"); From f79fb7d776a2a286d85a99fdf9b0e621f0432e60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E6=B8=85?= <62982788+597365581@users.noreply.github.com> Date: Sat, 30 Jan 2021 15:36:43 +0800 Subject: [PATCH 030/142] Update ShellCommandExecutorTest.java update unit test --- .../server/worker/shell/ShellCommandExecutorTest.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java index 10778b85e0..5a848dfb68 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java @@ -16,6 +16,9 @@ */ package org.apache.dolphinscheduler.server.worker.shell; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; import java.lang.reflect.Method; @@ -163,7 +166,7 @@ public class ShellCommandExecutorTest { public void destroy() { logger.info("unit test"); } - }}; + } }; ShellCommandExecutor result = (ShellCommandExecutor) method.invoke(instance, arg1s); } catch (Exception e) { logger.error(e.getMessage()); From 6ba290c4523883ba6c21ff946c879c15946d7495 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E6=B8=85?= <62982788+597365581@users.noreply.github.com> Date: Sat, 30 Jan 2021 16:06:20 +0800 Subject: [PATCH 031/142] Update ShellCommandExecutorTest.java update unit test --- .../shell/ShellCommandExecutorTest.java | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java index 5a848dfb68..3f65ec87f5 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java @@ -28,6 +28,7 @@ import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.utils.LoggerUtils; import org.apache.dolphinscheduler.common.utils.OSUtils; import org.apache.dolphinscheduler.dao.entity.TaskInstance; +import org.apache.dolphinscheduler.server.worker.task.AbstractCommandExecutor; import org.apache.dolphinscheduler.server.worker.task.AbstractTask; import org.apache.dolphinscheduler.server.worker.task.ShellCommandExecutor; import org.apache.dolphinscheduler.server.worker.task.TaskProps; @@ -172,4 +173,36 @@ public class ShellCommandExecutorTest { logger.error(e.getMessage()); } } + + @Test + public void testFindAppId(){ + Class shellCommandExecutorClass = ShellCommandExecutor.class; + try{ + Object instance = shellCommandExecutorClass.newInstance(); + + Method method = shellCommandExecutorClass.getDeclaredMethod("findAppId", new Class[]{String.class}); + method.setAccessible(true); + Object[] arg1s = {"11111"}; + ShellCommandExecutor result = (ShellCommandExecutor) method.invoke(instance, arg1s); + } + catch (Exception e){ + logger.error(e.getMessage()); + } + } + + @Test + public void testConvertFile2List(){ + Class shellCommandExecutorClass = AbstractCommandExecutor.class; + try{ + Object instance = shellCommandExecutorClass.newInstance(); + + Method method = shellCommandExecutorClass.getDeclaredMethod("convertFile2List", new Class[]{String.class}); + method.setAccessible(true); + Object[] arg1s = {"/opt/1.txt"}; + AbstractCommandExecutor result = (AbstractCommandExecutor) method.invoke(instance, arg1s); + } + catch (Exception e){ + logger.error(e.getMessage()); + } + } } From 3cba3e803c3f7bd82692f1b85066d0fabb1845a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E6=B8=85?= <62982788+597365581@users.noreply.github.com> Date: Sat, 30 Jan 2021 16:09:01 +0800 Subject: [PATCH 032/142] Update ShellCommandExecutorTest.java update unit test --- .../worker/shell/ShellCommandExecutorTest.java | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java index 3f65ec87f5..2d339e0b69 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java @@ -175,33 +175,31 @@ public class ShellCommandExecutorTest { } @Test - public void testFindAppId(){ + public void testFindAppId() { Class shellCommandExecutorClass = ShellCommandExecutor.class; - try{ + try { Object instance = shellCommandExecutorClass.newInstance(); Method method = shellCommandExecutorClass.getDeclaredMethod("findAppId", new Class[]{String.class}); method.setAccessible(true); Object[] arg1s = {"11111"}; ShellCommandExecutor result = (ShellCommandExecutor) method.invoke(instance, arg1s); - } - catch (Exception e){ + } catch (Exception e) { logger.error(e.getMessage()); } } @Test - public void testConvertFile2List(){ + public void testConvertFile2List() { Class shellCommandExecutorClass = AbstractCommandExecutor.class; - try{ + try { Object instance = shellCommandExecutorClass.newInstance(); Method method = shellCommandExecutorClass.getDeclaredMethod("convertFile2List", new Class[]{String.class}); method.setAccessible(true); Object[] arg1s = {"/opt/1.txt"}; AbstractCommandExecutor result = (AbstractCommandExecutor) method.invoke(instance, arg1s); - } - catch (Exception e){ + } catch (Exception e) { logger.error(e.getMessage()); } } From 6d9e496e9ce1809477712b1062840b09c1a5957e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E6=B8=85?= <62982788+597365581@users.noreply.github.com> Date: Mon, 1 Feb 2021 11:18:09 +0800 Subject: [PATCH 033/142] Update ShellCommandExecutorTest.java update unit test --- .../server/worker/shell/ShellCommandExecutorTest.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java index 2d339e0b69..5c4edeade0 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java @@ -121,7 +121,7 @@ public class ShellCommandExecutorTest { @Test public void testParseProcessOutput() { - Class shellCommandExecutorClass = ShellCommandExecutor.class; + Class shellCommandExecutorClass = AbstractCommandExecutor.class; try { Object instance = shellCommandExecutorClass.newInstance(); @@ -168,7 +168,7 @@ public class ShellCommandExecutorTest { logger.info("unit test"); } } }; - ShellCommandExecutor result = (ShellCommandExecutor) method.invoke(instance, arg1s); + AbstractCommandExecutor result = (AbstractCommandExecutor) method.invoke(instance, arg1s); } catch (Exception e) { logger.error(e.getMessage()); } @@ -176,14 +176,14 @@ public class ShellCommandExecutorTest { @Test public void testFindAppId() { - Class shellCommandExecutorClass = ShellCommandExecutor.class; + Class shellCommandExecutorClass = AbstractCommandExecutor.class; try { Object instance = shellCommandExecutorClass.newInstance(); Method method = shellCommandExecutorClass.getDeclaredMethod("findAppId", new Class[]{String.class}); method.setAccessible(true); Object[] arg1s = {"11111"}; - ShellCommandExecutor result = (ShellCommandExecutor) method.invoke(instance, arg1s); + AbstractCommandExecutor result = (AbstractCommandExecutor) method.invoke(instance, arg1s); } catch (Exception e) { logger.error(e.getMessage()); } From 32db201c29c7aa065b9538b6d7ec11eb6658e851 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E6=B8=85?= <62982788+597365581@users.noreply.github.com> Date: Mon, 1 Feb 2021 11:21:28 +0800 Subject: [PATCH 034/142] Update ShellCommandExecutorTest.java update unit test --- .../server/worker/shell/ShellCommandExecutorTest.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java index 5c4edeade0..23ef074ada 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java @@ -25,12 +25,10 @@ import java.lang.reflect.Method; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; -import org.apache.dolphinscheduler.common.utils.LoggerUtils; import org.apache.dolphinscheduler.common.utils.OSUtils; import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.server.worker.task.AbstractCommandExecutor; import org.apache.dolphinscheduler.server.worker.task.AbstractTask; -import org.apache.dolphinscheduler.server.worker.task.ShellCommandExecutor; import org.apache.dolphinscheduler.server.worker.task.TaskProps; import org.apache.dolphinscheduler.service.process.ProcessService; import org.junit.Before; From 310d3909c0a6767ebb90b843e35e5af3eced67bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E6=B8=85?= <62982788+597365581@users.noreply.github.com> Date: Mon, 1 Feb 2021 11:26:20 +0800 Subject: [PATCH 035/142] Update ShellCommandExecutorTest.java update unit test --- .../server/worker/shell/ShellCommandExecutorTest.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java index 23ef074ada..76e3d91967 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java @@ -20,11 +20,8 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.lang.reflect.Method; - - import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; - import org.apache.dolphinscheduler.common.utils.OSUtils; import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.server.worker.task.AbstractCommandExecutor; @@ -42,7 +39,6 @@ import org.powermock.modules.junit4.PowerMockRunner; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.context.ApplicationContext; - import java.util.Date; /** From 83bd0ce78b8df4e2ced7bc1fac553fa8f5888cef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E6=B8=85?= <62982788+597365581@users.noreply.github.com> Date: Mon, 1 Feb 2021 13:10:42 +0800 Subject: [PATCH 036/142] Update ShellCommandExecutorTest.java update unit test --- .../shell/ShellCommandExecutorTest.java | 70 ++++++++++++++++--- 1 file changed, 60 insertions(+), 10 deletions(-) diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java index 76e3d91967..fe22fe19ef 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java @@ -24,10 +24,14 @@ import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.utils.OSUtils; import org.apache.dolphinscheduler.dao.entity.TaskInstance; +import org.apache.dolphinscheduler.server.entity.TaskExecutionContext; import org.apache.dolphinscheduler.server.worker.task.AbstractCommandExecutor; import org.apache.dolphinscheduler.server.worker.task.AbstractTask; +import org.apache.dolphinscheduler.server.worker.task.ShellCommandExecutor; import org.apache.dolphinscheduler.server.worker.task.TaskProps; +import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; import org.apache.dolphinscheduler.service.process.ProcessService; +import org.junit.Assert; import org.junit.Before; import org.junit.Ignore; import org.junit.Test; @@ -40,6 +44,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.context.ApplicationContext; import java.util.Date; +import java.util.List; /** * python shell command executor test @@ -56,8 +61,11 @@ public class ShellCommandExecutorTest { @Before public void before() { - processService = PowerMockito.mock(ProcessService.class); applicationContext = PowerMockito.mock(ApplicationContext.class); + processService = PowerMockito.mock(ProcessService.class); + SpringApplicationContext springApplicationContext = new SpringApplicationContext(); + springApplicationContext.setApplicationContext(applicationContext); + PowerMockito.when(applicationContext.getBean(ProcessService.class)).thenReturn(processService); } @Ignore @@ -117,9 +125,8 @@ public class ShellCommandExecutorTest { public void testParseProcessOutput() { Class shellCommandExecutorClass = AbstractCommandExecutor.class; try { - Object instance = shellCommandExecutorClass.newInstance(); - Method method = shellCommandExecutorClass.getDeclaredMethod("parseProcessOutput", new Class[]{Process.class}); + Method method = shellCommandExecutorClass.getDeclaredMethod("parseProcessOutput", Process.class); method.setAccessible(true); Object[] arg1s = {new Process() { @Override @@ -162,7 +169,22 @@ public class ShellCommandExecutorTest { logger.info("unit test"); } } }; - AbstractCommandExecutor result = (AbstractCommandExecutor) method.invoke(instance, arg1s); + method.invoke(new AbstractCommandExecutor(null, new TaskExecutionContext(), logger) { + @Override + protected String buildCommandFilePath() { + return null; + } + + @Override + protected String commandInterpreter() { + return null; + } + + @Override + protected void createCommandFileIfNotExists(String execCommand, String commandFile) throws IOException { + + } + }, arg1s); } catch (Exception e) { logger.error(e.getMessage()); } @@ -172,12 +194,26 @@ public class ShellCommandExecutorTest { public void testFindAppId() { Class shellCommandExecutorClass = AbstractCommandExecutor.class; try { - Object instance = shellCommandExecutorClass.newInstance(); Method method = shellCommandExecutorClass.getDeclaredMethod("findAppId", new Class[]{String.class}); method.setAccessible(true); Object[] arg1s = {"11111"}; - AbstractCommandExecutor result = (AbstractCommandExecutor) method.invoke(instance, arg1s); + String result = (String) method.invoke(new AbstractCommandExecutor(null, null, null) { + @Override + protected String buildCommandFilePath() { + return null; + } + + @Override + protected String commandInterpreter() { + return null; + } + + @Override + protected void createCommandFileIfNotExists(String execCommand, String commandFile) throws IOException { + + } + }, arg1s); } catch (Exception e) { logger.error(e.getMessage()); } @@ -187,12 +223,26 @@ public class ShellCommandExecutorTest { public void testConvertFile2List() { Class shellCommandExecutorClass = AbstractCommandExecutor.class; try { - Object instance = shellCommandExecutorClass.newInstance(); - - Method method = shellCommandExecutorClass.getDeclaredMethod("convertFile2List", new Class[]{String.class}); + Method method = shellCommandExecutorClass.getDeclaredMethod("convertFile2List", String.class); method.setAccessible(true); Object[] arg1s = {"/opt/1.txt"}; - AbstractCommandExecutor result = (AbstractCommandExecutor) method.invoke(instance, arg1s); + List result = (List) method.invoke(new AbstractCommandExecutor(null, null, null) { + @Override + protected String buildCommandFilePath() { + return null; + } + + @Override + protected String commandInterpreter() { + return null; + } + + @Override + protected void createCommandFileIfNotExists(String execCommand, String commandFile) throws IOException { + + } + }, arg1s); + Assert.assertTrue(true); } catch (Exception e) { logger.error(e.getMessage()); } From d48e035362dfc9adafb55d44009582615e8a110d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E6=B8=85?= <62982788+597365581@users.noreply.github.com> Date: Mon, 1 Feb 2021 13:16:40 +0800 Subject: [PATCH 037/142] Update ShellCommandExecutorTest.java update unit test --- .../worker/shell/ShellCommandExecutorTest.java | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java index fe22fe19ef..c1c3954df6 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java @@ -27,7 +27,6 @@ import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.server.entity.TaskExecutionContext; import org.apache.dolphinscheduler.server.worker.task.AbstractCommandExecutor; import org.apache.dolphinscheduler.server.worker.task.AbstractTask; -import org.apache.dolphinscheduler.server.worker.task.ShellCommandExecutor; import org.apache.dolphinscheduler.server.worker.task.TaskProps; import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; import org.apache.dolphinscheduler.service.process.ProcessService; @@ -134,7 +133,7 @@ public class ShellCommandExecutorTest { return new OutputStream() { @Override public void write(int b) throws IOException { - + logger.info("unit test"); } }; } @@ -182,9 +181,8 @@ public class ShellCommandExecutorTest { @Override protected void createCommandFileIfNotExists(String execCommand, String commandFile) throws IOException { - - } - }, arg1s); + logger.info("unit test"); + } }, arg1s); } catch (Exception e) { logger.error(e.getMessage()); } @@ -211,9 +209,8 @@ public class ShellCommandExecutorTest { @Override protected void createCommandFileIfNotExists(String execCommand, String commandFile) throws IOException { - - } - }, arg1s); + logger.info("unit test"); + } }, arg1s); } catch (Exception e) { logger.error(e.getMessage()); } @@ -239,9 +236,8 @@ public class ShellCommandExecutorTest { @Override protected void createCommandFileIfNotExists(String execCommand, String commandFile) throws IOException { - - } - }, arg1s); + logger.info("unit test"); + } }, arg1s); Assert.assertTrue(true); } catch (Exception e) { logger.error(e.getMessage()); From e383ef4351662bcaf7cae0d97a2a4340887b2710 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E6=B8=85?= <62982788+597365581@users.noreply.github.com> Date: Mon, 1 Feb 2021 13:23:30 +0800 Subject: [PATCH 038/142] Update ShellCommandExecutorTest.java update unit test --- .../worker/shell/ShellCommandExecutorTest.java | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java index c1c3954df6..3b22307f0c 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java @@ -20,6 +20,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.lang.reflect.Method; + import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.utils.OSUtils; @@ -30,6 +31,7 @@ import org.apache.dolphinscheduler.server.worker.task.AbstractTask; import org.apache.dolphinscheduler.server.worker.task.TaskProps; import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; import org.apache.dolphinscheduler.service.process.ProcessService; + import org.junit.Assert; import org.junit.Before; import org.junit.Ignore; @@ -42,6 +44,7 @@ import org.powermock.modules.junit4.PowerMockRunner; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.context.ApplicationContext; + import java.util.Date; import java.util.List; @@ -167,8 +170,8 @@ public class ShellCommandExecutorTest { public void destroy() { logger.info("unit test"); } - } }; - method.invoke(new AbstractCommandExecutor(null, new TaskExecutionContext(), logger) { + }}; + method.invoke(new AbstractCommandExecutor(null, new TaskExecutionContext(), logger) { @Override protected String buildCommandFilePath() { return null; @@ -182,7 +185,8 @@ public class ShellCommandExecutorTest { @Override protected void createCommandFileIfNotExists(String execCommand, String commandFile) throws IOException { logger.info("unit test"); - } }, arg1s); + } + }, arg1s); } catch (Exception e) { logger.error(e.getMessage()); } @@ -210,7 +214,8 @@ public class ShellCommandExecutorTest { @Override protected void createCommandFileIfNotExists(String execCommand, String commandFile) throws IOException { logger.info("unit test"); - } }, arg1s); + } + }, arg1s); } catch (Exception e) { logger.error(e.getMessage()); } @@ -237,7 +242,8 @@ public class ShellCommandExecutorTest { @Override protected void createCommandFileIfNotExists(String execCommand, String commandFile) throws IOException { logger.info("unit test"); - } }, arg1s); + } + }, arg1s); Assert.assertTrue(true); } catch (Exception e) { logger.error(e.getMessage()); From 833c06b9d41dd710075aa7123f1d238f981ab036 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E6=B8=85?= <62982788+597365581@users.noreply.github.com> Date: Mon, 1 Feb 2021 13:26:25 +0800 Subject: [PATCH 039/142] Update ShellCommandExecutorTest.java update unit test --- .../server/worker/shell/ShellCommandExecutorTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java index 3b22307f0c..ac91e1d9b8 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/shell/ShellCommandExecutorTest.java @@ -170,7 +170,7 @@ public class ShellCommandExecutorTest { public void destroy() { logger.info("unit test"); } - }}; + } }; method.invoke(new AbstractCommandExecutor(null, new TaskExecutionContext(), logger) { @Override protected String buildCommandFilePath() { From 6e713a3832fcf7d3180c20dffd19c9646f170b23 Mon Sep 17 00:00:00 2001 From: zhuangchong <37063904+zhuangchong@users.noreply.github.com> Date: Thu, 4 Feb 2021 23:16:51 +0800 Subject: [PATCH 040/142] fix sed command execution abnormal in the install file. (#4685) --- install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install.sh b/install.sh index 7991275a01..723ee908e8 100755 --- a/install.sh +++ b/install.sh @@ -53,7 +53,7 @@ sed -i ${txt} "s#java.security.krb5.conf.path.*#java.security.krb5.conf.path=${k sed -i ${txt} "s#login.user.keytab.username.*#login.user.keytab.username=${keytabUserName}#g" conf/common.properties sed -i ${txt} "s#login.user.keytab.path.*#login.user.keytab.path=${keytabPath}#g" conf/common.properties sed -i ${txt} "s#zookeeper.quorum.*#zookeeper.quorum=${zkQuorum}#g" conf/zookeeper.properties -sed -i ${txt} "s\#zookeeper.dolphinscheduler.root.*#zookeeper.dolphinscheduler.root=${zkRoot}#g" conf/zookeeper.properties +sed -i ${txt} "s#\#zookeeper.dolphinscheduler.root.*#zookeeper.dolphinscheduler.root=${zkRoot}#g" conf/zookeeper.properties sed -i ${txt} "s#server.port.*#server.port=${apiServerPort}#g" conf/application-api.properties sed -i ${txt} "s#mail.server.host.*#mail.server.host=${mailServerHost}#g" conf/alert.properties sed -i ${txt} "s#mail.server.port.*#mail.server.port=${mailServerPort}#g" conf/alert.properties From b3bb84631ec268a9ab7e19ebb6c6bfa3c092f7f5 Mon Sep 17 00:00:00 2001 From: Shiwen Cheng Date: Thu, 4 Feb 2021 23:21:44 +0800 Subject: [PATCH 041/142] [Fix][Flink] Fix flink -yn option missing after upgrading (#4682) --- .../apache/dolphinscheduler/server/utils/FlinkArgsUtils.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/FlinkArgsUtils.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/FlinkArgsUtils.java index 2431eedd16..519ddf205c 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/FlinkArgsUtils.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/FlinkArgsUtils.java @@ -64,9 +64,9 @@ public class FlinkArgsUtils { args.add(ArgsUtils.escape(appName)); } - // judge flink version,from flink1.10,the parameter -yn removed + // judge flink version, the parameter -yn has removed from flink 1.10 String flinkVersion = param.getFlinkVersion(); - if (FLINK_VERSION_BEFORE_1_10.equals(flinkVersion)) { + if (flinkVersion == null || FLINK_VERSION_BEFORE_1_10.equals(flinkVersion)) { int taskManager = param.getTaskManager(); if (taskManager != 0) { //-yn args.add(Constants.FLINK_TASK_MANAGE); From 9a3853424ba66f411e99e1954ba596fcc1d1a370 Mon Sep 17 00:00:00 2001 From: kalencaya <1942460489@qq.com> Date: Fri, 5 Feb 2021 11:27:19 +0800 Subject: [PATCH 042/142] Improvement/UI no #3725 (#4316) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 工作流定义，工作流实例，任务实例的编号设置为id * Change the no to taskinstance, processinstance, processdefinition id --- .../pages/projects/pages/definition/pages/list/_source/list.vue | 2 +- .../pages/projects/pages/instance/pages/list/_source/list.vue | 2 +- .../home/pages/projects/pages/taskInstance/_source/list.vue | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/list.vue b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/list.vue index 32612213ac..e148119b98 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/list.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/list.vue @@ -19,7 +19,7 @@
- +