From 6f6554ab9c0c74e8010c756d0200377d37931fbf Mon Sep 17 00:00:00 2001 From: liwenhe1993 Date: Fri, 29 May 2020 15:48:56 +0800 Subject: [PATCH] Fix errors when running on win10 (#2842) 1. DS cant run bat script 2. DS cant run python script 3. DS cant run datax script --- .../worker/task/AbstractCommandExecutor.java | 83 ++++++++++++++----- .../worker/task/ShellCommandExecutor.java | 7 +- .../server/worker/task/datax/DataxTask.java | 5 +- .../server/worker/task/shell/ShellTask.java | 2 +- 4 files changed, 69 insertions(+), 28 deletions(-) diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java index 7224d349f5..3459e35b72 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java @@ -16,32 +16,39 @@ */ package org.apache.dolphinscheduler.server.worker.task; +import com.sun.jna.platform.win32.Kernel32; +import com.sun.jna.platform.win32.WinNT; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.thread.Stopper; import org.apache.dolphinscheduler.common.thread.ThreadUtils; import org.apache.dolphinscheduler.common.utils.HadoopUtils; -import org.apache.dolphinscheduler.common.utils.StringUtils; import org.apache.dolphinscheduler.common.utils.LoggerUtils; +import org.apache.dolphinscheduler.common.utils.OSUtils; +import org.apache.dolphinscheduler.common.utils.StringUtils; +import org.apache.dolphinscheduler.common.utils.process.ProcessBuilderForWin32; import org.apache.dolphinscheduler.server.entity.TaskExecutionContext; import org.apache.dolphinscheduler.server.utils.ProcessUtils; import org.apache.dolphinscheduler.server.worker.cache.TaskExecutionContextCacheManager; import org.apache.dolphinscheduler.server.worker.cache.impl.TaskExecutionContextCacheManagerImpl; import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; - import org.slf4j.Logger; import java.io.*; import java.lang.reflect.Field; import java.nio.charset.StandardCharsets; -import java.util.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedList; +import java.util.List; import java.util.concurrent.ExecutorService; import java.util.concurrent.TimeUnit; import java.util.function.Consumer; import java.util.regex.Matcher; import java.util.regex.Pattern; -import static org.apache.dolphinscheduler.common.Constants.*; +import static org.apache.dolphinscheduler.common.Constants.EXIT_CODE_FAILURE; +import static org.apache.dolphinscheduler.common.Constants.EXIT_CODE_SUCCESS; /** * abstract command executor @@ -99,26 +106,51 @@ public abstract class AbstractCommandExecutor { * @throws IOException IO Exception */ private void buildProcess(String commandFile) throws IOException { - //init process builder - ProcessBuilder processBuilder = new ProcessBuilder(); - // setting up a working directory - processBuilder.directory(new File(taskExecutionContext.getExecutePath())); - // merge error information to standard output stream - processBuilder.redirectErrorStream(true); // setting up user to run commands List command = new LinkedList<>(); - command.add("sudo"); - command.add("-u"); - command.add(taskExecutionContext.getTenantCode()); - command.add(commandInterpreter()); - command.addAll(commandOptions()); - command.add(commandFile); - processBuilder.command(command); - process = processBuilder.start(); + if (OSUtils.isWindows()) { + //init process builder + ProcessBuilderForWin32 processBuilder = new ProcessBuilderForWin32(); + // setting up a working directory + processBuilder.directory(new File(taskExecutionContext.getExecutePath())); + // setting up a username and password + processBuilder.user(taskExecutionContext.getTenantCode(), StringUtils.EMPTY); + // merge error information to standard output stream + processBuilder.redirectErrorStream(true); + + // setting up user to run commands + command.add(commandInterpreter()); + command.add("/c"); + command.addAll(commandOptions()); + command.add(commandFile); + + // setting commands + processBuilder.command(command); + process = processBuilder.start(); + } else { + //init process builder + ProcessBuilder processBuilder = new ProcessBuilder(); + // setting up a working directory + processBuilder.directory(new File(taskExecutionContext.getExecutePath())); + // merge error information to standard output stream + processBuilder.redirectErrorStream(true); + + // setting up user to run commands + command.add("sudo"); + command.add("-u"); + command.add(taskExecutionContext.getTenantCode()); + command.add(commandInterpreter()); + command.addAll(commandOptions()); + command.add(commandFile); + + // setting commands + processBuilder.command(command); + process = processBuilder.start(); + } // print command - printCommand(processBuilder); + printCommand(command); } /** @@ -270,13 +302,13 @@ public abstract class AbstractCommandExecutor { /** * print command - * @param processBuilder process builder + * @param commands process builder */ - private void printCommand(ProcessBuilder processBuilder) { + private void printCommand(List commands) { String cmdStr; try { - cmdStr = ProcessUtils.buildCommandStr(processBuilder.command()); + cmdStr = ProcessUtils.buildCommandStr(commands); logger.info("task run command:\n{}", cmdStr); } catch (IOException e) { logger.error(e.getMessage(), e); @@ -465,7 +497,12 @@ public abstract class AbstractCommandExecutor { Field f = process.getClass().getDeclaredField(Constants.PID); f.setAccessible(true); - processId = f.getInt(process); + if (OSUtils.isWindows()) { + WinNT.HANDLE handle = (WinNT.HANDLE) f.get(process); + processId = Kernel32.INSTANCE.GetProcessId(handle); + } else { + processId = f.getInt(process); + } } catch (Throwable e) { logger.error(e.getMessage(), e); } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/ShellCommandExecutor.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/ShellCommandExecutor.java index 21418104a0..5e297abbf0 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/ShellCommandExecutor.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/ShellCommandExecutor.java @@ -17,8 +17,8 @@ package org.apache.dolphinscheduler.server.worker.task; import org.apache.commons.io.FileUtils; -import org.apache.dolphinscheduler.server.entity.TaskExecutionContext; import org.apache.dolphinscheduler.common.utils.OSUtils; +import org.apache.dolphinscheduler.server.entity.TaskExecutionContext; import org.slf4j.Logger; import java.io.File; @@ -60,7 +60,10 @@ public class ShellCommandExecutor extends AbstractCommandExecutor { @Override protected String buildCommandFilePath() { // command file - return String.format("%s/%s.command", taskExecutionContext.getExecutePath(), taskExecutionContext.getTaskAppId()); + return String.format("%s/%s.%s" + , taskExecutionContext.getExecutePath() + , taskExecutionContext.getTaskAppId() + , OSUtils.isWindows() ? "bat" : "command"); } /** diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java index 218906d91c..9589a305c1 100755 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/datax/DataxTask.java @@ -362,9 +362,10 @@ public class DataxTask extends AbstractTask { private String buildShellCommandFile(String jobConfigFilePath) throws Exception { // generate scripts - String fileName = String.format("%s/%s_node.sh", + String fileName = String.format("%s/%s_node.%s", taskExecutionContext.getExecutePath(), - taskExecutionContext.getTaskAppId()); + taskExecutionContext.getTaskAppId(), + OSUtils.isWindows() ? "bat" : "sh"); Path path = new File(fileName).toPath(); diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTask.java index f24aa54ffb..075be60690 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/shell/ShellTask.java @@ -116,7 +116,7 @@ public class ShellTask extends AbstractTask { */ private String buildCommand() throws Exception { // generate scripts - String fileName = String.format("%s/%s_node.sh", + String fileName = String.format("%s/%s_node.%s", taskExecutionContext.getExecutePath(), taskExecutionContext.getTaskAppId(), OSUtils.isWindows() ? "bat" : "sh");