diff --git a/README.md b/README.md index b52e00ef1f..faa7eefb4d 100644 --- a/README.md +++ b/README.md @@ -87,7 +87,7 @@ We would like to express our deep gratitude to all the open-source projects used ## Community You are very welcome to communicate with the developers and users of Dolphin Scheduler. There are two ways to find them: -1. Join the Slack channel by [this invitation link](https://join.slack.com/t/asf-dolphinscheduler/shared_invite/zt-mzqu52gi-rCggPkSHQ0DZYkwbTxO1Gw). +1. Join the Slack channel by [this invitation link](https://join.slack.com/t/asf-dolphinscheduler/shared_invite/zt-omtdhuio-_JISsxYhiVsltmC5h38yfw). 2. Follow the [Twitter account of Dolphin Scheduler](https://twitter.com/dolphinschedule) and get the latest news on time. ## How to Contribute diff --git a/README_zh_CN.md b/README_zh_CN.md index abd3d378f4..502bff55be 100644 --- a/README_zh_CN.md +++ b/README_zh_CN.md @@ -92,7 +92,7 @@ Dolphin Scheduler使用了很多优秀的开源项目，比如google的guava、g ## 社区 -1. 通过[该申请链接](https://join.slack.com/t/asf-dolphinscheduler/shared_invite/zt-mzqu52gi-rCggPkSHQ0DZYkwbTxO1Gw)加入slack channel +1. 通过[该申请链接](https://join.slack.com/t/asf-dolphinscheduler/shared_invite/zt-omtdhuio-_JISsxYhiVsltmC5h38yfw)加入slack channel 2. 关注[Apache Dolphin Scheduler的Twitter账号](https://twitter.com/dolphinschedule)获取实时动态 ## 版权 diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java index 0270d8fdbe..736eb83f04 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java @@ -155,6 +155,11 @@ public final class Constants { public static final String DEVELOPMENT_STATE = "development.state"; public static final String DEVELOPMENT_STATE_DEFAULT_VALUE = "true"; + /** + * sudo enable + */ + public static final String SUDO_ENABLE = "sudo.enable"; + /** * string true */ diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CommonUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CommonUtils.java index cf307b402f..e68c342ea4 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CommonUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CommonUtils.java @@ -68,6 +68,13 @@ public class CommonUtils { return PropertyUtils.getBoolean(Constants.DEVELOPMENT_STATE, true); } + /** + * @return sudo enable + */ + public static boolean isSudoEnable() { + return PropertyUtils.getBoolean(Constants.SUDO_ENABLE, true); + } + /** * if upload resource is HDFS and kerberos startup is true , else false * diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java index 0dcfbddaf4..ae6291a854 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java @@ -121,27 +121,8 @@ public class FileUtils { * @return directory of process execution */ public static String getProcessExecDir(int projectId, int processDefineId, int processInstanceId, int taskInstanceId) { - String fileName = String.format("%s/exec/process/%s/%s/%s/%s", DATA_BASEDIR, Integer.toString(projectId), - Integer.toString(processDefineId), Integer.toString(processInstanceId), Integer.toString(taskInstanceId)); - File file = new File(fileName); - if (!file.getParentFile().exists()) { - file.getParentFile().mkdirs(); - } - - return fileName; - } - - /** - * directory of process instances - * - * @param projectId project id - * @param processDefineId process definition id - * @param processInstanceId process instance id - * @return directory of process instances - */ - public static String getProcessExecDir(int projectId, int processDefineId, int processInstanceId) { - String fileName = String.format("%s/exec/process/%s/%s/%s", DATA_BASEDIR, Integer.toString(projectId), - Integer.toString(processDefineId), Integer.toString(processInstanceId)); + String fileName = String.format("%s/exec/process/%d/%d/%d/%d", DATA_BASEDIR, + projectId, processDefineId, processInstanceId, taskInstanceId); File file = new File(fileName); if (!file.getParentFile().exists()) { file.getParentFile().mkdirs(); diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java index 92a06fe568..09b4e06d9f 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java @@ -22,12 +22,12 @@ import org.apache.dolphinscheduler.common.shell.ShellExecutor; import org.apache.commons.configuration.Configuration; -import java.lang.management.OperatingSystemMXBean; import java.io.BufferedReader; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStreamReader; import java.lang.management.ManagementFactory; +import java.lang.management.OperatingSystemMXBean; import java.lang.management.RuntimeMXBean; import java.math.RoundingMode; import java.text.DecimalFormat; @@ -406,13 +406,17 @@ public class OSUtils { } /** - * get sudo command + * get sudo command + * * @param tenantCode tenantCode * @param command command * @return result of sudo execute command */ public static String getSudoCmd(String tenantCode, String command) { - return StringUtils.isEmpty(tenantCode) ? command : "sudo -u " + tenantCode + " " + command; + if (!CommonUtils.isSudoEnable() || StringUtils.isEmpty(tenantCode)) { + return command; + } + return String.format("sudo -u %s %s", tenantCode, command); } /** diff --git a/dolphinscheduler-common/src/main/resources/common.properties b/dolphinscheduler-common/src/main/resources/common.properties index b3d9156c76..6aad27146d 100644 --- a/dolphinscheduler-common/src/main/resources/common.properties +++ b/dolphinscheduler-common/src/main/resources/common.properties @@ -78,3 +78,6 @@ datasource.encryption.salt=!@#$%^&* #dolphin.scheduler.network.priority.strategy=default # 0 getGlobalParamsMap() { - Map globalParamsMap = new HashMap<>(16); + Map globalParamsMap = new HashMap<>(16); // global params string String globalParamsStr = taskExecutionContext.getGlobalParams(); @@ -241,7 +272,7 @@ public class TaskExecuteThread implements Runnable, Delayed { } /** - * kill task + * kill task */ public void kill() { if (task != null) { @@ -261,7 +292,7 @@ public class TaskExecuteThread implements Runnable, Delayed { * @param logger */ private void downloadResource(String execLocalPath, - Map projectRes, + Map projectRes, Logger logger) throws Exception { if (MapUtils.isEmpty(projectRes)) { return; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java index 73f2e700f0..88af2d7f2c 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java @@ -25,6 +25,7 @@ import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.thread.Stopper; import org.apache.dolphinscheduler.common.thread.ThreadUtils; +import org.apache.dolphinscheduler.common.utils.CommonUtils; import org.apache.dolphinscheduler.common.utils.HadoopUtils; import org.apache.dolphinscheduler.common.utils.LoggerUtils; import org.apache.dolphinscheduler.common.utils.OSUtils; @@ -84,7 +85,7 @@ public abstract class AbstractCommandExecutor { * log list */ protected final List logBuffer; - + protected boolean logOutputIsScuccess = false; /** @@ -134,9 +135,11 @@ public abstract class AbstractCommandExecutor { processBuilder.redirectErrorStream(true); // setting up user to run commands - command.add("sudo"); - command.add("-u"); - command.add(taskExecutionContext.getTenantCode()); + if (CommonUtils.isSudoEnable()) { + command.add("sudo"); + command.add("-u"); + command.add(taskExecutionContext.getTenantCode()); + } command.add(commandInterpreter()); command.addAll(commandOptions()); command.add(commandFile); @@ -289,7 +292,7 @@ public abstract class AbstractCommandExecutor { } } - return process.isAlive(); + return !process.isAlive(); } /** @@ -595,4 +598,4 @@ public abstract class AbstractCommandExecutor { public void setTaskResultString(String taskResultString) { this.taskResultString = taskResultString; } -} \ No newline at end of file +} diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/MasterExecThreadTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/MasterExecThreadTest.java index 6979a939e7..cdea2526af 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/MasterExecThreadTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/MasterExecThreadTest.java @@ -102,16 +102,12 @@ public class MasterExecThreadTest { processDefinition.setGlobalParamList(Collections.EMPTY_LIST); Mockito.when(processInstance.getProcessDefinition()).thenReturn(processDefinition); - masterExecThread = PowerMockito.spy(new MasterExecThread( - processInstance - , processService - , null, null, config)); + masterExecThread = PowerMockito.spy(new MasterExecThread(processInstance, processService, null, null, config)); // prepareProcess init dag Field dag = MasterExecThread.class.getDeclaredField("dag"); dag.setAccessible(true); dag.set(masterExecThread, new DAG()); PowerMockito.doNothing().when(masterExecThread, "executeProcess"); - PowerMockito.doNothing().when(masterExecThread, "postHandle"); PowerMockito.doNothing().when(masterExecThread, "prepareProcess"); PowerMockito.doNothing().when(masterExecThread, "runProcess"); PowerMockito.doNothing().when(masterExecThread, "endProcess"); diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/config/MasterConfigTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/config/MasterConfigTest.java new file mode 100644 index 0000000000..43d2469275 --- /dev/null +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/config/MasterConfigTest.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.server.master.config; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; + +@RunWith(SpringJUnit4ClassRunner.class) +@ContextConfiguration(classes = {MasterConfig.class}) +public class MasterConfigTest { + + @Autowired + private MasterConfig masterConfig; + + @Test + public void getMasterDispatchTaskNumber() { + int masterDispatchTaskNumber = masterConfig.getMasterDispatchTaskNumber(); + Assert.assertEquals(6, masterDispatchTaskNumber); + } +} \ No newline at end of file diff --git a/dolphinscheduler-server/src/test/resources/master.properties b/dolphinscheduler-server/src/test/resources/master.properties new file mode 100644 index 0000000000..91d84ebd10 --- /dev/null +++ b/dolphinscheduler-server/src/test/resources/master.properties @@ -0,0 +1,43 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# master execute thread num +#master.exec.threads=100 + +# master execute task number in parallel +#master.exec.task.num=20 + +# master dispatch task number +master.dispatch.task.num=6 + +# master heartbeat interval +#master.heartbeat.interval=10 + +# master commit task retry times +#master.task.commit.retryTimes=5 + +# master commit task interval +#master.task.commit.interval=1000 + +# only less than cpu avg load, master server can work. default value -1 : the number of cpu cores * 2 +#master.max.cpuload.avg=-1 + +# only larger than reserved memory, master server can work. default value : physical memory * 1/10, unit is G. +#master.reserved.memory=0.3 + +# master listen port +#master.listen.port=5678 diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/instance/pages/gantt/index.vue b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/instance/pages/gantt/index.vue index bcd2ab6996..c9389485d8 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/instance/pages/gantt/index.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/instance/pages/gantt/index.vue @@ -32,7 +32,7 @@ - - -