[FIX-#3177]Task time parameter parsing error (#4224)
* [FIX-#3177]Task time parameter parsing error rerun schedule time error this closes #3177 * [FIX-#3177]Task time parameter parsing error rerun schedule time error this closes #3177
This commit is contained in:
Kirs
GitHub
parent
95d320d7da
commit
8919e3838f
|
|
@ -111,7 +111,9 @@ public class ParameterUtils {
|
|||
}
|
||||
|
||||
// replace variable ${} form,refers to the replacement of system variables and custom variables
|
||||
parameterString = PlaceholderUtils.replacePlaceholders(parameterString, parameterMap, true);
|
||||
if (!parameterMap.isEmpty()) {
|
||||
parameterString = PlaceholderUtils.replacePlaceholders(parameterString, parameterMap, true);
|
||||
}
|
||||
|
||||
// replace time $[...] form, eg. $[yyyyMMdd]
|
||||
if (cronTime != null) {
|
||||
|
|
|
|||
|
|
@ -14,8 +14,8 @@
|
|||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.dolphinscheduler.server.worker.task.shell;
|
||||
|
||||
package org.apache.dolphinscheduler.server.worker.task.shell;
|
||||
|
||||
import org.apache.dolphinscheduler.common.Constants;
|
||||
import org.apache.dolphinscheduler.common.enums.CommandType;
|
||||
|
|
@ -23,7 +23,7 @@ import org.apache.dolphinscheduler.common.process.Property;
|
|||
import org.apache.dolphinscheduler.common.task.AbstractParameters;
|
||||
import org.apache.dolphinscheduler.common.task.shell.ShellParameters;
|
||||
import org.apache.dolphinscheduler.common.utils.DateUtils;
|
||||
import org.apache.dolphinscheduler.common.utils.*;
|
||||
import org.apache.dolphinscheduler.common.utils.JSONUtils;
|
||||
import org.apache.dolphinscheduler.common.utils.OSUtils;
|
||||
import org.apache.dolphinscheduler.common.utils.ParameterUtils;
|
||||
import org.apache.dolphinscheduler.server.entity.TaskExecutionContext;
|
||||
|
|
@ -31,7 +31,6 @@ import org.apache.dolphinscheduler.server.utils.ParamUtils;
|
|||
import org.apache.dolphinscheduler.server.worker.task.AbstractTask;
|
||||
import org.apache.dolphinscheduler.server.worker.task.CommandExecuteResult;
|
||||
import org.apache.dolphinscheduler.server.worker.task.ShellCommandExecutor;
|
||||
import org.slf4j.Logger;
|
||||
|
||||
import java.io.File;
|
||||
import java.nio.file.Files;
|
||||
|
|
@ -43,136 +42,136 @@ import java.nio.file.attribute.PosixFilePermissions;
|
|||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
|
||||
/**
|
||||
* shell task
|
||||
*/
|
||||
public class ShellTask extends AbstractTask {
|
||||
|
||||
/**
|
||||
* shell parameters
|
||||
*/
|
||||
private ShellParameters shellParameters;
|
||||
/**
|
||||
* shell parameters
|
||||
*/
|
||||
private ShellParameters shellParameters;
|
||||
|
||||
/**
|
||||
* shell command executor
|
||||
*/
|
||||
private ShellCommandExecutor shellCommandExecutor;
|
||||
/**
|
||||
* shell command executor
|
||||
*/
|
||||
private ShellCommandExecutor shellCommandExecutor;
|
||||
|
||||
/**
|
||||
* taskExecutionContext
|
||||
*/
|
||||
private TaskExecutionContext taskExecutionContext;
|
||||
/**
|
||||
* taskExecutionContext
|
||||
*/
|
||||
private TaskExecutionContext taskExecutionContext;
|
||||
|
||||
/**
|
||||
* constructor
|
||||
* @param taskExecutionContext taskExecutionContext
|
||||
* @param logger logger
|
||||
*/
|
||||
public ShellTask(TaskExecutionContext taskExecutionContext, Logger logger) {
|
||||
super(taskExecutionContext, logger);
|
||||
/**
|
||||
* constructor
|
||||
*
|
||||
* @param taskExecutionContext taskExecutionContext
|
||||
* @param logger logger
|
||||
*/
|
||||
public ShellTask(TaskExecutionContext taskExecutionContext, Logger logger) {
|
||||
super(taskExecutionContext, logger);
|
||||
|
||||
this.taskExecutionContext = taskExecutionContext;
|
||||
this.shellCommandExecutor = new ShellCommandExecutor(this::logHandle,
|
||||
this.taskExecutionContext = taskExecutionContext;
|
||||
this.shellCommandExecutor = new ShellCommandExecutor(this::logHandle,
|
||||
taskExecutionContext,
|
||||
logger);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void init() {
|
||||
logger.info("shell task params {}", taskExecutionContext.getTaskParams());
|
||||
|
||||
shellParameters = JSONUtils.parseObject(taskExecutionContext.getTaskParams(), ShellParameters.class);
|
||||
|
||||
if (!shellParameters.checkParameters()) {
|
||||
throw new RuntimeException("shell task params is not valid");
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void handle() throws Exception {
|
||||
try {
|
||||
// construct process
|
||||
CommandExecuteResult commandExecuteResult = shellCommandExecutor.run(buildCommand());
|
||||
setExitStatusCode(commandExecuteResult.getExitStatusCode());
|
||||
setAppIds(commandExecuteResult.getAppIds());
|
||||
setProcessId(commandExecuteResult.getProcessId());
|
||||
} catch (Exception e) {
|
||||
logger.error("shell task error", e);
|
||||
setExitStatusCode(Constants.EXIT_CODE_FAILURE);
|
||||
throw e;
|
||||
@Override
|
||||
public void init() {
|
||||
logger.info("shell task params {}", taskExecutionContext.getTaskParams());
|
||||
|
||||
shellParameters = JSONUtils.parseObject(taskExecutionContext.getTaskParams(), ShellParameters.class);
|
||||
|
||||
if (!shellParameters.checkParameters()) {
|
||||
throw new RuntimeException("shell task params is not valid");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void cancelApplication(boolean cancelApplication) throws Exception {
|
||||
// cancel process
|
||||
shellCommandExecutor.cancelApplication();
|
||||
}
|
||||
@Override
|
||||
public void handle() throws Exception {
|
||||
try {
|
||||
// construct process
|
||||
CommandExecuteResult commandExecuteResult = shellCommandExecutor.run(buildCommand());
|
||||
setExitStatusCode(commandExecuteResult.getExitStatusCode());
|
||||
setAppIds(commandExecuteResult.getAppIds());
|
||||
setProcessId(commandExecuteResult.getProcessId());
|
||||
} catch (Exception e) {
|
||||
logger.error("shell task error", e);
|
||||
setExitStatusCode(Constants.EXIT_CODE_FAILURE);
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* create command
|
||||
* @return file name
|
||||
* @throws Exception exception
|
||||
*/
|
||||
private String buildCommand() throws Exception {
|
||||
// generate scripts
|
||||
String fileName = String.format("%s/%s_node.%s",
|
||||
@Override
|
||||
public void cancelApplication(boolean cancelApplication) throws Exception {
|
||||
// cancel process
|
||||
shellCommandExecutor.cancelApplication();
|
||||
}
|
||||
|
||||
/**
|
||||
* create command
|
||||
*
|
||||
* @return file name
|
||||
* @throws Exception exception
|
||||
*/
|
||||
private String buildCommand() throws Exception {
|
||||
// generate scripts
|
||||
String fileName = String.format("%s/%s_node.%s",
|
||||
taskExecutionContext.getExecutePath(),
|
||||
taskExecutionContext.getTaskAppId(), OSUtils.isWindows() ? "bat" : "sh");
|
||||
|
||||
Path path = new File(fileName).toPath();
|
||||
Path path = new File(fileName).toPath();
|
||||
|
||||
if (Files.exists(path)) {
|
||||
return fileName;
|
||||
}
|
||||
if (Files.exists(path)) {
|
||||
return fileName;
|
||||
}
|
||||
|
||||
String script = shellParameters.getRawScript().replaceAll("\\r\\n", "\n");
|
||||
/**
|
||||
* combining local and global parameters
|
||||
*/
|
||||
Map<String, Property> paramsMap = ParamUtils.convert(ParamUtils.getUserDefParamsMap(taskExecutionContext.getDefinedParams()),
|
||||
String script = shellParameters.getRawScript().replaceAll("\\r\\n", "\n");
|
||||
// combining local and global parameters
|
||||
Map<String, Property> paramsMap = ParamUtils.convert(ParamUtils.getUserDefParamsMap(taskExecutionContext.getDefinedParams()),
|
||||
taskExecutionContext.getDefinedParams(),
|
||||
shellParameters.getLocalParametersMap(),
|
||||
CommandType.of(taskExecutionContext.getCmdTypeIfComplement()),
|
||||
taskExecutionContext.getScheduleTime());
|
||||
if (paramsMap != null){
|
||||
script = ParameterUtils.convertParameterPlaceholders(script, ParamUtils.convert(paramsMap));
|
||||
}
|
||||
// new
|
||||
// replace variable TIME with $[YYYYmmddd...] in shell file when history run job and batch complement job
|
||||
if (paramsMap != null) {
|
||||
if (taskExecutionContext.getScheduleTime() != null) {
|
||||
String dateTime = DateUtils.format(taskExecutionContext.getScheduleTime(), Constants.PARAMETER_FORMAT_TIME);
|
||||
Property p = new Property();
|
||||
p.setValue(dateTime);
|
||||
p.setProp(Constants.PARAMETER_SHECDULE_TIME);
|
||||
paramsMap.put(Constants.PARAMETER_SHECDULE_TIME, p);
|
||||
}
|
||||
script = ParameterUtils.convertParameterPlaceholders2(script, ParamUtils.convert(paramsMap));
|
||||
|
||||
// replace variable TIME with $[YYYYmmddd...] in shell file when history run job and batch complement job
|
||||
if (paramsMap != null) {
|
||||
if (taskExecutionContext.getScheduleTime() != null) {
|
||||
String dateTime = DateUtils.format(taskExecutionContext.getScheduleTime(), Constants.PARAMETER_FORMAT_TIME);
|
||||
Property p = new Property();
|
||||
p.setValue(dateTime);
|
||||
p.setProp(Constants.PARAMETER_SHECDULE_TIME);
|
||||
paramsMap.put(Constants.PARAMETER_SHECDULE_TIME, p);
|
||||
}
|
||||
}
|
||||
|
||||
script = ParameterUtils.convertParameterPlaceholders2(script, ParamUtils.convert(paramsMap));
|
||||
|
||||
shellParameters.setRawScript(script);
|
||||
|
||||
logger.info("raw script : {}", shellParameters.getRawScript());
|
||||
logger.info("task execute path : {}", taskExecutionContext.getExecutePath());
|
||||
|
||||
Set<PosixFilePermission> perms = PosixFilePermissions.fromString(Constants.RWXR_XR_X);
|
||||
FileAttribute<Set<PosixFilePermission>> attr = PosixFilePermissions.asFileAttribute(perms);
|
||||
|
||||
if (OSUtils.isWindows()) {
|
||||
Files.createFile(path);
|
||||
} else {
|
||||
Files.createFile(path, attr);
|
||||
}
|
||||
|
||||
Files.write(path, shellParameters.getRawScript().getBytes(), StandardOpenOption.APPEND);
|
||||
|
||||
return fileName;
|
||||
}
|
||||
|
||||
shellParameters.setRawScript(script);
|
||||
|
||||
logger.info("raw script : {}", shellParameters.getRawScript());
|
||||
logger.info("task execute path : {}", taskExecutionContext.getExecutePath());
|
||||
|
||||
Set<PosixFilePermission> perms = PosixFilePermissions.fromString(Constants.RWXR_XR_X);
|
||||
FileAttribute<Set<PosixFilePermission>> attr = PosixFilePermissions.asFileAttribute(perms);
|
||||
|
||||
if (OSUtils.isWindows()) {
|
||||
Files.createFile(path);
|
||||
} else {
|
||||
Files.createFile(path, attr);
|
||||
@Override
|
||||
public AbstractParameters getParameters() {
|
||||
return shellParameters;
|
||||
}
|
||||
|
||||
Files.write(path, shellParameters.getRawScript().getBytes(), StandardOpenOption.APPEND);
|
||||
|
||||
return fileName;
|
||||
}
|
||||
|
||||
@Override
|
||||
public AbstractParameters getParameters() {
|
||||
return shellParameters;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue