natas_hw
/
dolphinscheduler0701
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

EMR task support replace params placeholder (#15975) 

Co-authored-by: Eric Gao <ericgao.apache@gmail.com>
This commit is contained in:
Wenjun Ruan 2024-05-10 17:34:26 +08:00 committed by GitHub
parent ace20f96c2
commit 3446fd8ab1
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 14 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
dolphinscheduler-task-plugin/dolphinscheduler-task-emr/src/main/java/org/apache/dolphinscheduler/plugin/task/emr/EmrAddStepsTask.java
View File

@ -20,6 +20,7 @@ package org.apache.dolphinscheduler.plugin.task.emr;
import org.apache.dolphinscheduler.plugin.task.api.TaskConstants;
import org.apache.dolphinscheduler.plugin.task.api.TaskException;
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils;
import java.util.Collections;
import java.util.HashSet;
@ -126,11 +127,15 @@ public class EmrAddStepsTask extends AbstractEmrTask {
protected AddJobFlowStepsRequest createAddJobFlowStepsRequest() {
final AddJobFlowStepsRequest addJobFlowStepsRequest;
String jobStepDefineJson = null;
try {
jobStepDefineJson = ParameterUtils.convertParameterPlaceholders(
emrParameters.getStepsDefineJson(),
ParameterUtils.convert(taskExecutionContext.getPrepareParamsMap()));
addJobFlowStepsRequest =
objectMapper.readValue(emrParameters.getStepsDefineJson(), AddJobFlowStepsRequest.class);
objectMapper.readValue(jobStepDefineJson, AddJobFlowStepsRequest.class);
} catch (JsonProcessingException e) {
throw new EmrTaskException("can not parse AddJobFlowStepsRequest from json", e);
throw new EmrTaskException("can not parse AddJobFlowStepsRequest from json: " + jobStepDefineJson, e);
}
// When a single task definition is associated with multiple steps, the state tracking will have high

9
dolphinscheduler-task-plugin/dolphinscheduler-task-emr/src/main/java/org/apache/dolphinscheduler/plugin/task/emr/EmrJobFlowTask.java
View File

@ -20,6 +20,7 @@ package org.apache.dolphinscheduler.plugin.task.emr;
import org.apache.dolphinscheduler.plugin.task.api.TaskConstants;
import org.apache.dolphinscheduler.plugin.task.api.TaskException;
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils;
import java.util.Collections;
import java.util.HashSet;
@ -120,10 +121,14 @@ public class EmrJobFlowTask extends AbstractEmrTask {
protected RunJobFlowRequest createRunJobFlowRequest() {
final RunJobFlowRequest runJobFlowRequest;
String jobFlowDefineJson = null;
try {
runJobFlowRequest = objectMapper.readValue(emrParameters.getJobFlowDefineJson(), RunJobFlowRequest.class);
jobFlowDefineJson = ParameterUtils.convertParameterPlaceholders(
emrParameters.getJobFlowDefineJson(),
ParameterUtils.convert(taskExecutionContext.getPrepareParamsMap()));
runJobFlowRequest = objectMapper.readValue(jobFlowDefineJson, RunJobFlowRequest.class);
} catch (JsonProcessingException e) {
throw new EmrTaskException("can not parse RunJobFlowRequest from json", e);
throw new EmrTaskException("can not parse RunJobFlowRequest from json: " + jobFlowDefineJson, e);
}
return runJobFlowRequest;