natas_hw
/
dolphinscheduler0701
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[fix][plugin]: fix spark sql use custom parma. (#10821) 

* [fix][plugin]: fix spark sql use custom parma.

* [fix][plugin] roll back part of the code

Co-authored-by: huaangcg <huangcg@getui.com>
This commit is contained in:
chuxing 2022-07-13 11:44:13 +08:00 committed by GitHub
parent 2397423eb6
commit 427c58546c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 12 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/AbstractParameters.java
View File

@ -30,6 +30,7 @@ import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ArrayNode;
@ -90,9 +91,8 @@ public abstract class AbstractParameters implements IParameters {
public Map<String, Property> getInputLocalParametersMap() {
Map<String, Property> localParametersMaps = new LinkedHashMap<>();
if (localParams != null) {
for (Property property : localParams) {
if (property.getDirect().equals(Direct.IN)) {
if (Objects.equals(Direct.IN, property.getDirect())) {
localParametersMaps.put(property.getProp(), property);
}
}

10
dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/main/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTask.java
View File

@ -229,7 +229,7 @@ public class SparkTask extends AbstractYarnTask {
Path path = file.toPath();
if (!Files.exists(path)) {
String script = sparkParameters.getRawScript().replaceAll("\\r\\n", "\n");
String script = replaceParam(sparkParameters.getRawScript());
sparkParameters.setRawScript(script);
logger.info("raw script : {}", sparkParameters.getRawScript());
@ -255,6 +255,14 @@ public class SparkTask extends AbstractYarnTask {
return scriptFileName;
}
private String replaceParam(String script) {
script = script.replaceAll("\\r\\n", "\n");
// replace placeholder, and combining local and global parameters
Map<String, Property> paramsMap = taskExecutionContext.getPrepareParamsMap();
script = ParameterUtils.convertParameterPlaceholders(script, ParamUtils.convert(paramsMap));
return script;
}
@Override
protected void setMainJarName() {
// main jar

2
dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-spark.ts
View File

@ -130,7 +130,7 @@ export function useSpark(model: { [field: string]: any }): IJsonItem[] {
}
},
useResources(),
...useCustomParams({ model, field: 'localParams', isSimple: true })
...useCustomParams({ model, field: 'localParams', isSimple: false })
]
}