natas_hw
/
dolphinscheduler
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[Feature][Datax Task]Datax task plugin support custom parameters (#10950) 

* datax task plugin support custom parameters
This commit is contained in:
zhuxt2015 2022-07-15 09:27:23 +08:00 committed by GitHub
parent a38fa34d43
commit 7196b125d9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 16 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTask.java
View File

@ -82,6 +82,8 @@ public class DataxTask extends AbstractTaskExecutor {
* jvm parameters
*/
public static final String JVM_PARAM = " --jvm=\"-Xms%sG -Xmx%sG\" ";
public static final String CUSTOM_PARAM = " -D%s=%s";
/**
* python process(datax only supports version 2.7 by default)
*/
@ -392,6 +394,8 @@ public class DataxTask extends AbstractTaskExecutor {
sbr.append(DATAX_PATH);
sbr.append(" ");
sbr.append(loadJvmEnv(dataXParameters));
sbr.append(addCustomParameters(paramsMap));
sbr.append(" ");
sbr.append(jobConfigFilePath);
// replace placeholder
@ -414,6 +418,15 @@ public class DataxTask extends AbstractTaskExecutor {
return fileName;
}
private StringBuilder addCustomParameters(Map<String, Property> paramsMap) {
StringBuilder customParameters = new StringBuilder("-p\"");
for (Map.Entry<String, Property> entry : paramsMap.entrySet()) {
customParameters.append(String.format(CUSTOM_PARAM, entry.getKey(), entry.getValue().getValue()));
}
customParameters.append("\"");
return customParameters;
}
public String getPythonCommand() {
String pythonHome = System.getenv("PYTHON_HOME");
return getPythonCommand(pythonHome);

4
dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-datax.ts
View File

@ -20,6 +20,7 @@ import type { IJsonItem } from '../types'
import { find } from 'lodash'
import { TypeReq } from '@/service/modules/data-source/types'
import { queryDataSourceList } from '@/service/modules/data-source'
import {useCustomParams} from "."
export function useDataX(model: { [field: string]: any }): IJsonItem[] {
const { t } = useI18n()
@ -443,6 +444,7 @@ export function useDataX(model: { [field: string]: any }): IJsonItem[] {
span: 12,
options: memoryLimitOptions,
value: 1
}
},
...useCustomParams({ model, field: 'localParams', isSimple: true })
]
}