natas_hw
/
dolphinscheduler
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix jar path is not correct in java task (#15906)

This commit is contained in:
Wenjun Ruan 2024-04-29 18:07:05 +08:00 committed by GitHub
parent 647cbae400
commit 0a11cd21bd
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 23 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/resource/ResourceContext.java
View File

@ -60,7 +60,6 @@ public class ResourceContext {
public static class ResourceItem {
private String resourceAbsolutePathInStorage;
private String resourceRelativePath;
private String resourceAbsolutePathInLocal;
}

4
dolphinscheduler-task-plugin/dolphinscheduler-task-hivecli/src/test/java/org/apache/dolphinscheduler/plugin/task/hivecli/HiveCliTaskTest.java
View File

@ -65,7 +65,7 @@ public class HiveCliTaskTest {
}
@Test
public void hiveCliTaskExecuteSqlFromScript() throws Exception {
public void hiveCliTaskExecuteSqlFromScript() {
String hiveCliTaskParameters = buildHiveCliTaskExecuteSqlFromScriptParameters();
HiveCliTask hiveCliTask = prepareHiveCliTaskForTest(hiveCliTaskParameters);
hiveCliTask.init();
@ -78,7 +78,7 @@ public class HiveCliTaskTest {
TaskExecutionContext taskExecutionContext = new TaskExecutionContext();
taskExecutionContext.setTaskParams(hiveCliTaskParameters);
ResourceContext resourceContext = new ResourceContext();
resourceContext.addResourceItem(new ResourceContext.ResourceItem("/sql_tasks/hive_task.sql", "123_node.sql",
resourceContext.addResourceItem(new ResourceContext.ResourceItem("/sql_tasks/hive_task.sql",
"/sql_tasks/hive_task.sql"));
taskExecutionContext.setResourceContext(resourceContext);

41
dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/JavaTask.java
View File

@ -88,6 +88,7 @@ public class JavaTask extends AbstractTask {
/**
* Initializes a Java task
*
* @return void
**/
@Override
@ -178,7 +179,8 @@ public class JavaTask extends AbstractTask {
**/
protected String buildJarCommand() {
ResourceContext resourceContext = taskRequest.getResourceContext();
String mainJarName = resourceContext.getResourceItem(javaParameters.getMainJar().getResourceName())
String mainJarAbsolutePathInLocal = resourceContext
.getResourceItem(javaParameters.getMainJar().getResourceName())
.getResourceAbsolutePathInLocal();
StringBuilder builder = new StringBuilder();
builder.append(getJavaCommandPath())
@ -186,7 +188,7 @@ public class JavaTask extends AbstractTask {
.append(buildResourcePath()).append(" ")
.append("-jar").append(" ")
.append(taskRequest.getExecutePath()).append(FOLDER_SEPARATOR)
.append(mainJarName).append(" ")
.append(mainJarAbsolutePathInLocal).append(" ")
.append(javaParameters.getMainArgs().trim()).append(" ")
.append(javaParameters.getJvmArgs().trim());
return builder.toString();
@ -207,39 +209,6 @@ public class JavaTask extends AbstractTask {
return javaParameters;
}
/**
* Replaces placeholders such as local variables in source files
*
* @param rawScript
* @return String
* @throws StringIndexOutOfBoundsException
*/
protected static String convertJavaSourceCodePlaceholders(String rawScript) throws StringIndexOutOfBoundsException {
int len = "${setShareVar(${".length();
int scriptStart = 0;
while ((scriptStart = rawScript.indexOf("${setShareVar(${", scriptStart)) != -1) {
int start = -1;
int end = rawScript.indexOf('}', scriptStart + len);
String prop = rawScript.substring(scriptStart + len, end);
start = rawScript.indexOf(',', end);
end = rawScript.indexOf(')', start);
String value = rawScript.substring(start + 1, end);
start = rawScript.indexOf('}', start) + 1;
end = rawScript.length();
String replaceScript = String.format("print(\"${{setValue({},{})}}\".format(\"%s\",%s))", prop, value);
rawScript = rawScript.substring(0, scriptStart) + replaceScript + rawScript.substring(start, end);
scriptStart += replaceScript.length();
}
return rawScript;
}
/**
* Creates a Java source file when it does not exist
*
@ -290,8 +259,6 @@ public class JavaTask extends AbstractTask {
for (ResourceInfo info : javaParameters.getResourceFilesList()) {
builder.append(JavaConstants.PATH_SEPARATOR);
builder
.append(taskRequest.getExecutePath())
.append(FOLDER_SEPARATOR)
.append(resourceContext.getResourceItem(info.getResourceName()).getResourceAbsolutePathInLocal());
}
return builder.toString();

32
dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/test/java/org/apache/dolphinscheduler/plugin/task/java/JavaTaskTest.java
View File

@ -17,6 +17,7 @@
package org.apache.dolphinscheduler.plugin.task.java;
import static com.google.common.truth.Truth.assertThat;
import static org.apache.dolphinscheduler.plugin.task.api.enums.DataType.VARCHAR;
import static org.apache.dolphinscheduler.plugin.task.api.enums.Direct.IN;
import static org.apache.dolphinscheduler.plugin.task.java.JavaConstants.RUN_TYPE_JAR;
@ -34,7 +35,6 @@ import org.apache.dolphinscheduler.plugin.task.java.exception.JavaSourceFileExis
import org.apache.dolphinscheduler.plugin.task.java.exception.PublicClassNotFoundException;
import org.apache.dolphinscheduler.plugin.task.java.exception.RunTypeNotFoundException;
import java.io.File;
import java.io.IOException;
import java.lang.reflect.Field;
import java.nio.file.Files;
@ -82,10 +82,10 @@ public class JavaTaskTest {
**/
@Test
public void buildJarCommand() {
String homeBinPath = JavaConstants.JAVA_HOME_VAR + File.separator + "bin" + File.separator;
JavaTask javaTask = runJarType();
Assertions.assertEquals(javaTask.buildJarCommand(), homeBinPath
+ "java -classpath .:/tmp/dolphinscheduler/test/executepath:/tmp/dolphinscheduler/test/executepath/opt/share/jar/resource2.jar -jar /tmp/dolphinscheduler/test/executepath/opt/share/jar/main.jar -host 127.0.0.1 -port 8080 -xms:50m");
assertThat(javaTask.buildJarCommand())
.isEqualTo(
"${JAVA_HOME}/bin/java -classpath .:/tmp/dolphinscheduler/test/executepath:opt/share/jar/resource2.jar -jar /tmp/dolphinscheduler/test/executepath/opt/share/jar/main.jar -host 127.0.0.1 -port 8080 -xms:50m");
}
/**
@ -101,14 +101,13 @@ public class JavaTaskTest {
Assertions.assertEquals("JavaTaskTest", publicClassName);
String fileName = javaTask.buildJavaSourceCodeFileFullName(publicClassName);
try {
String homeBinPath = JavaConstants.JAVA_HOME_VAR + File.separator + "bin" + File.separator;
Path path = Paths.get(fileName);
if (Files.exists(path)) {
Files.delete(path);
}
Assertions.assertEquals(homeBinPath
+ "javac -classpath .:/tmp/dolphinscheduler/test/executepath:/tmp/dolphinscheduler/test/executepath/opt/share/jar/resource2.jar /tmp/dolphinscheduler/test/executepath/JavaTaskTest.java",
javaTask.buildJavaCompileCommand(sourceCode));
assertThat(javaTask.buildJavaCompileCommand(sourceCode))
.isEqualTo(
"${JAVA_HOME}/bin/javac -classpath .:/tmp/dolphinscheduler/test/executepath:opt/share/jar/resource2.jar /tmp/dolphinscheduler/test/executepath/JavaTaskTest.java");
} finally {
Path path = Paths.get(fileName);
if (Files.exists(path)) {
@ -121,26 +120,29 @@ public class JavaTaskTest {
/**
* Construct java to run the command
*
* @return void
* @return void
**/
@Test
public void buildJavaCommand() throws Exception {
String wantJavaCommand =
"${JAVA_HOME}/bin/javac -classpath .:/tmp/dolphinscheduler/test/executepath:/tmp/dolphinscheduler/test/executepath/opt/share/jar/resource2.jar /tmp/dolphinscheduler/test/executepath/JavaTaskTest.java;${JAVA_HOME}/bin/java -classpath .:/tmp/dolphinscheduler/test/executepath:/tmp/dolphinscheduler/test/executepath/opt/share/jar/resource2.jar JavaTaskTest -host 127.0.0.1 -port 8080 -xms:50m";
JavaTask javaTask = runJavaType();
String sourceCode = javaTask.buildJavaSourceContent();
String publicClassName = javaTask.getPublicClassName(sourceCode);
Assertions.assertEquals("JavaTaskTest", publicClassName);
String fileName = javaTask.buildJavaSourceCodeFileFullName(publicClassName);
Path path = Paths.get(fileName);
if (Files.exists(path)) {
Files.delete(path);
}
Assertions.assertEquals(wantJavaCommand, javaTask.buildJavaCommand());
assertThat(javaTask.buildJavaCommand())
.isEqualTo(
"${JAVA_HOME}/bin/javac -classpath .:/tmp/dolphinscheduler/test/executepath:opt/share/jar/resource2.jar /tmp/dolphinscheduler/test/executepath/JavaTaskTest.java;${JAVA_HOME}/bin/java -classpath .:/tmp/dolphinscheduler/test/executepath:opt/share/jar/resource2.jar JavaTaskTest -host 127.0.0.1 -port 8080 -xms:50m");
}
/**
* There is no exception to overwriting the Java source file
*
* @return void
* @throws IOException
**/
@ -259,8 +261,8 @@ public class JavaTaskTest {
resourceItem2.setResourceAbsolutePathInLocal("opt/share/jar/main.jar");
ResourceContext.ResourceItem resourceItem3 = new ResourceContext.ResourceItem();
resourceItem2.setResourceAbsolutePathInStorage("/JavaTaskTest.java");
resourceItem2.setResourceAbsolutePathInLocal("JavaTaskTest.java");
resourceItem3.setResourceAbsolutePathInStorage("/JavaTaskTest.java");
resourceItem3.setResourceAbsolutePathInLocal("JavaTaskTest.java");
ResourceContext resourceContext = new ResourceContext();
resourceContext.addResourceItem(resourceItem1);
@ -275,7 +277,7 @@ public class JavaTaskTest {
/**
* The Java task to construct the jar run mode
*
* @return JavaTask
* @return JavaTask
**/
private JavaTask runJarType() {
TaskExecutionContext taskExecutionContext = new TaskExecutionContext();

1
dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/utils/TaskExecutionContextUtils.java
View File

@ -151,7 +151,6 @@ public class TaskExecutionContextUtils {
}
ResourceContext.ResourceItem resourceItem = ResourceContext.ResourceItem.builder()
.resourceAbsolutePathInStorage(resourceAbsolutePathInStorage)
.resourceRelativePath(resourceRelativePath)
.resourceAbsolutePathInLocal(resourceAbsolutePathInLocal)
.build();
resourceContext.addResourceItem(resourceItem);