natas_hw
/
dolphinscheduler0701
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix python home conflict (#12112) 

(cherry picked from commit 6eb1eb722a)
This commit is contained in:
JieguangZhou 2022-09-23 09:40:18 +08:00 committed by Jiajie Zhong
parent c47e088b73
commit f871a4a41f
2 changed files with 8 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
dolphinscheduler-task-plugin/dolphinscheduler-task-pytorch/src/main/java/org/apache/dolphinscheduler/plugin/task/pytorch/PythonEnvManager.java
View File

@ -40,7 +40,7 @@ public class PythonEnvManager {
private static final String VIRTUALENV_BUILD = "virtualenv -p ${PYTHON_HOME} %s";
private static final String INSTALL_COMMAND = "python -m pip install -r %s";
private static final String INSTALL_COMMAND = "%s -m pip install -r %s";
private String pythonEnvTool = ENV_TOOL_VENV;
@ -54,7 +54,7 @@ public class PythonEnvManager {
} else if (pythonEnvTool.equals(ENV_TOOL_CONDA)) {
buildCommand = String.format(CONDA_BUILD, condaPythonVersion, CREATE_ENV_NAME);
}
String installCommand = String.format(INSTALL_COMMAND, requirementPath);
String installCommand = String.format(INSTALL_COMMAND, getPythonCommand(), requirementPath);
return buildCommand + " && " + sourceCommand + " && " + installCommand;
}

12
dolphinscheduler-task-plugin/dolphinscheduler-task-pytorch/src/test/java/org/apache/dolphinscheduler/plugin/task/pytorch/PytorchTaskTest.java
View File

@ -72,16 +72,16 @@ public class PytorchTaskTest {
envManager.setPythonEnvTool(PythonEnvManager.ENV_TOOL_CONDA);
envManager.setCondaPythonVersion("3.9");
String condaEnvCommand39 = envManager.getBuildEnvCommand(requirementPath);
Assert.assertEquals(condaEnvCommand39, "conda create -y python=3.9 -p ./venv && source activate ./venv && python -m pip install -r " + requirementPath);
Assert.assertEquals(condaEnvCommand39, "conda create -y python=3.9 -p ./venv && source activate ./venv && ./venv/bin/python -m pip install -r " + requirementPath);
envManager.setCondaPythonVersion("3.8");
String condaEnvCommand38 = envManager.getBuildEnvCommand(requirementPath);
Assert.assertEquals(condaEnvCommand38, "conda create -y python=3.8 -p ./venv && source activate ./venv && python -m pip install -r " + requirementPath);
Assert.assertEquals(condaEnvCommand38, "conda create -y python=3.8 -p ./venv && source activate ./venv && ./venv/bin/python -m pip install -r " + requirementPath);
envManager.setPythonEnvTool(PythonEnvManager.ENV_TOOL_VENV);
String venvEnvCommand = envManager.getBuildEnvCommand(requirementPath);
Assert.assertEquals(venvEnvCommand, "virtualenv -p ${PYTHON_HOME} ./venv && source ./venv/bin/activate && python -m pip install -r " + requirementPath);
Assert.assertEquals(venvEnvCommand, "virtualenv -p ${PYTHON_HOME} ./venv && source ./venv/bin/activate && ./venv/bin/python -m pip install -r " + requirementPath);
}
@ -146,7 +146,7 @@ public class PytorchTaskTest {
PytorchTask task = initTask(parameters);
Assert.assertEquals(task.buildPythonExecuteCommand(),
"export PYTHONPATH=.\n" +
"conda create -y python=3.6 -p ./venv && source activate ./venv && python -m pip install -r requirements.txt\n" +
"conda create -y python=3.6 -p ./venv && source activate ./venv && ./venv/bin/python -m pip install -r requirements.txt\n" +
"./venv/bin/python main.py --epochs=1 --dry-run");
}
@ -163,7 +163,7 @@ public class PytorchTaskTest {
PytorchTask task = initTask(parameters);
Assert.assertEquals(task.buildPythonExecuteCommand(),
"export PYTHONPATH=.\n" +
"virtualenv -p ${PYTHON_HOME} ./venv && source ./venv/bin/activate && python -m pip install -r requirements.txt\n" +
"virtualenv -p ${PYTHON_HOME} ./venv && source ./venv/bin/activate && ./venv/bin/python -m pip install -r requirements.txt\n" +
"./venv/bin/python main.py --epochs=1 --dry-run");
}
@ -189,7 +189,7 @@ public class PytorchTaskTest {
createFile(scriptFile);
String expected = "export PYTHONPATH=%s\n" +
"virtualenv -p ${PYTHON_HOME} ./venv && source ./venv/bin/activate && python -m pip install -r %s\n" +
"virtualenv -p ${PYTHON_HOME} ./venv && source ./venv/bin/activate && ./venv/bin/python -m pip install -r %s\n" +
"./venv/bin/python %s";
System.out.println(task.buildPythonExecuteCommand());
Assert.assertEquals(String.format(expected, pythonPath, requirementFile, scriptFile), task.buildPythonExecuteCommand());