fix python home conflict (#12112)

jieguangzhou · web-flow · commit 6eb1eb722af0 · 2022-09-23T09:40:18.000+08:00
diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-pytorch/src/main/java/org/apache/dolphinscheduler/plugin/task/pytorch/PythonEnvManager.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-pytorch/src/main/java/org/apache/dolphinscheduler/plugin/task/pytorch/PythonEnvManager.java
@@ -40,7 +40,7 @@ public class PythonEnvManager {
 
     private static final String VIRTUALENV_BUILD = "virtualenv -p ${PYTHON_HOME} %s";
 
-    private static final String INSTALL_COMMAND = "python -m pip install -r %s";
+    private static final String INSTALL_COMMAND = "%s -m pip install -r %s";
 
     private String pythonEnvTool = ENV_TOOL_VENV;
 
@@ -54,7 +54,7 @@ public String getBuildEnvCommand(String requirementPath) {
         } else if (pythonEnvTool.equals(ENV_TOOL_CONDA)) {
             buildCommand = String.format(CONDA_BUILD, condaPythonVersion, CREATE_ENV_NAME);
         }
-        String installCommand = String.format(INSTALL_COMMAND, requirementPath);
+        String installCommand = String.format(INSTALL_COMMAND, getPythonCommand(), requirementPath);
 
         return buildCommand + " && " + sourceCommand + " && " + installCommand;
     }
diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-pytorch/src/test/java/org/apache/dolphinscheduler/plugin/task/pytorch/PytorchTaskTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-pytorch/src/test/java/org/apache/dolphinscheduler/plugin/task/pytorch/PytorchTaskTest.java
@@ -72,16 +72,16 @@ public void testPythonEnvManager() {
         envManager.setPythonEnvTool(PythonEnvManager.ENV_TOOL_CONDA);
         envManager.setCondaPythonVersion("3.9");
         String condaEnvCommand39 = envManager.getBuildEnvCommand(requirementPath);
-        Assert.assertEquals(condaEnvCommand39, "conda create -y python=3.9 -p ./venv && source activate ./venv && python -m pip install -r " + requirementPath);
+        Assert.assertEquals(condaEnvCommand39, "conda create -y python=3.9 -p ./venv && source activate ./venv && ./venv/bin/python -m pip install -r " + requirementPath);
 
         envManager.setCondaPythonVersion("3.8");
         String condaEnvCommand38 = envManager.getBuildEnvCommand(requirementPath);
-        Assert.assertEquals(condaEnvCommand38, "conda create -y python=3.8 -p ./venv && source activate ./venv && python -m pip install -r " + requirementPath);
+        Assert.assertEquals(condaEnvCommand38, "conda create -y python=3.8 -p ./venv && source activate ./venv && ./venv/bin/python -m pip install -r " + requirementPath);
 
 
         envManager.setPythonEnvTool(PythonEnvManager.ENV_TOOL_VENV);
         String venvEnvCommand = envManager.getBuildEnvCommand(requirementPath);
-        Assert.assertEquals(venvEnvCommand, "virtualenv -p ${PYTHON_HOME} ./venv && source ./venv/bin/activate && python -m pip install -r " + requirementPath);
+        Assert.assertEquals(venvEnvCommand, "virtualenv -p ${PYTHON_HOME} ./venv && source ./venv/bin/activate && ./venv/bin/python -m pip install -r " + requirementPath);
 
     }
 
@@ -146,7 +146,7 @@ public void testBuildPythonCommandWithCreateCondeEnv() throws Exception {
         PytorchTask task = initTask(parameters);
         Assert.assertEquals(task.buildPythonExecuteCommand(),
             "export PYTHONPATH=.\n" +
-                "conda create -y python=3.6 -p ./venv && source activate ./venv && python -m pip install -r requirements.txt\n" +
+                "conda create -y python=3.6 -p ./venv && source activate ./venv && ./venv/bin/python -m pip install -r requirements.txt\n" +
                 "./venv/bin/python main.py --epochs=1 --dry-run");
     }
 
@@ -163,7 +163,7 @@ public void testBuildPythonCommandWithCreateVenvEnv() throws Exception {
         PytorchTask task = initTask(parameters);
         Assert.assertEquals(task.buildPythonExecuteCommand(),
             "export PYTHONPATH=.\n" +
-                "virtualenv -p ${PYTHON_HOME} ./venv && source ./venv/bin/activate && python -m pip install -r requirements.txt\n" +
+                "virtualenv -p ${PYTHON_HOME} ./venv && source ./venv/bin/activate && ./venv/bin/python -m pip install -r requirements.txt\n" +
                 "./venv/bin/python main.py --epochs=1 --dry-run");
 
     }
@@ -189,7 +189,7 @@ public void testGetPossiblePath() throws Exception {
         createFile(scriptFile);
 
         String expected = "export PYTHONPATH=%s\n" +
-            "virtualenv -p ${PYTHON_HOME} ./venv && source ./venv/bin/activate && python -m pip install -r %s\n" +
+            "virtualenv -p ${PYTHON_HOME} ./venv && source ./venv/bin/activate && ./venv/bin/python -m pip install -r %s\n" +
             "./venv/bin/python %s";
         System.out.println(task.buildPythonExecuteCommand());
         Assert.assertEquals(String.format(expected, pythonPath, requirementFile, scriptFile), task.buildPythonExecuteCommand());

Original file line number	Diff line number	Diff line change
`@@ -40,7 +40,7 @@ public class PythonEnvManager {`
`40`	`40`
`41`	`41`	`private static final String VIRTUALENV_BUILD = "virtualenv -p ${PYTHON_HOME} %s";`
`42`	`42`
`43`		`- private static final String INSTALL_COMMAND = "python -m pip install -r %s";`
	`43`	`+ private static final String INSTALL_COMMAND = "%s -m pip install -r %s";`
`44`	`44`
`45`	`45`	`private String pythonEnvTool = ENV_TOOL_VENV;`
`46`	`46`
`@@ -54,7 +54,7 @@ public String getBuildEnvCommand(String requirementPath) {`
`54`	`54`	`} else if (pythonEnvTool.equals(ENV_TOOL_CONDA)) {`
`55`	`55`	`buildCommand = String.format(CONDA_BUILD, condaPythonVersion, CREATE_ENV_NAME);`
`56`	`56`	`}`
`57`		`- String installCommand = String.format(INSTALL_COMMAND, requirementPath);`
	`57`	`+ String installCommand = String.format(INSTALL_COMMAND, getPythonCommand(), requirementPath);`
`58`	`58`
`59`	`59`	`return buildCommand + " && " + sourceCommand + " && " + installCommand;`
`60`	`60`	`}`