From aa117136cf61e631d0ca9eba0a58f71972af36f2 Mon Sep 17 00:00:00 2001
From: Jiajie Zhong <zhongjiajie955@hotmail.com>
Date: Tue, 22 Feb 2022 15:14:04 +0800
Subject: [PATCH] [deploy] Make python api release to ASF dist independent
 (#8472)

This patch add python api package as independent
directory in final dist directory when user run
command `mvn -U install package -Prelease`.

There have `tar.gz` and `whl` in the python dist directory

close: #8343

Change python api version to 2.0.4

Add pre_clean command to clean env
---
 dolphinscheduler-dist/pom.xml                 | 16 +++++
 .../assembly/dolphinscheduler-python-api.xml  | 34 ++++++++++
 .../main/assembly/dolphinscheduler-src.xml    |  7 ++
 dolphinscheduler-python/pom.xml               | 68 ++++++++++++++++++-
 .../pydolphinscheduler/setup.py               | 56 +++++++++++++--
 5 files changed, 176 insertions(+), 5 deletions(-)
 create mode 100644 dolphinscheduler-dist/src/main/assembly/dolphinscheduler-python-api.xml

diff --git a/dolphinscheduler-dist/pom.xml b/dolphinscheduler-dist/pom.xml
index cdbaa87c1a..a288356526 100644
--- a/dolphinscheduler-dist/pom.xml
+++ b/dolphinscheduler-dist/pom.xml
@@ -95,6 +95,22 @@
                                 </configuration>
                             </execution>
 
+                            <execution>
+                                <id>python</id>
+                                <phase>package</phase>
+                                <goals>
+                                    <goal>single</goal>
+                                </goals>
+                                <configuration>
+                                    <!-- Make final directory with simple name `python`, and without any addtion information -->
+                                    <finalName>python</finalName>
+                                    <appendAssemblyId>false</appendAssemblyId>
+                                    <descriptors>
+                                        <descriptor>src/main/assembly/dolphinscheduler-python-api.xml</descriptor>
+                                    </descriptors>
+                                </configuration>
+                            </execution>
+
                         </executions>
                     </plugin>
 
diff --git a/dolphinscheduler-dist/src/main/assembly/dolphinscheduler-python-api.xml b/dolphinscheduler-dist/src/main/assembly/dolphinscheduler-python-api.xml
new file mode 100644
index 0000000000..cd37acee62
--- /dev/null
+++ b/dolphinscheduler-dist/src/main/assembly/dolphinscheduler-python-api.xml
@@ -0,0 +1,34 @@
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+
+<assembly
+        xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
+        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+        xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
+    <id>python-api</id>
+    <formats>
+        <format>dir</format>
+    </formats>
+    <includeBaseDirectory>false</includeBaseDirectory>
+
+    <fileSets>
+        <fileSet>
+            <directory>${basedir}/../dolphinscheduler-python/pydolphinscheduler/dist</directory>
+            <outputDirectory>.</outputDirectory>
+        </fileSet>
+    </fileSets>
+</assembly>
diff --git a/dolphinscheduler-dist/src/main/assembly/dolphinscheduler-src.xml b/dolphinscheduler-dist/src/main/assembly/dolphinscheduler-src.xml
index e675cfb2ba..e275d6e42d 100644
--- a/dolphinscheduler-dist/src/main/assembly/dolphinscheduler-src.xml
+++ b/dolphinscheduler-dist/src/main/assembly/dolphinscheduler-src.xml
@@ -58,6 +58,13 @@
                 <exclude>**/dolphinscheduler-ui/node/**</exclude>
                 <exclude>**/dolphinscheduler-ui/node_modules/**</exclude>
 
+                <!-- python ignore -->
+                <exclude>**/dolphinscheduler-python/pydolphinscheduler/.pytest_cache/**</exclude>
+                <exclude>**/dolphinscheduler-python/pydolphinscheduler/build/**</exclude>
+                <exclude>**/dolphinscheduler-python/pydolphinscheduler/dist/**</exclude>
+                <exclude>**/dolphinscheduler-python/pydolphinscheduler/dist/**</exclude>
+                <exclude>**/dolphinscheduler-python/pydolphinscheduler/htmlcov/**</exclude>
+
                 <!-- eclipse ignore -->
                 <exclude>**/.settings/**</exclude>
                 <exclude>**/.project</exclude>
diff --git a/dolphinscheduler-python/pom.xml b/dolphinscheduler-python/pom.xml
index f669d634dd..74a0880e9c 100644
--- a/dolphinscheduler-python/pom.xml
+++ b/dolphinscheduler-python/pom.xml
@@ -16,7 +16,8 @@
   ~ limitations under the License.
   -->
 
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <modelVersion>4.0.0</modelVersion>
     <parent>
         <groupId>org.apache.dolphinscheduler</groupId>
@@ -56,4 +57,69 @@
         </dependency>
 
     </dependencies>
+
+    <profiles>
+        <profile>
+            <id>release</id>
+            <build>
+                <plugins>
+                    <plugin>
+                        <groupId>org.codehaus.mojo</groupId>
+                        <artifactId>exec-maven-plugin</artifactId>
+                        <executions>
+                            <execution>
+                                <id>python-api-prepare</id>
+                                <phase>prepare-package</phase>
+                                <goals>
+                                    <goal>exec</goal>
+                                </goals>
+                                <configuration>
+                                    <executable>python3</executable>
+                                    <workingDirectory>${project.basedir}/pydolphinscheduler</workingDirectory>
+                                    <arguments>
+                                        <argument>-m</argument>
+                                        <argument>pip</argument>
+                                        <argument>install</argument>
+                                        <argument>--upgrade</argument>
+                                        <argument>pip</argument>
+                                        <argument>.[build]</argument>
+                                    </arguments>
+                                </configuration>
+                            </execution>
+                            <execution>
+                                <id>python-api-clean</id>
+                                <phase>prepare-package</phase>
+                                <goals>
+                                    <goal>exec</goal>
+                                </goals>
+                                <configuration>
+                                    <executable>python3</executable>
+                                    <workingDirectory>${project.basedir}/pydolphinscheduler</workingDirectory>
+                                    <arguments>
+                                        <argument>setup.py</argument>
+                                        <argument>pre_clean</argument>
+                                    </arguments>
+                                </configuration>
+                            </execution>
+                            <execution>
+                                <id>python-api-build</id>
+                                <phase>prepare-package</phase>
+                                <goals>
+                                    <goal>exec</goal>
+                                </goals>
+                                <configuration>
+                                    <executable>python3</executable>
+                                    <workingDirectory>${project.basedir}/pydolphinscheduler</workingDirectory>
+                                    <arguments>
+                                        <argument>-m</argument>
+                                        <argument>build</argument>
+                                    </arguments>
+                                </configuration>
+                            </execution>
+                        </executions>
+                    </plugin>
+                </plugins>
+            </build>
+        </profile>
+    </profiles>
 </project>
diff --git a/dolphinscheduler-python/pydolphinscheduler/setup.py b/dolphinscheduler-python/pydolphinscheduler/setup.py
index cd6eb3eb77..dacb91b536 100644
--- a/dolphinscheduler-python/pydolphinscheduler/setup.py
+++ b/dolphinscheduler-python/pydolphinscheduler/setup.py
@@ -16,24 +16,35 @@
 # under the License.
 
 """The script for setting up pydolphinscheduler."""
-
+import logging
+import os
 import sys
+from distutils.dir_util import remove_tree
 from os.path import dirname, join
+from typing import List
 
-from setuptools import find_packages, setup
+from setuptools import Command, find_packages, setup
 
 if sys.version_info[0] < 3:
     raise Exception(
         "pydolphinscheduler does not support Python 2. Please upgrade to Python 3."
     )
 
-version = "0.1.0"
+logger = logging.getLogger(__name__)
+
+version = "2.0.4"
 
 # Start package required
 prod = [
     "py4j~=0.10",
 ]
 
+build = [
+    "build",
+    "setuptools>=42",
+    "wheel",
+]
+
 doc = [
     "sphinx>=4.3",
     "sphinx_rtd_theme>=1.0",
@@ -52,7 +63,7 @@ style = [
     "isort>=5.10",
 ]
 
-dev = style + test + doc
+dev = style + test + doc + build
 
 all_dep = prod + dev
 # End package required
@@ -65,6 +76,39 @@ def read(*names, **kwargs):
     ).read()
 
 
+class CleanCommand(Command):
+    """Command to clean up python api before setup by running `python setup.py pre_clean`."""
+
+    description = "Clean up project root"
+    user_options: List[str] = []
+    clean_list = [
+        "build",
+        "htmlcov",
+        "dist",
+        ".pytest_cache",
+        ".coverage",
+    ]
+
+    def initialize_options(self) -> None:
+        """Set default values for options."""
+        pass
+
+    def finalize_options(self) -> None:
+        """Set final values for options."""
+        pass
+
+    def run(self) -> None:
+        """Run and remove temporary files."""
+        for cl in self.clean_list:
+            if not os.path.exists(cl):
+                logger.info("Path %s do not exists.", cl)
+            elif os.path.isdir(cl):
+                remove_tree(cl)
+            else:
+                os.remove(cl)
+        logger.info("Finish pre_clean process.")
+
+
 setup(
     name="apache-dolphinscheduler",
     version=version,
@@ -124,5 +168,9 @@ setup(
         "style": style,
         "test": test,
         "doc": doc,
+        "build": build,
+    },
+    cmdclass={
+        "pre_clean": CleanCommand,
     },
 )