diff --git a/deploy/docker/docker-compose.yml b/deploy/docker/docker-compose.yml index 83ca8a644a..2c557c4dd7 100644 --- a/deploy/docker/docker-compose.yml +++ b/deploy/docker/docker-compose.yml @@ -140,29 +140,6 @@ services: networks: - dolphinscheduler - dolphinscheduler-python-gateway: - image: ${HUB}/dolphinscheduler-python:${TAG} - ports: - - "54321:54321" - - "25333:25333" - env_file: .env - healthcheck: - test: [ "CMD", "curl", "http://localhost:54321/actuator/health" ] - interval: 30s - timeout: 5s - retries: 3 - depends_on: - dolphinscheduler-schema-initializer: - condition: service_completed_successfully - dolphinscheduler-zookeeper: - condition: service_healthy - volumes: - - dolphinscheduler-logs:/opt/dolphinscheduler/logs - - dolphinscheduler-shared-local:/opt/soft - - dolphinscheduler-resource-local:/dolphinscheduler - networks: - - dolphinscheduler - networks: dolphinscheduler: driver: bridge diff --git a/deploy/docker/docker-stack.yml b/deploy/docker/docker-stack.yml index 8d0ae92553..02ee37be02 100644 --- a/deploy/docker/docker-stack.yml +++ b/deploy/docker/docker-stack.yml @@ -118,27 +118,6 @@ services: mode: replicated replicas: 1 - dolphinscheduler-python-gateway: - image: apache/dolphinscheduler-python-gateway - ports: - - 54321:54321 - - 25333:25333 - env_file: .env - healthcheck: - test: [ "CMD", "curl", "http://localhost:54321/actuator/health" ] - interval: 30s - timeout: 5s - retries: 3 - volumes: - - dolphinscheduler-logs:/opt/dolphinscheduler/logs - - dolphinscheduler-shared-local:/opt/soft - - dolphinscheduler-resource-local:/dolphinscheduler - networks: - - dolphinscheduler - deploy: - mode: replicated - replicas: 1 - networks: dolphinscheduler: driver: overlay diff --git a/deploy/kubernetes/dolphinscheduler/templates/_helpers.tpl b/deploy/kubernetes/dolphinscheduler/templates/_helpers.tpl index f8fef1167a..c2684e029a 100644 --- a/deploy/kubernetes/dolphinscheduler/templates/_helpers.tpl +++ b/deploy/kubernetes/dolphinscheduler/templates/_helpers.tpl @@ -44,9 +44,6 @@ Create default docker images' fullname. {{- define "dolphinscheduler.image.fullname.tools" -}} {{- .Values.image.registry }}/dolphinscheduler-tools:{{ .Values.image.tag | default .Chart.AppVersion -}} {{- end -}} -{{- define "dolphinscheduler.image.fullname.python-gateway" -}} -{{- .Values.image.registry }}/dolphinscheduler-python-gateway:{{ .Values.image.tag | default .Chart.AppVersion -}} -{{- end -}} {{/* Create a default common labels. diff --git a/docs/docs/en/faq.md b/docs/docs/en/faq.md index c920dfa327..f4a9d44bfc 100644 --- a/docs/docs/en/faq.md +++ b/docs/docs/en/faq.md @@ -712,6 +712,12 @@ A：The repair can be completed by executing the following SQL in the database: update t_ds_version set version='2.0.1'; ``` +## Can not find python-gateway-server in distribute package + +After version 3.0.0-alpha, Python gateway server integrate into API server, and Python gateway service will start when you +start API server. If you want disabled when Python gateway service you could change API server configuration in path +`api-server/conf/application.yaml` and change attribute `python-gateway.enabled : false`. + --- ## We will collect more FAQ later \ No newline at end of file diff --git a/docs/docs/en/guide/installation/pseudo-cluster.md b/docs/docs/en/guide/installation/pseudo-cluster.md index b887030e4a..333aecc762 100644 --- a/docs/docs/en/guide/installation/pseudo-cluster.md +++ b/docs/docs/en/guide/installation/pseudo-cluster.md @@ -193,7 +193,9 @@ sh ./bin/dolphinscheduler-daemon.sh start alert-server sh ./bin/dolphinscheduler-daemon.sh stop alert-server ``` -> **_Note:_**: Please refer to the section of "System Architecture Design" for service usage +> **_Note:_**: Please refer to the section of "System Architecture Design" for service usage. Python gateway service is +> started along with the api-server, and if you do not want to start Python gateway service please disabled it by changing +> the yaml config `python-gateway.enabled : false` in api-server's configuration path `api-server/conf/application.yaml` [jdk]: https://www.oracle.com/technetwork/java/javase/downloads/index.html [zookeeper]: https://zookeeper.apache.org/releases.html diff --git a/docs/docs/en/guide/installation/standalone.md b/docs/docs/en/guide/installation/standalone.md index a03944c17c..c9e37db7f1 100644 --- a/docs/docs/en/guide/installation/standalone.md +++ b/docs/docs/en/guide/installation/standalone.md @@ -39,4 +39,8 @@ sh ./bin/dolphinscheduler-daemon.sh start standalone-server sh ./bin/dolphinscheduler-daemon.sh stop standalone-server ``` +> Note: Python gateway service is started along with the api-server, and if you do not want to start Python gateway +> service please disabled it by changing the yaml config `python-gateway.enabled : false` in api-server's configuration +> path `api-server/conf/application.yaml` + [jdk]: https://www.oracle.com/technetwork/java/javase/downloads/index.html diff --git a/docs/docs/en/guide/upgrade.md b/docs/docs/en/guide/upgrade.md index b401ce1354..38f80d8c73 100644 --- a/docs/docs/en/guide/upgrade.md +++ b/docs/docs/en/guide/upgrade.md @@ -6,7 +6,7 @@ `sh ./script/stop-all.sh` -## Download the Newest Version Installation Package +## Download the Latest Version Installation Package - [download](/en-us/download/download.html) the latest version of the installation packages. - The following upgrade operations need to be performed in the new version's directory. diff --git a/dolphinscheduler-api/pom.xml b/dolphinscheduler-api/pom.xml index d162239c7c..156a4d1160 100644 --- a/dolphinscheduler-api/pom.xml +++ b/dolphinscheduler-api/pom.xml @@ -275,6 +275,12 @@ test + + + + net.sf.py4j + py4j + diff --git a/dolphinscheduler-api/src/main/docker/Dockerfile b/dolphinscheduler-api/src/main/docker/Dockerfile index 9a1a8c0ca1..946f2ff46a 100644 --- a/dolphinscheduler-api/src/main/docker/Dockerfile +++ b/dolphinscheduler-api/src/main/docker/Dockerfile @@ -29,6 +29,6 @@ WORKDIR $DOLPHINSCHEDULER_HOME ADD ./target/api-server $DOLPHINSCHEDULER_HOME -EXPOSE 12345 +EXPOSE 12345 25333 CMD [ "/bin/bash", "./bin/start.sh" ] diff --git a/dolphinscheduler-python/src/main/java/org/apache/dolphinscheduler/server/config/PythonGatewayConfig.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/configuration/PythonGatewayConfiguration.java similarity index 86% rename from dolphinscheduler-python/src/main/java/org/apache/dolphinscheduler/server/config/PythonGatewayConfig.java rename to dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/configuration/PythonGatewayConfiguration.java index f06ea765b6..5735e27fd2 100644 --- a/dolphinscheduler-python/src/main/java/org/apache/dolphinscheduler/server/config/PythonGatewayConfig.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/configuration/PythonGatewayConfiguration.java @@ -15,17 +15,17 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.server.config; +package org.apache.dolphinscheduler.api.configuration; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.context.annotation.Configuration; import org.springframework.stereotype.Component; @Component @EnableConfigurationProperties -@ConfigurationProperties("python-gateway") -public class PythonGatewayConfig { +@ConfigurationProperties(value = "python-gateway", ignoreUnknownFields = false) +public class PythonGatewayConfiguration { + private boolean enabled; private String gatewayServerAddress; private int gatewayServerPort; private String pythonAddress; @@ -33,6 +33,14 @@ public class PythonGatewayConfig { private int connectTimeout; private int readTimeout; + public boolean getEnabled() { + return enabled; + } + + public void setEnabled(boolean enabled) { + this.enabled = enabled; + } + public String getGatewayServerAddress() { return gatewayServerAddress; } diff --git a/dolphinscheduler-python/src/main/java/org/apache/dolphinscheduler/server/PythonGatewayServer.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/python/PythonGateway.java similarity index 95% rename from dolphinscheduler-python/src/main/java/org/apache/dolphinscheduler/server/PythonGatewayServer.java rename to dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/python/PythonGateway.java index 2f7fc94da7..8aea47471b 100644 --- a/dolphinscheduler-python/src/main/java/org/apache/dolphinscheduler/server/PythonGatewayServer.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/python/PythonGateway.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.server; +package org.apache.dolphinscheduler.api.python; import org.apache.dolphinscheduler.api.dto.resources.ResourceComponent; import org.apache.dolphinscheduler.api.enums.Status; @@ -56,7 +56,7 @@ import org.apache.dolphinscheduler.dao.mapper.ProjectMapper; import org.apache.dolphinscheduler.dao.mapper.ProjectUserMapper; import org.apache.dolphinscheduler.dao.mapper.ScheduleMapper; import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionMapper; -import org.apache.dolphinscheduler.server.config.PythonGatewayConfig; +import org.apache.dolphinscheduler.api.configuration.PythonGatewayConfiguration; import org.apache.dolphinscheduler.spi.enums.ResourceType; import org.apache.commons.collections.CollectionUtils; @@ -75,17 +75,13 @@ import javax.annotation.PostConstruct; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.SpringApplication; -import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.boot.web.servlet.support.SpringBootServletInitializer; -import org.springframework.context.annotation.ComponentScan; +import org.springframework.stereotype.Component; import py4j.GatewayServer; -@SpringBootApplication -@ComponentScan(value = "org.apache.dolphinscheduler") -public class PythonGatewayServer extends SpringBootServletInitializer { - private static final Logger logger = LoggerFactory.getLogger(PythonGatewayServer.class); +@Component +public class PythonGateway { + private static final Logger logger = LoggerFactory.getLogger(PythonGateway.class); private static final WarningType DEFAULT_WARNING_TYPE = WarningType.NONE; private static final int DEFAULT_WARNING_GROUP_ID = 0; @@ -141,7 +137,7 @@ public class PythonGatewayServer extends SpringBootServletInitializer { private DataSourceMapper dataSourceMapper; @Autowired - private PythonGatewayConfig pythonGatewayConfig; + private PythonGatewayConfiguration pythonGatewayConfiguration; @Autowired private ProjectUserMapper projectUserMapper; @@ -546,30 +542,32 @@ public class PythonGatewayServer extends SpringBootServletInitializer { } @PostConstruct - public void run() { - GatewayServer server; - try { - InetAddress gatewayHost = InetAddress.getByName(pythonGatewayConfig.getGatewayServerAddress()); - InetAddress pythonHost = InetAddress.getByName(pythonGatewayConfig.getPythonAddress()); - server = new GatewayServer( - this, - pythonGatewayConfig.getGatewayServerPort(), - pythonGatewayConfig.getPythonPort(), - gatewayHost, - pythonHost, - pythonGatewayConfig.getConnectTimeout(), - pythonGatewayConfig.getReadTimeout(), - null - ); - GatewayServer.turnLoggingOn(); - logger.info("PythonGatewayServer started on: " + gatewayHost.toString()); - server.start(); - } catch (UnknownHostException e) { - logger.error("exception occurred while constructing PythonGatewayServer().", e); + public void init() { + if (pythonGatewayConfiguration.getEnabled()) { + this.start(); } } - public static void main(String[] args) { - SpringApplication.run(PythonGatewayServer.class, args); + private void start() { + GatewayServer server; + try { + InetAddress gatewayHost = InetAddress.getByName(pythonGatewayConfiguration.getGatewayServerAddress()); + InetAddress pythonHost = InetAddress.getByName(pythonGatewayConfiguration.getPythonAddress()); + server = new GatewayServer( + this, + pythonGatewayConfiguration.getGatewayServerPort(), + pythonGatewayConfiguration.getPythonPort(), + gatewayHost, + pythonHost, + pythonGatewayConfiguration.getConnectTimeout(), + pythonGatewayConfiguration.getReadTimeout(), + null + ); + GatewayServer.turnLoggingOn(); + logger.info("PythonGatewayService started on: " + gatewayHost.toString()); + server.start(); + } catch (UnknownHostException e) { + logger.error("exception occurred while constructing PythonGatewayService().", e); + } } } diff --git a/dolphinscheduler-api/src/main/resources/application.yaml b/dolphinscheduler-api/src/main/resources/application.yaml index 3db21c3a04..23b93028af 100644 --- a/dolphinscheduler-api/src/main/resources/application.yaml +++ b/dolphinscheduler-api/src/main/resources/application.yaml @@ -108,6 +108,26 @@ audit: metrics: enabled: true +python-gateway: + # Weather enable python gateway server or not. The default value is true. + enabled: true + # The address of Python gateway server start. Set its value to `0.0.0.0` if your Python API run in different + # between Python gateway server. It could be be specific to other address like `127.0.0.1` or `localhost` + gateway-server-address: 0.0.0.0 + # The port of Python gateway server start. Define which port you could connect to Python gateway server from + # Python API side. + gateway-server-port: 25333 + # The address of Python callback client. + python-address: 127.0.0.1 + # The port of Python callback client. + python-port: 25334 + # Close connection of socket server if no other request accept after x milliseconds. Define value is (0 = infinite), + # and socket server would never close even though no requests accept + connect-timeout: 0 + # Close each active connection of socket server if python program not active after x milliseconds. Define value is + # (0 = infinite), and socket server would never close even though no requests accept + read-timeout: 0 + # Override by profile --- diff --git a/dolphinscheduler-dist/src/main/assembly/dolphinscheduler-bin.xml b/dolphinscheduler-dist/src/main/assembly/dolphinscheduler-bin.xml index 3b113741c1..1049d2720a 100644 --- a/dolphinscheduler-dist/src/main/assembly/dolphinscheduler-bin.xml +++ b/dolphinscheduler-dist/src/main/assembly/dolphinscheduler-bin.xml @@ -50,11 +50,6 @@ logger-server - - ${basedir}/../dolphinscheduler-python/target/python-gateway-server - python-gateway-server - - ${basedir}/../dolphinscheduler-standalone-server/target/standalone-server standalone-server diff --git a/dolphinscheduler-python/pom.xml b/dolphinscheduler-python/pom.xml index 00aaaf8f38..4a66459cae 100644 --- a/dolphinscheduler-python/pom.xml +++ b/dolphinscheduler-python/pom.xml @@ -28,82 +28,7 @@ ${project.artifactId} jar - - - - org.apache.dolphinscheduler - dolphinscheduler-api - - - - - org.springframework.boot - spring-boot-starter-web - - - org.springframework.boot - spring-boot-starter-tomcat - - - log4j-to-slf4j - org.apache.logging.log4j - - - - - - net.sf.py4j - py4j - - - - - - - - org.apache.maven.plugins - maven-jar-plugin - - - *.yaml - *.xml - - - - - maven-assembly-plugin - - - dolphinscheduler-python-gateway-server - package - - single - - - python-gateway-server - - src/main/assembly/dolphinscheduler-python-gateway-server.xml - - false - - - - - - - - - docker - - - - org.codehaus.mojo - exec-maven-plugin - - - - python diff --git a/dolphinscheduler-python/pydolphinscheduler/UPDATING.md b/dolphinscheduler-python/pydolphinscheduler/UPDATING.md index cf45c097fb..430e4b626f 100644 --- a/dolphinscheduler-python/pydolphinscheduler/UPDATING.md +++ b/dolphinscheduler-python/pydolphinscheduler/UPDATING.md @@ -24,6 +24,9 @@ It started after version 2.0.5 released ## dev +* Integrate Python gateway server into Dolphinscheduler API server, and you could start Python gateway service by command + `./bin/dolphinscheduler-daemon.sh start api-server` instead of independent command + `./bin/dolphinscheduler-daemon.sh start python-gateway-server`. * Remove parameter `queue` from class `ProcessDefinition` to avoid confuse user when it change but not work * Change `yaml_parser.py` method `to_string` to magic method `__str__` make it more pythonic. * Use package ``ruamel.yaml`` replace ``pyyaml`` for write yaml file with comment. diff --git a/dolphinscheduler-python/pydolphinscheduler/docs/source/start.rst b/dolphinscheduler-python/pydolphinscheduler/docs/source/start.rst index 0af90d5494..489f4e8eb3 100644 --- a/dolphinscheduler-python/pydolphinscheduler/docs/source/start.rst +++ b/dolphinscheduler-python/pydolphinscheduler/docs/source/start.rst @@ -55,7 +55,7 @@ After Python is already installed on your machine following section $ pip install apache-dolphinscheduler The latest version of *PyDolphinScheduler* would be installed after you run above -command in your terminal. You could go and `start Python Gateway Server`_ to finish +command in your terminal. You could go and `start Python Gateway Service`_ to finish the prepare, and then go to :doc:`tutorial` to make your hand dirty. But if you want to install the unreleased version of *PyDolphinScheduler*, you could go and see section `installing PyDolphinScheduler in dev`_ for more detail. @@ -74,33 +74,39 @@ which we hold in GitHub # Install PyDolphinScheduler in develop mode $ cd dolphinscheduler-python/pydolphinscheduler && pip install -e . -After you installed *PyDolphinScheduler*, please remember `start Python Gateway Server`_ +After you installed *PyDolphinScheduler*, please remember `start Python Gateway Service`_ which waiting for *PyDolphinScheduler*'s workflow definition require. -Start Python Gateway Server ---------------------------- +Start Python Gateway Service +---------------------------- Since **PyDolphinScheduler** is Python API for `Apache DolphinScheduler`_, it could define workflow and tasks structure, but could not run it unless you -`install Apache DolphinScheduler`_ and start Python gateway server. We only -and some key steps here and you could go `install Apache DolphinScheduler`_ -for more detail +`install Apache DolphinScheduler`_ and start its API server which including +Python gateway service in it. We only and some key steps here and you could +go `install Apache DolphinScheduler`_ for more detail .. code-block:: bash - # Start pythonGatewayServer - $ ./bin/dolphinscheduler-daemon.sh start pythonGatewayServer + # Start DolphinScheduler api-server which including python gateway service + $ ./bin/dolphinscheduler-daemon.sh start api-server To check whether the server is alive or not, you could run :code:`jps`. And -the server is health if keyword `PythonGatewayServer` in the console. +the server is health if keyword `ApiApplicationServer` in the console. .. code-block:: bash $ jps .... - 201472 PythonGatewayServer + 201472 ApiApplicationServer .... +.. note:: + + Please make sure you already enabled started Python gateway service along with `api-server`. The configuration is in + yaml config path `python-gateway.enabled : true` in api-server's configuration path in `api-server/conf/application.yaml`. + The default value is true and Python gateway service start when api server is been started. + What's More ----------- diff --git a/dolphinscheduler-python/pydolphinscheduler/docs/source/tutorial.rst b/dolphinscheduler-python/pydolphinscheduler/docs/source/tutorial.rst index 83c5746aba..e0f22fb816 100644 --- a/dolphinscheduler-python/pydolphinscheduler/docs/source/tutorial.rst +++ b/dolphinscheduler-python/pydolphinscheduler/docs/source/tutorial.rst @@ -130,7 +130,7 @@ Now, we could run the Python code like other Python script, for the basic usage :end-before: [end submit_or_run] If you not start your Apache DolphinScheduler server, you could find the way in -:ref:`start:start Python gateway server` and it would have more detail about related server +:ref:`start:start Python gateway service` and it would have more detail about related server start. Beside attribute `run`, we have attribute `submit` for object `ProcessDefinition` and it just submit workflow to the daemon but not setting the schedule information. For more detail you could see :ref:`concept:process definition`. diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/integration/__init__.py b/dolphinscheduler-python/pydolphinscheduler/tests/integration/__init__.py index 8079bfba1f..65625a9f04 100644 --- a/dolphinscheduler-python/pydolphinscheduler/tests/integration/__init__.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/integration/__init__.py @@ -15,4 +15,4 @@ # specific language governing permissions and limitations # under the License. -"""Test integration between Python API and PythonGatewayServer.""" +"""Test integration between Python API and PythonGatewayService.""" diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/integration/test_submit_examples.py b/dolphinscheduler-python/pydolphinscheduler/tests/integration/test_submit_examples.py index 0964e1b975..85e5e23e31 100644 --- a/dolphinscheduler-python/pydolphinscheduler/tests/integration/test_submit_examples.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/integration/test_submit_examples.py @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -"""Test whether success submit examples DAG to PythonGatewayServer.""" +"""Test whether success submit examples DAG to PythonGatewayService.""" from pathlib import Path @@ -34,7 +34,7 @@ from tests.testing.path import path_example ], ) def test_exec_white_list_example(example_path: Path): - """Test execute examples and submit DAG to PythonGatewayServer.""" + """Test execute examples and submit DAG to PythonGatewayService.""" try: exec(example_path.read_text()) except Exception: diff --git a/dolphinscheduler-python/src/main/assembly/dolphinscheduler-python-gateway-server.xml b/dolphinscheduler-python/src/main/assembly/dolphinscheduler-python-gateway-server.xml deleted file mode 100644 index 49e03206ea..0000000000 --- a/dolphinscheduler-python/src/main/assembly/dolphinscheduler-python-gateway-server.xml +++ /dev/null @@ -1,64 +0,0 @@ - - - - dolphinscheduler-python-gateway-server - - dir - - false - python-gateway-server - - - ${basedir}/src/main/resources - - *.yaml - *.xml - - conf - - - ${basedir}/src/main/bin - bin - 0755 - 0755 - - - ${basedir}/../script/env - bin - - dolphinscheduler_env.sh - - 0755 - 0755 - - - ${basedir}/../dolphinscheduler-common/src/main/resources - - **/*.properties - - conf - - - - - libs - - - diff --git a/dolphinscheduler-python/src/main/bin/start.sh b/dolphinscheduler-python/src/main/bin/start.sh deleted file mode 100644 index d588a2b2d9..0000000000 --- a/dolphinscheduler-python/src/main/bin/start.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -BIN_DIR=$(dirname $0) -DOLPHINSCHEDULER_HOME=${DOLPHINSCHEDULER_HOME:-$(cd $BIN_DIR/..; pwd)} - -source "$BIN_DIR/dolphinscheduler_env.sh" - -JAVA_OPTS=${JAVA_OPTS:-"-server -Duser.timezone=${SPRING_JACKSON_TIME_ZONE} -Xms1g -Xmx1g -Xmn512m -XX:+PrintGCDetails -Xloggc:gc.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=dump.hprof"} - -if [[ "$DOCKER" == "true" ]]; then - JAVA_OPTS="${JAVA_OPTS} -XX:-UseContainerSupport" -fi - -java $JAVA_OPTS \ - -cp "$DOLPHINSCHEDULER_HOME/conf":"$DOLPHINSCHEDULER_HOME/libs/*" \ - org.apache.dolphinscheduler.server.PythonGatewayServer diff --git a/dolphinscheduler-python/src/main/docker/Dockerfile b/dolphinscheduler-python/src/main/docker/Dockerfile deleted file mode 100644 index 7f50a8fc39..0000000000 --- a/dolphinscheduler-python/src/main/docker/Dockerfile +++ /dev/null @@ -1,34 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -FROM openjdk:8-jre-slim-buster - -ENV DOCKER true -ENV TZ Asia/Shanghai -ENV DOLPHINSCHEDULER_HOME /opt/dolphinscheduler - -RUN apt update ; \ - apt install -y curl sudo ; \ - rm -rf /var/lib/apt/lists/* - -WORKDIR $DOLPHINSCHEDULER_HOME - -ADD ./target/python-gateway-server $DOLPHINSCHEDULER_HOME - -EXPOSE 25333 54321 - -CMD [ "/bin/bash", "./bin/start.sh" ] diff --git a/dolphinscheduler-python/src/main/resources/application.yaml b/dolphinscheduler-python/src/main/resources/application.yaml deleted file mode 100644 index b899c03638..0000000000 --- a/dolphinscheduler-python/src/main/resources/application.yaml +++ /dev/null @@ -1,83 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -spring: - application: - name: python-gateway-server - main: - banner-mode: off - datasource: - driver-class-name: org.h2.Driver - url: jdbc:h2:mem:dolphinscheduler;MODE=MySQL;DB_CLOSE_DELAY=-1;DATABASE_TO_LOWER=true - username: sa - password: "" - jackson: - time-zone: UTC - date-format: "yyyy-MM-dd HH:mm:ss" - servlet: - multipart: - max-file-size: 1024MB - max-request-size: 1024MB - messages: - basename: i18n/messages - jpa: - hibernate: - ddl-auto: none - -python-gateway: - # The address of Python gateway server start. Set its value to `0.0.0.0` if your Python API run in different - # between Python gateway server. It could be be specific to other address like `127.0.0.1` or `localhost` - gateway-server-address: 0.0.0.0 - # The port of Python gateway server start. Define which port you could connect to Python gateway server from - # Python API side. - gateway-server-port: 25333 - # The address of Python callback client. - python-address: 127.0.0.1 - # The port of Python callback client. - python-port: 25334 - # Close connection of socket server if no other request accept after x milliseconds. Define value is (0 = infinite), - # and socket server would never close even though no requests accept - connect-timeout: 0 - # Close each active connection of socket server if python program not active after x milliseconds. Define value is - # (0 = infinite), and socket server would never close even though no requests accept - read-timeout: 0 - -server: - port: 54321 - -management: - endpoints: - web: - exposure: - include: '*' - metrics: - tags: - application: ${spring.application.name} - -metrics: - enabled: true - -# Override by profile ---- -spring: - config: - activate: - on-profile: postgresql - quartz: - properties: - org.quartz.jobStore.driverDelegateClass: org.quartz.impl.jdbcjobstore.PostgreSQLDelegate - diff --git a/dolphinscheduler-python/src/main/resources/logback-spring.xml b/dolphinscheduler-python/src/main/resources/logback-spring.xml deleted file mode 100644 index 4f06d160d6..0000000000 --- a/dolphinscheduler-python/src/main/resources/logback-spring.xml +++ /dev/null @@ -1,57 +0,0 @@ - - - - - - - - - - [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n - - UTF-8 - - - - - ${log.base}/dolphinscheduler-python-gateway.log - - INFO - - - ${log.base}/dolphinscheduler-python-gateway.%d{yyyy-MM-dd_HH}.%i.log - 168 - 64MB - - - - [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n - - UTF-8 - - - - - - - - - - - - diff --git a/dolphinscheduler-standalone-server/src/main/assembly/dolphinscheduler-standalone-server.xml b/dolphinscheduler-standalone-server/src/main/assembly/dolphinscheduler-standalone-server.xml index a201d509db..e66fce1287 100644 --- a/dolphinscheduler-standalone-server/src/main/assembly/dolphinscheduler-standalone-server.xml +++ b/dolphinscheduler-standalone-server/src/main/assembly/dolphinscheduler-standalone-server.xml @@ -45,10 +45,6 @@ ${basedir}/../dolphinscheduler-log-server/target/logger-server/libs libs/logger-server - - ${basedir}/../dolphinscheduler-python/target/python-gateway-server/libs - libs/python-gateway - ${basedir}/src/main/resources diff --git a/dolphinscheduler-standalone-server/src/main/dist-bin/start.sh b/dolphinscheduler-standalone-server/src/main/dist-bin/start.sh index 4ef3a11d36..019c61a7bf 100755 --- a/dolphinscheduler-standalone-server/src/main/dist-bin/start.sh +++ b/dolphinscheduler-standalone-server/src/main/dist-bin/start.sh @@ -28,7 +28,7 @@ if [[ "$DOCKER" == "true" ]]; then fi CP=$DOLPHINSCHEDULER_HOME/libs/standalone-server/* -for d in alert-server api-server master-server python-gateway-server worker-server; do +for d in alert-server api-server master-server worker-server; do for f in $DOLPHINSCHEDULER_HOME/../$d/libs/*.jar; do CP=$CP:$f done diff --git a/dolphinscheduler-standalone-server/src/main/resources/application.yaml b/dolphinscheduler-standalone-server/src/main/resources/application.yaml index 7950d80177..a2894bfd2a 100644 --- a/dolphinscheduler-standalone-server/src/main/resources/application.yaml +++ b/dolphinscheduler-standalone-server/src/main/resources/application.yaml @@ -140,6 +140,8 @@ alert: port: 50052 python-gateway: + # Weather enable python gateway server or not. The default value is true. + enabled: true # The address of Python gateway server start. Set its value to `0.0.0.0` if your Python API run in different # between Python gateway server. It could be be specific to other address like `127.0.0.1` or `localhost` gateway-server-address: 0.0.0.0 diff --git a/script/dolphinscheduler-daemon.sh b/script/dolphinscheduler-daemon.sh index 61e732590c..1ded5074fd 100755 --- a/script/dolphinscheduler-daemon.sh +++ b/script/dolphinscheduler-daemon.sh @@ -16,7 +16,7 @@ # limitations under the License. # -usage="Usage: dolphinscheduler-daemon.sh (start|stop|status) " +usage="Usage: dolphinscheduler-daemon.sh (start|stop|status) " # if no args specified, show usage if [ $# -le 1 ]; then @@ -61,8 +61,6 @@ elif [ "$command" = "alert-server" ]; then log=$DOLPHINSCHEDULER_HOME/alert-server/logs/$command-$HOSTNAME.out elif [ "$command" = "standalone-server" ]; then log=$DOLPHINSCHEDULER_HOME/standalone-server/logs/$command-$HOSTNAME.out -elif [ "$command" = "python-gateway-server" ]; then - log=$DOLPHINSCHEDULER_HOME/python-gateway-server/logs/$command-$HOSTNAME.out else echo "Error: No command named '$command' was found." exit 1 diff --git a/script/env/install_env.sh b/script/env/install_env.sh index 2df11c14ae..8b68c0ea00 100644 --- a/script/env/install_env.sh +++ b/script/env/install_env.sh @@ -48,11 +48,6 @@ alertServer=${alertServer:-"ds3"} # Example for hostname: apiServers="ds1", Example for IP: apiServers="192.168.8.1" apiServers=${apiServers:-"ds1"} -# A comma separated list of machine hostname or IP would be installed Python gateway server, it -# must be a subset of configuration `ips`. -# Example for hostname: pythonGatewayServers="ds1", Example for IP: pythonGatewayServers="192.168.8.1" -pythonGatewayServers=${pythonGatewayServers:-"ds1"} - # The directory to install DolphinScheduler for all machine we config above. It will automatically be created by `install.sh` script if not exists. # Do not set this configuration same as the current path (pwd) installPath=${installPath:-"/tmp/dolphinscheduler"} diff --git a/script/scp-hosts.sh b/script/scp-hosts.sh index 1b216ab41e..2397ede936 100755 --- a/script/scp-hosts.sh +++ b/script/scp-hosts.sh @@ -49,7 +49,7 @@ do echo "scp dirs to $host/$installPath starting" ssh -p $sshPort $host "cd $installPath/; rm -rf bin/ conf/ lib/ script/ sql/ ui/" - for dsDir in bin master-server worker-server alert-server api-server ui python-gateway-server + for dsDir in bin master-server worker-server alert-server api-server ui do # if worker in workersGroupMap if [[ "${workersGroupMap[${host}]}" ]]; then diff --git a/script/start-all.sh b/script/start-all.sh index e7934da8c1..002f99d873 100755 --- a/script/start-all.sh +++ b/script/start-all.sh @@ -56,13 +56,6 @@ do ssh -p $sshPort $apiServer "cd $installPath/; sh bin/dolphinscheduler-daemon.sh start api-server;" done -pythonGatewayHost=(${pythonGatewayServers//,/ }) -for pythonGatewayServer in "${pythonGatewayHost[@]}" -do - echo "$pythonGatewayServer python gateway server is starting" - ssh -p $sshPort $pythonGatewayServer "cd $installPath/; sh bin/dolphinscheduler-daemon.sh start python-gateway-server;" -done - # query server status echo "query server status" cd $installPath/; sh bin/status-all.sh diff --git a/script/status-all.sh b/script/status-all.sh index c3b4591a62..df19eb9c37 100755 --- a/script/status-all.sh +++ b/script/status-all.sh @@ -74,11 +74,3 @@ do apiState=`ssh -p $sshPort $apiServer "cd $installPath/; sh bin/dolphinscheduler-daemon.sh status api-server;"` echo "$apiServer $apiState" done - -# python gateway server check state -pythonGatewayHost=(${pythonGatewayServers//,/ }) -for pythonGatewayServer in "${pythonGatewayHost[@]}" -do - pythonGatewayState=`ssh -p $sshPort $pythonGatewayServer "cd $installPath/; sh bin/dolphinscheduler-daemon.sh status python-gateway-server;"` - echo "$pythonGatewayServer $pythonGatewayState" -done diff --git a/script/stop-all.sh b/script/stop-all.sh index 856340c390..6592374659 100755 --- a/script/stop-all.sh +++ b/script/stop-all.sh @@ -54,10 +54,3 @@ do echo "$apiServer api server is stopping" ssh -p $sshPort $apiServer "cd $installPath/; sh bin/dolphinscheduler-daemon.sh stop api-server;" done - -pythonGatewayHost=(${pythonGatewayServers//,/ }) -for pythonGatewayServer in "${pythonGatewayHost[@]}" -do - echo "$pythonGatewayServer python gateway server is stopping" - ssh -p $sshPort $pythonGatewayServer "cd $installPath/; sh bin/dolphinscheduler-daemon.sh stop python-gateway-server;" -done