[Feature-8222][python] move examples into the scope of source package (#8340)

This commit is contained in:
Eric Gao 2022-02-14 15:07:00 +08:00 committed by GitHub
parent 703d790415
commit 49a1270ef5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
21 changed files with 41 additions and 18 deletions

View File

@ -23,7 +23,7 @@ A condition task type's example and dive into information of **PyDolphinSchedule
Example
-------
.. literalinclude:: ../../../examples/task_condition_example.py
.. literalinclude:: ../../../src/pydolphinscheduler/examples/task_condition_example.py
:start-after: [start workflow_declare]
:end-before: [end workflow_declare]

View File

@ -23,7 +23,7 @@ A DataX task type's example and dive into information of **PyDolphinScheduler**.
Example
-------
.. literalinclude:: ../../../examples/task_datax_example.py
.. literalinclude:: ../../../src/pydolphinscheduler/examples/task_datax_example.py
:start-after: [start workflow_declare]
:end-before: [end workflow_declare]

View File

@ -23,7 +23,7 @@ A dependent task type's example and dive into information of **PyDolphinSchedule
Example
-------
.. literalinclude:: ../../../examples/task_dependent_example.py
.. literalinclude:: ../../../src/pydolphinscheduler/examples/task_dependent_example.py
:start-after: [start workflow_declare]
:end-before: [end workflow_declare]

View File

@ -23,7 +23,7 @@ A flink task type's example and dive into information of **PyDolphinScheduler**.
Example
-------
.. literalinclude:: ../../../examples/task_flink_example.py
.. literalinclude:: ../../../src/pydolphinscheduler/examples/task_flink_example.py
:start-after: [start workflow_declare]
:end-before: [end workflow_declare]

View File

@ -24,7 +24,7 @@ A Map Reduce task type's example and dive into information of **PyDolphinSchedul
Example
-------
.. literalinclude:: ../../../examples/task_map_reduce_example.py
.. literalinclude:: ../../../src/pydolphinscheduler/examples/task_map_reduce_example.py
:start-after: [start workflow_declare]
:end-before: [end workflow_declare]

View File

@ -23,7 +23,7 @@ A shell task type's example and dive into information of **PyDolphinScheduler**.
Example
-------
.. literalinclude:: ../../../examples/tutorial.py
.. literalinclude:: ../../../src/pydolphinscheduler/examples/tutorial.py
:start-after: [start workflow_declare]
:end-before: [end task_relation_declare]

View File

@ -23,7 +23,7 @@ A spark task type's example and dive into information of **PyDolphinScheduler**.
Example
-------
.. literalinclude:: ../../../examples/task_spark_example.py
.. literalinclude:: ../../../src/pydolphinscheduler/examples/task_spark_example.py
:start-after: [start workflow_declare]
:end-before: [end workflow_declare]

View File

@ -23,7 +23,7 @@ A switch task type's example and dive into information of **PyDolphinScheduler**
Example
-------
.. literalinclude:: ../../../examples/task_switch_example.py
.. literalinclude:: ../../../src/pydolphinscheduler/examples/task_switch_example.py
:start-after: [start workflow_declare]
:end-before: [end workflow_declare]

View File

@ -29,7 +29,7 @@ Overview of Tutorial
Here have an overview of our tutorial, and it look a little complex but do not
worry about that because we explain this example below as detailed as possible.
.. literalinclude:: ../../examples/tutorial.py
.. literalinclude:: ../../src/pydolphinscheduler/examples/tutorial.py
:start-after: [start tutorial]
:end-before: [end tutorial]
@ -41,7 +41,7 @@ like other Python package. We just create a minimum demo here, so we just import
:class:`pydolphinscheduler.core.process_definition` and
:class:`pydolphinscheduler.tasks.shell`.
.. literalinclude:: ../../examples/tutorial.py
.. literalinclude:: ../../src/pydolphinscheduler/examples/tutorial.py
:start-after: [start package_import]
:end-before: [end package_import]
@ -60,7 +60,7 @@ interval and schedule start_time, and argument `tenant` which changing workflow'
task running user in the worker, :ref:`section tenant <concept:tenant>` in *PyDolphinScheduler*
:doc:`concept` page have more detail information.
.. literalinclude:: ../../examples/tutorial.py
.. literalinclude:: ../../src/pydolphinscheduler/examples/tutorial.py
:start-after: [start workflow_declare]
:end-before: [end workflow_declare]
@ -77,7 +77,7 @@ Here we declare four tasks, and bot of them are simple task of
Beside the argument `command`, we also need setting argument `name` for each task *(not
only shell task, `name` is required for each type of task)*.
.. literalinclude:: ../../examples/tutorial.py
.. literalinclude:: ../../src/pydolphinscheduler/examples/tutorial.py
:dedent: 0
:start-after: [start task_declare]
:end-before: [end task_declare]
@ -99,7 +99,7 @@ In this example, we set task `task_parent` is the upstream task of task
`task_child_one` and `task_child_two`, and task `task_union` is the downstream
task of both these two task.
.. literalinclude:: ../../examples/tutorial.py
.. literalinclude:: ../../src/pydolphinscheduler/examples/tutorial.py
:dedent: 0
:start-after: [start task_relation_declare]
:end-before: [end task_relation_declare]
@ -124,7 +124,7 @@ to the daemon, and set the schedule time we just declare in `process definition
Now, we could run the Python code like other Python script, for the basic usage run
:code:`python tutorial.py` to trigger and run it.
.. literalinclude:: ../../examples/tutorial.py
.. literalinclude:: ../../src/pydolphinscheduler/examples/tutorial.py
:dedent: 0
:start-after: [start submit_or_run]
:end-before: [end submit_or_run]
@ -142,7 +142,7 @@ After we run the tutorial code, you could login Apache DolphinScheduler web UI,
go and see the `DolphinScheduler project page`_. they is a new process definition be
created and named "Tutorial". It create by *PyDolphinScheduler* and the DAG graph as below
.. literalinclude:: ../../examples/tutorial.py
.. literalinclude:: ../../src/pydolphinscheduler/examples/tutorial.py
:language: text
:lines: 24-28

View File

@ -0,0 +1,18 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
"""Init examples package which provides users with pydolphinscheduler examples."""

View File

@ -96,8 +96,8 @@ def test_example_basic():
ex.suffix == ".py"
), f"We expect all examples is python script, but get {ex.name}."
# All except tutorial is end with keyword "_example"
if ex.stem != "tutorial":
# All except tutorial and __init__ is end with keyword "_example"
if ex.stem != "tutorial" and ex.stem != "__init__":
assert ex.stem.endswith(
"_example"
), f"We expect all examples script end with keyword '_example', but get {ex.stem}."
@ -159,6 +159,9 @@ def test_file_name_in_process_definition(mock_code_version, mock_submit, mock_st
"""
global process_definition_name
for ex in get_all_examples():
# Skip __init__ file
if ex.stem == "__init__":
continue
# Skip bulk_create_example check, cause it contain multiple workflow and
# without one named bulk_create_example
if ex.stem == "bulk_create_example":

View File

@ -23,7 +23,9 @@ from typing import Any, Generator
path_code_tasks = Path(__file__).parent.parent.parent.joinpath(
"src", "pydolphinscheduler", "tasks"
)
path_example = Path(__file__).parent.parent.parent.joinpath("examples")
path_example = Path(__file__).parent.parent.parent.joinpath(
"src", "pydolphinscheduler", "examples"
)
path_doc_tasks = Path(__file__).parent.parent.parent.joinpath("docs", "source", "tasks")