Compare commits
18 Commits
dev
...
3.2.2-prep
| Author | SHA1 | Date |
|---|---|---|
wangxj3
|
0a21422558 | |
|
Wenjun Ruan
|
1e3bbfce77 | |
|
Wenjun Ruan
|
f13f070242 | |
|
xiangzihao
|
66b7d1274a | |
|
xiangzihao
|
fe96907fbb | |
|
Wenjun Ruan
|
e65208b34e | |
|
Wenjun Ruan
|
2f928b15c9 | |
|
zhuxt2015
|
e524cfe744 | |
|
xiangzihao
|
16bc1e7bfb | |
|
xiangzihao
|
3cdf754b1b | |
|
Wenjun Ruan
|
82ec84f85c | |
|
wangxj3
|
3ad53c5200 | |
|
Wenjun Ruan
|
e91a272965 | |
|
Wenjun Ruan
|
31e79d10ea | |
|
Wenjun Ruan
|
50c434e6c4 | |
|
Wenjun Ruan
|
72b402422a | |
|
xiangzihao
|
fd864bf351 | |
|
wangxj3
|
646bbf4881 |
|
|
@ -46,8 +46,7 @@ github:
|
|||
- E2E
|
||||
- Docs
|
||||
- Frontend Build
|
||||
- "Mergeable: milestone-label-check"
|
||||
- "Title Validator"
|
||||
# - "Mergeable: milestone-label-check"
|
||||
required_pull_request_reviews:
|
||||
dismiss_stale_reviews: true
|
||||
required_approving_review_count: 2
|
||||
|
|
|
|||
|
|
@ -16,39 +16,30 @@
|
|||
#
|
||||
|
||||
/.github/ @SbloodyS
|
||||
/deploy/ @SbloodyS
|
||||
/dolphinscheduler-alert/ @SbloodyS
|
||||
/dolphinscheduler-api/ @caishunfeng @SbloodyS
|
||||
/dolphinscheduler-api-test/ @SbloodyS
|
||||
/dolphinscheduler-authentication/ @ruanwenjun
|
||||
/dolphinscheduler-bom/ @ruanwenjun
|
||||
/dolphinscheduler-common/ @SbloodyS
|
||||
/dolphinscheduler-dao/ @SbloodyS @ruanwenjun
|
||||
/dolphinscheduler-dao-plugin/ @SbloodyS @ruanwenjun
|
||||
/dolphinscheduler-data-quality/ @SbloodyS
|
||||
/dolphinscheduler-datasource-plugin/ @SbloodyS
|
||||
/dolphinscheduler-dist/ @SbloodyS
|
||||
/deploy/ @caishunfeng
|
||||
/dolphinscheduler-alert/ @caishunfeng
|
||||
/dolphinscheduler-e2e/ @SbloodyS
|
||||
/dolphinscheduler-extract/ @ruanwenjun
|
||||
/dolphinscheduler-api-test/ @SbloodyS
|
||||
/dolphinscheduler-registry/ @caishunfeng @ruanwenjun
|
||||
/dolphinscheduler-api/ @caishunfeng @SbloodyS
|
||||
/dolphinscheduler-dao/ @caishunfeng @SbloodyS
|
||||
/dolphinscheduler-dao/src/main/resources/sql/ @EricGao888
|
||||
/dolphinscheduler-common/ @caishunfeng
|
||||
/dolphinscheduler-standalone-server/ @caishunfeng
|
||||
/dolphinscheduler-datasource-plugin/ @caishunfeng
|
||||
/dolphinscheduler-dist/ @caishunfeng
|
||||
/dolphinscheduler-meter/ @caishunfeng @ruanwenjun @EricGao888
|
||||
/dolphinscheduler-scheduler-plugin/ @caishunfeng
|
||||
/dolphinscheduler-master/ @caishunfeng @SbloodyS @ruanwenjun
|
||||
/dolphinscheduler-meter/ @ruanwenjun @EricGao888
|
||||
/dolphinscheduler-microbench/ @SbloodyS @ruanwenjun
|
||||
/dolphinscheduler-registry/ @ruanwenjun
|
||||
/dolphinscheduler-scheduler-plugin/ @ruanwenjun
|
||||
/dolphinscheduler-worker/ @caishunfeng @SbloodyS @ruanwenjun
|
||||
/dolphinscheduler-service/ @caishunfeng
|
||||
/dolphinscheduler-extract/ @caishunfeng @ruanwenjun
|
||||
/dolphinscheduler-spi/ @caishunfeng
|
||||
/dolphinscheduler-standalone-server/ @SbloodyS
|
||||
/dolphinscheduler-storage-plugin/ @ruanwenjun @SbloodyS
|
||||
/dolphinscheduler-task-plugin/ @caishunfeng @SbloodyS @zhuangchong
|
||||
/dolphinscheduler-tools/ @caishunfeng @SbloodyS @EricGao888
|
||||
/script/ @caishunfeng @SbloodyS @EricGao888
|
||||
/dolphinscheduler-ui/ @songjianet @Amy0104
|
||||
/dolphinscheduler-worker/ @caishunfeng @SbloodyS @ruanwenjun
|
||||
/dolphinscheduler-yarn-aop/ @ruanwenjun
|
||||
|
||||
/dolphinscheduler-dao/src/main/resources/sql/ @EricGao888
|
||||
|
||||
/script/ @SbloodyS @EricGao888
|
||||
/docs/ @EricGao888 @SbloodyS
|
||||
/docs/ @EricGao888
|
||||
/licenses/ @EricGao888
|
||||
/images/ @EricGao888
|
||||
/style/ @SbloodyS @EricGao888
|
||||
/style/ @caishunfeng
|
||||
|
|
|
|||
|
|
@ -31,7 +31,4 @@ This change added tests and can be verified as follows:
|
|||
|
||||
(or)
|
||||
|
||||
## Pull Request Notice
|
||||
[Pull Request Notice](https://github.com/apache/dolphinscheduler/blob/dev/docs/docs/en/contribute/join/pull-request.md)
|
||||
|
||||
If your pull request contain incompatible change, you should also add it to `docs/docs/en/guide/upgrede/incompatible.md`
|
||||
|
|
|
|||
|
|
@ -1 +0,0 @@
|
|||
Subproject commit 288f36f8cea1dded676b9c4477a64ff91e50c104
|
||||
|
|
@ -18,54 +18,37 @@
|
|||
backend:
|
||||
- 'dolphinscheduler-alert/**/*'
|
||||
- 'dolphinscheduler-api/**/*'
|
||||
- 'dolphinscheduler-authentication/**/*'
|
||||
- 'dolphinscheduler-bom/**/*'
|
||||
- 'dolphinscheduler-common/**/*'
|
||||
- 'dolphinscheduler-dao/**/*'
|
||||
- 'dolphinscheduler-dao-plugin/**/*'
|
||||
- 'dolphinscheduler-data-quality/**/*'
|
||||
- 'dolphinscheduler-datasource-plugin/**/*'
|
||||
- 'dolphinscheduler-dist/**/*'
|
||||
- 'dolphinscheduler-extract/**/*'
|
||||
- 'dolphinscheduler-master/**/*'
|
||||
- 'dolphinscheduler-meter/**/*'
|
||||
- 'dolphinscheduler-microbench/**/*'
|
||||
- 'dolphinscheduler-registry/**/*'
|
||||
- 'dolphinscheduler-extract/**/*'
|
||||
- 'dolphinscheduler-scheduler-plugin/**/*'
|
||||
- 'dolphinscheduler-service/**/*'
|
||||
- 'dolphinscheduler-spi/**/*'
|
||||
- 'dolphinscheduler-standalone-server/**/*'
|
||||
- 'dolphinscheduler-storage-plugin/**/*'
|
||||
- 'dolphinscheduler-task-plugin/**/*'
|
||||
- 'dolphinscheduler-tools/**/*'
|
||||
- 'dolphinscheduler-worker/**/*'
|
||||
- 'dolphinscheduler-yarn-aop/**/*'
|
||||
- 'script/**/*'
|
||||
- '.github/workflows/backend.yml'
|
||||
|
||||
document:
|
||||
- 'docs/**/*'
|
||||
- '**/*.md'
|
||||
|
||||
CI&CD:
|
||||
- any: ['.github/**/*']
|
||||
|
||||
docker:
|
||||
- any: ['deploy/docker/**/*']
|
||||
- any: ['.deploy/**/*']
|
||||
|
||||
UI:
|
||||
- any: ['dolphinscheduler-ui/**/*']
|
||||
- any: ['.github/workflows/frontend.yml']
|
||||
|
||||
e2e:
|
||||
- any: ['dolphinscheduler-e2e/**/*']
|
||||
- any: ['dolphinscheduler-api-test/**/*']
|
||||
- any: ['.github/workflows/e2e.yml']
|
||||
- any: ['.github/workflows/api-test.yml']
|
||||
- any: ['.github/workflows/e2e-k8s.yml']
|
||||
|
||||
test:
|
||||
- any: ['**/test/**/*']
|
||||
|
||||
kubernetes:
|
||||
- any: ['deploy/kubernetes/**/*']
|
||||
- any: ['dolphinscheduler-api-test/**/*']
|
||||
|
|
|
|||
|
|
@ -1,72 +0,0 @@
|
|||
################################################################################
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# Details: https://github.com/apps/boring-cyborg
|
||||
|
||||
###### IssueLink Adder #################################################################################################
|
||||
# Insert Issue (Jira/Github etc) link in PR description based on the Issue ID in PR title.
|
||||
insertIssueLinkInPrDescription:
|
||||
# specify the placeholder for the issue link that should be present in the description
|
||||
descriptionIssuePlaceholderRegexp: "^fix: (.*)$"
|
||||
matchers:
|
||||
# you can have several matches - for different types of issues
|
||||
# only the first matching entry is replaced
|
||||
jiraIssueMatch:
|
||||
# specify the regexp of issue id that you can find in the title of the PR
|
||||
# the match groups can be used to build the issue id (${1}, ${2}, etc.).
|
||||
titleIssueIdRegexp: \[(Feature|Fix|Improvement|Test|E2E|Doc-[0-9]+)\]
|
||||
# the issue link to be added. ${1}, ${2} ... are replaced with the match groups from the
|
||||
# title match (remember to use quotes)
|
||||
descriptionIssueLink: "[${1}](https://github.com/apache/dolphinschedule/${1}/)"
|
||||
|
||||
###### Title Validator #################################################################################################
|
||||
# Verifies if commit/PR titles match the regexp specified
|
||||
verifyTitles:
|
||||
# Regular expression that should be matched by titles of commits or PR
|
||||
titleRegexp: ^\[DSIP-[0-9]+\].*$|^\[Feature-[0-9]+\].*$|^\[Fix-[0-9]+\].*$|^\[Doc-[0-9]+\].*$|^\[Improvement-[0-9]+\].*$|^\[Test-[0-9]+\].*$|^\[E2E-[0-9]+\].*$|^\[CI].*$|^\[Chore].*$
|
||||
# If set to true, it will always check the PR title (as opposed to the individual commits).
|
||||
alwaysUsePrTitle: true
|
||||
# If set to true, it will only check the commit in case there is a single commit.
|
||||
# In case of multiple commits it will check PR title.
|
||||
# This reflects the standard behaviour of Github that for `Squash & Merge` GitHub
|
||||
# uses the PR title rather than commit messages for the squashed commit ¯\_(ツ)_/¯
|
||||
# For single-commit PRs it takes the squashed commit message from the commit as expected.
|
||||
#
|
||||
# If set to false it will check all commit messages. This is useful when you do not squash commits at merge.
|
||||
validateEitherPrOrSingleCommitTitle: true
|
||||
# The title the GitHub status should appear from.
|
||||
statusTitle: "Title Validator"
|
||||
# A custom message to be displayed when the title passes validation.
|
||||
successMessage: "Title Validation successful!"
|
||||
# A custom message to be displayed when the title fails validation.
|
||||
# Allows insertion of ${type} (commit/PR), ${title} (the title validated) and ${regex} (the titleRegexp above).
|
||||
failureMessage: "Wrong ${type} title: ${title}"
|
||||
|
||||
# Various Flags to control behaviour of the "Labeler"
|
||||
labelerFlags:
|
||||
# If this flag is changed to 'false', labels would only be added when the PR is first created
|
||||
# and not when existing PR is updated.
|
||||
# The default is 'true' which means the labels would be added when PR is updated even if they
|
||||
# were removed by the user
|
||||
labelOnPRUpdates: true
|
||||
|
||||
# Comment to be posted to welcome users when they open their first PR
|
||||
firstPRWelcomeComment: >
|
||||
Thanks for opening this pull request! Please check out our contributing guidelines. (https://github.com/apache/dolphinscheduler/blob/dev/docs/docs/en/contribute/join/pull-request.md)
|
||||
|
||||
# Comment to be posted to congratulate user on their first merged PR
|
||||
firstPRMergeComment: >
|
||||
Awesome work, congrats on your first merged pull request!
|
||||
|
|
@ -20,8 +20,6 @@ on:
|
|||
push:
|
||||
branches:
|
||||
- dev
|
||||
- '[0-9]+.[0-9]+.[0-9]+-prepare'
|
||||
- '[0-9]+.[0-9]+.[0-9]+-release'
|
||||
|
||||
name: API-Test
|
||||
|
||||
|
|
@ -37,7 +35,7 @@ jobs:
|
|||
outputs:
|
||||
not-ignore: ${{ steps.filter.outputs.not-ignore }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@v2
|
||||
- uses: dorny/paths-filter@b2feaf19c27470162a626bd6fa8438ae5b263721
|
||||
id: filter
|
||||
with:
|
||||
|
|
@ -51,7 +49,7 @@ jobs:
|
|||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@v2
|
||||
with:
|
||||
submodules: true
|
||||
- name: Sanity Check
|
||||
|
|
@ -75,7 +73,7 @@ jobs:
|
|||
run: |
|
||||
docker save apache/dolphinscheduler-standalone-server:ci -o /tmp/standalone-image.tar \
|
||||
&& du -sh /tmp/standalone-image.tar
|
||||
- uses: actions/upload-artifact@v4
|
||||
- uses: actions/upload-artifact@v2
|
||||
name: Upload Docker Images
|
||||
with:
|
||||
name: standalone-image
|
||||
|
|
@ -106,7 +104,7 @@ jobs:
|
|||
env:
|
||||
RECORDING_PATH: /tmp/recording-${{ matrix.case.name }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@v2
|
||||
with:
|
||||
submodules: true
|
||||
- name: Cache local Maven repository
|
||||
|
|
@ -115,7 +113,7 @@ jobs:
|
|||
path: ~/.m2/repository
|
||||
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-api-test
|
||||
restore-keys: ${{ runner.os }}-maven-
|
||||
- uses: actions/download-artifact@v4
|
||||
- uses: actions/download-artifact@v2
|
||||
name: Download Docker Images
|
||||
with:
|
||||
name: standalone-image
|
||||
|
|
@ -127,9 +125,9 @@ jobs:
|
|||
run: |
|
||||
./mvnw -B -f dolphinscheduler-api-test/pom.xml -am \
|
||||
-DfailIfNoTests=false \
|
||||
-Dspotless.skip=true \
|
||||
-Dspotless.skip=false \
|
||||
-Dtest=${{ matrix.case.class }} test
|
||||
- uses: actions/upload-artifact@v4
|
||||
- uses: actions/upload-artifact@v2
|
||||
if: always()
|
||||
name: Upload Recording
|
||||
with:
|
||||
|
|
|
|||
|
|
@ -44,7 +44,7 @@ jobs:
|
|||
not-ignore: ${{ steps.filter.outputs.not-ignore }}
|
||||
db-schema: ${{ steps.filter.outputs.db-schema }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@v2
|
||||
- uses: dorny/paths-filter@b2feaf19c27470162a626bd6fa8438ae5b263721
|
||||
id: filter
|
||||
with:
|
||||
|
|
@ -63,7 +63,7 @@ jobs:
|
|||
java: [ '8', '11' ]
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@v2
|
||||
with:
|
||||
submodules: true
|
||||
- name: Set up JDK ${{ matrix.java }}
|
||||
|
|
@ -91,7 +91,7 @@ jobs:
|
|||
-Dmaven.wagon.httpconnectionManager.ttlSeconds=120
|
||||
- name: Check dependency license
|
||||
run: tools/dependencies/check-LICENSE.sh
|
||||
- uses: actions/upload-artifact@v4
|
||||
- uses: actions/upload-artifact@v2
|
||||
if: ${{ matrix.java == '8' }}
|
||||
name: Upload Binary Package
|
||||
with:
|
||||
|
|
@ -115,10 +115,10 @@ jobs:
|
|||
- name: cluster-test-postgresql-with-postgresql-registry
|
||||
script: .github/workflows/cluster-test/postgresql_with_postgresql_registry/start-job.sh
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@v2
|
||||
with:
|
||||
submodules: true
|
||||
- uses: actions/download-artifact@v4
|
||||
- uses: actions/download-artifact@v2
|
||||
name: Download Binary Package
|
||||
with:
|
||||
# Only run cluster test on jdk8
|
||||
|
|
@ -165,7 +165,7 @@ jobs:
|
|||
mkdir -p dolphinscheduler/dev dolphinscheduler/${{ matrix.version }}
|
||||
curl -sSf https://atlasgo.sh | sh
|
||||
- name: Download Tarball
|
||||
uses: actions/download-artifact@v4
|
||||
uses: actions/download-artifact@v2
|
||||
with:
|
||||
name: binary-package-8
|
||||
path: dolphinscheduler/dev
|
||||
|
|
|
|||
|
|
@ -23,8 +23,6 @@ on:
|
|||
pull_request:
|
||||
branches:
|
||||
- 'dev'
|
||||
- '[0-9]+.[0-9]+.[0-9]+-prepare'
|
||||
- '[0-9]+.[0-9]+.[0-9]+-release'
|
||||
|
||||
concurrency:
|
||||
group: codeql-${{ github.event.pull_request.number || github.ref }}
|
||||
|
|
|
|||
|
|
@ -30,7 +30,7 @@ jobs:
|
|||
timeout-minutes: 10
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@v2
|
||||
- name: Style Check
|
||||
run: ./mvnw spotless:check
|
||||
img-check:
|
||||
|
|
@ -40,7 +40,7 @@ jobs:
|
|||
run:
|
||||
working-directory: docs
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@v2
|
||||
- name: Set up Python 3.9
|
||||
uses: actions/setup-python@v2
|
||||
with:
|
||||
|
|
@ -54,7 +54,7 @@ jobs:
|
|||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@v2
|
||||
- run: sudo npm install -g markdown-link-check@3.11.2
|
||||
- run: sudo apt install plocate -y
|
||||
# NOTE: Change command from `find . -name "*.md"` to `find . -not -path "*/node_modules/*" -not -path "*/.tox/*" -name "*.md"`
|
||||
|
|
@ -70,7 +70,7 @@ jobs:
|
|||
outputs:
|
||||
helm-doc: ${{ steps.filter.outputs.helm-doc }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@v2
|
||||
- uses: dorny/paths-filter@b2feaf19c27470162a626bd6fa8438ae5b263721
|
||||
id: filter
|
||||
with:
|
||||
|
|
@ -84,7 +84,7 @@ jobs:
|
|||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@v2
|
||||
with:
|
||||
submodules: true
|
||||
- name: Generating helm-doc
|
||||
|
|
|
|||
|
|
@ -20,8 +20,6 @@ on:
|
|||
push:
|
||||
branches:
|
||||
- dev
|
||||
- '[0-9]+.[0-9]+.[0-9]+-prepare'
|
||||
- '[0-9]+.[0-9]+.[0-9]+-release'
|
||||
|
||||
name: E2E-K8S
|
||||
|
||||
|
|
@ -37,7 +35,7 @@ jobs:
|
|||
outputs:
|
||||
not-ignore: ${{ steps.filter.outputs.not-ignore }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@v2
|
||||
- uses: dorny/paths-filter@b2feaf19c27470162a626bd6fa8438ae5b263721
|
||||
id: filter
|
||||
with:
|
||||
|
|
@ -51,7 +49,7 @@ jobs:
|
|||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@v2
|
||||
with:
|
||||
submodules: true
|
||||
- name: Build Image
|
||||
|
|
|
|||
|
|
@ -20,8 +20,6 @@ on:
|
|||
push:
|
||||
branches:
|
||||
- dev
|
||||
- '[0-9]+.[0-9]+.[0-9]+-prepare'
|
||||
- '[0-9]+.[0-9]+.[0-9]+-release'
|
||||
|
||||
name: E2E
|
||||
|
||||
|
|
@ -37,7 +35,7 @@ jobs:
|
|||
outputs:
|
||||
not-ignore: ${{ steps.filter.outputs.not-ignore }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@v2
|
||||
- uses: dorny/paths-filter@b2feaf19c27470162a626bd6fa8438ae5b263721
|
||||
id: filter
|
||||
with:
|
||||
|
|
@ -51,7 +49,7 @@ jobs:
|
|||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@v2
|
||||
with:
|
||||
submodules: true
|
||||
- name: Sanity Check
|
||||
|
|
@ -76,7 +74,7 @@ jobs:
|
|||
run: |
|
||||
docker save apache/dolphinscheduler-standalone-server:ci -o /tmp/standalone-image.tar \
|
||||
&& du -sh /tmp/standalone-image.tar
|
||||
- uses: actions/upload-artifact@v4
|
||||
- uses: actions/upload-artifact@v2
|
||||
name: Upload Docker Images
|
||||
with:
|
||||
name: standalone-image
|
||||
|
|
@ -116,18 +114,20 @@ jobs:
|
|||
# class: org.apache.dolphinscheduler.e2e.cases.WorkflowSwitchE2ETest
|
||||
- name: FileManage
|
||||
class: org.apache.dolphinscheduler.e2e.cases.FileManageE2ETest
|
||||
- name: UdfManage
|
||||
class: org.apache.dolphinscheduler.e2e.cases.UdfManageE2ETest
|
||||
- name: FunctionManage
|
||||
class: org.apache.dolphinscheduler.e2e.cases.FunctionManageE2ETest
|
||||
- name: MysqlDataSource
|
||||
class: org.apache.dolphinscheduler.e2e.cases.MysqlDataSourceE2ETest
|
||||
- name: ClickhouseDataSource
|
||||
class: org.apache.dolphinscheduler.e2e.cases.ClickhouseDataSourceE2ETest
|
||||
- name: PostgresDataSource
|
||||
class: org.apache.dolphinscheduler.e2e.cases.PostgresDataSourceE2ETest
|
||||
- name: ShellTaskE2ETest
|
||||
class: org.apache.dolphinscheduler.e2e.cases.tasks.ShellTaskE2ETest
|
||||
env:
|
||||
RECORDING_PATH: /tmp/recording-${{ matrix.case.name }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@v2
|
||||
with:
|
||||
submodules: true
|
||||
- name: Cache local Maven repository
|
||||
|
|
@ -136,7 +136,7 @@ jobs:
|
|||
path: ~/.m2/repository
|
||||
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-e2e
|
||||
restore-keys: ${{ runner.os }}-maven-
|
||||
- uses: actions/download-artifact@v4
|
||||
- uses: actions/download-artifact@v2
|
||||
name: Download Docker Images
|
||||
with:
|
||||
name: standalone-image
|
||||
|
|
@ -149,7 +149,7 @@ jobs:
|
|||
./mvnw -B -f dolphinscheduler-e2e/pom.xml -am \
|
||||
-DfailIfNoTests=false \
|
||||
-Dtest=${{ matrix.case.class }} test
|
||||
- uses: actions/upload-artifact@v4
|
||||
- uses: actions/upload-artifact@v2
|
||||
if: always()
|
||||
name: Upload Recording
|
||||
with:
|
||||
|
|
@ -171,7 +171,7 @@ jobs:
|
|||
env:
|
||||
RECORDING_PATH: /tmp/recording-${{ matrix.case.name }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@v2
|
||||
with:
|
||||
submodules: true
|
||||
- name: Cache local Maven repository
|
||||
|
|
@ -180,7 +180,7 @@ jobs:
|
|||
path: ~/.m2/repository
|
||||
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-e2e
|
||||
restore-keys: ${{ runner.os }}-maven-
|
||||
- uses: actions/download-artifact@v4
|
||||
- uses: actions/download-artifact@v2
|
||||
name: Download Docker Images
|
||||
with:
|
||||
name: standalone-image
|
||||
|
|
@ -193,7 +193,7 @@ jobs:
|
|||
./mvnw -B -f dolphinscheduler-e2e/pom.xml -am \
|
||||
-DfailIfNoTests=false \
|
||||
-Dtest=${{ matrix.case.class }} test
|
||||
- uses: actions/upload-artifact@v4
|
||||
- uses: actions/upload-artifact@v2
|
||||
if: always()
|
||||
name: Upload Recording
|
||||
with:
|
||||
|
|
|
|||
|
|
@ -20,8 +20,7 @@ name: Frontend
|
|||
on:
|
||||
push:
|
||||
branches:
|
||||
- '[0-9]+.[0-9]+.[0-9]+-prepare'
|
||||
- '[0-9]+.[0-9]+.[0-9]+-release'
|
||||
- dev
|
||||
paths:
|
||||
- '.github/workflows/frontend.yml'
|
||||
- 'dolphinscheduler-ui/**'
|
||||
|
|
@ -42,7 +41,7 @@ jobs:
|
|||
outputs:
|
||||
not-ignore: ${{ steps.filter.outputs.not-ignore }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@v2
|
||||
- uses: dorny/paths-filter@b2feaf19c27470162a626bd6fa8438ae5b263721
|
||||
id: filter
|
||||
with:
|
||||
|
|
@ -59,7 +58,7 @@ jobs:
|
|||
matrix:
|
||||
os: [ ubuntu-latest, macos-latest ]
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@v2
|
||||
with:
|
||||
submodules: true
|
||||
- if: matrix.os == 'ubuntu-latest'
|
||||
|
|
@ -83,7 +82,7 @@ jobs:
|
|||
needs: [ build, paths-filter ]
|
||||
if: always()
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@v2
|
||||
- name: Status
|
||||
run: |
|
||||
if [[ ${{ needs.paths-filter.outputs.not-ignore }} == 'false' && ${{ github.event_name }} == 'pull_request' ]]; then
|
||||
|
|
|
|||
|
|
@ -26,7 +26,7 @@ jobs:
|
|||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: "Checkout ${{ github.ref }}"
|
||||
uses: actions/checkout@v4
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
persist-credentials: false
|
||||
submodules: true
|
||||
|
|
|
|||
|
|
@ -14,50 +14,65 @@
|
|||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
---
|
||||
#version: 2
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
types:
|
||||
- opened
|
||||
- reopened
|
||||
- synchronize
|
||||
- labeled
|
||||
- unlabeled
|
||||
- milestoned
|
||||
- demilestoned
|
||||
|
||||
|
||||
name: "Mergeable"
|
||||
|
||||
|
||||
jobs:
|
||||
result:
|
||||
name: "Mergeable: milestone-label-check"
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- name: Check milestone and labels
|
||||
uses: actions/github-script@v7
|
||||
with:
|
||||
script: |
|
||||
const { data } = await github.request("GET /repos/{owner}/{repo}/pulls/{pr}", {
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
pr: context.payload.pull_request.number
|
||||
});
|
||||
if (data.milestone) {
|
||||
core.info(`This pull request has a milestone: ${data.milestone.title}`);
|
||||
} else {
|
||||
core.setFailed(`A maintainer needs to set the milestone for this pull request.`);
|
||||
}
|
||||
let labeledFlag = false;
|
||||
for (const label of data.labels) {
|
||||
if (['feature', 'bug', 'improvement', 'document', 'chore', 'DSIP', 'CI&CD', 'revert'].includes(label.name)) {
|
||||
core.info(`This pull request has a valid label: ${label.name}`);
|
||||
labeledFlag = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!labeledFlag) {
|
||||
core.setFailed(`A maintainer needs to set a valid label for this pull request.`);
|
||||
}
|
||||
- name: Status
|
||||
run: |
|
||||
echo "Temporary skipping this check"
|
||||
|
||||
#mergeable:
|
||||
# # we can not use `pull_request.*` which including event `pull_request.labeled`, according to https://github.com/mergeability/mergeable/issues/643,
|
||||
# # otherwise mergeable will keep add or remove label endless, we just need this CI act like the default behavior as
|
||||
# # GitHub action workflow `pull_requests` https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request like,
|
||||
# # which only trigger runs when a pull_request event's activity type is opened, synchronize, or reopened
|
||||
# - when: pull_request.opened, pull_request.reopened, pull_request.synchronize
|
||||
# name: sync-sql-ddl
|
||||
# validate:
|
||||
# # Sql files must change synchronize
|
||||
# - do: dependent
|
||||
# files:
|
||||
# - 'dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_h2.sql'
|
||||
# - 'dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql'
|
||||
# - 'dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql'
|
||||
# message: 'Sql files not change synchronize'
|
||||
# # Add labels 'sql not sync' and comment to reviewers if Sql files not change synchronize
|
||||
# fail:
|
||||
# - do: comment
|
||||
# payload:
|
||||
# body: >
|
||||
# :warning: This PR do not change database DDL synchronize.
|
||||
# leave_old_comment: false
|
||||
# - do: labels
|
||||
# add: 'sql not sync'
|
||||
# # Remove labels 'sql not sync' if pass
|
||||
# pass:
|
||||
# - do: labels
|
||||
# delete: 'sql not sync'
|
||||
#
|
||||
# - when: pull_request.*
|
||||
# name: milestone-label-check
|
||||
# validate:
|
||||
# - do: milestone
|
||||
# no_empty:
|
||||
# enabled: false # Cannot be empty when true.
|
||||
# message: 'Milestone is required and cannot be empty.'
|
||||
# - do: label
|
||||
# and:
|
||||
# - must_include:
|
||||
# regex: 'feature|bug|improvement|document|chore|revert'
|
||||
# message: 'Label must include one of the following: `feature`, `bug`, `improvement`, `document`, `chore`, `revert`'
|
||||
# - must_include:
|
||||
# regex: 'ready-to-merge'
|
||||
# message: 'Please check if there are PRs that already have a `ready-to-merge` label and can be merged, if exists please merge them first.'
|
||||
|
|
|
|||
|
|
@ -20,8 +20,7 @@ name: OWASP Dependency Check
|
|||
on:
|
||||
push:
|
||||
branches:
|
||||
- '[0-9]+.[0-9]+.[0-9]+-prepare'
|
||||
- '[0-9]+.[0-9]+.[0-9]+-release'
|
||||
- dev
|
||||
pull_request:
|
||||
paths:
|
||||
- '**/pom.xml'
|
||||
|
|
@ -31,9 +30,8 @@ env:
|
|||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 120
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@v2
|
||||
with:
|
||||
submodules: true
|
||||
- name: Set up JDK 8
|
||||
|
|
@ -44,9 +42,9 @@ jobs:
|
|||
- name: Run OWASP Dependency Check
|
||||
run: ./mvnw -B clean install verify dependency-check:check -DskipDepCheck=false -Dmaven.test.skip=true -Dspotless.skip=true
|
||||
- name: Upload report
|
||||
uses: actions/upload-artifact@v4
|
||||
uses: actions/upload-artifact@v3
|
||||
if: ${{ cancelled() || failure() }}
|
||||
continue-on-error: true
|
||||
with:
|
||||
name: dependency report
|
||||
path: target/dependency-check-report.html
|
||||
path: target/dependency-check-report.html
|
||||
|
|
|
|||
|
|
@ -33,7 +33,7 @@ jobs:
|
|||
packages: write
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@v2
|
||||
- name: Cache local Maven repository
|
||||
uses: actions/cache@v3
|
||||
with:
|
||||
|
|
|
|||
|
|
@ -33,7 +33,7 @@ jobs:
|
|||
packages: write
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@v2
|
||||
- name: Set environment variables
|
||||
run: |
|
||||
if [[ ${{ github.event_name }} == "release" ]]; then
|
||||
|
|
|
|||
|
|
@ -15,21 +15,20 @@
|
|||
# limitations under the License.
|
||||
#
|
||||
|
||||
name: "pull-request-target-robot"
|
||||
name: "pull-request-robot"
|
||||
|
||||
on:
|
||||
pull_request_target:
|
||||
|
||||
jobs:
|
||||
pullRequestTargetRobot:
|
||||
labelRobot:
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- name: "Checkout ${{ github.ref }}"
|
||||
uses: actions/checkout@v4
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
persist-credentials: false
|
||||
submodules: true
|
||||
|
|
@ -40,8 +39,3 @@ jobs:
|
|||
repo-token: "${{ secrets.GITHUB_TOKEN }}"
|
||||
configuration-path: .github/actions/labeler/labeler.yml
|
||||
sync-labels: true
|
||||
|
||||
- name: "Auto assign in pull request"
|
||||
uses: ./.github/actions/auto-assign-action
|
||||
with:
|
||||
configuration-path: .github/actions/auto-assign/auto-assign.yml
|
||||
|
|
@ -24,8 +24,7 @@ on:
|
|||
- '**/*.md'
|
||||
- 'dolphinscheduler-ui'
|
||||
branches:
|
||||
- '[0-9]+.[0-9]+.[0-9]+-prepare'
|
||||
- '[0-9]+.[0-9]+.[0-9]+-release'
|
||||
- dev
|
||||
|
||||
env:
|
||||
LOG_DIR: /tmp/dolphinscheduler
|
||||
|
|
@ -41,7 +40,7 @@ jobs:
|
|||
outputs:
|
||||
not-ignore: ${{ steps.filter.outputs.not-ignore }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@v2
|
||||
- uses: dorny/paths-filter@b2feaf19c27470162a626bd6fa8438ae5b263721
|
||||
id: filter
|
||||
with:
|
||||
|
|
@ -58,7 +57,7 @@ jobs:
|
|||
java: ['8', '11']
|
||||
timeout-minutes: 45
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@v2
|
||||
with:
|
||||
submodules: true
|
||||
- name: Sanity Check
|
||||
|
|
|
|||
|
|
@ -7,7 +7,6 @@
|
|||
.target
|
||||
.idea/
|
||||
!.idea/icon.png
|
||||
!.idea/vcs.xml
|
||||
.run/
|
||||
target/
|
||||
dist/
|
||||
|
|
|
|||
|
|
@ -21,7 +21,3 @@
|
|||
[submodule ".github/actions/translate-on-issue"]
|
||||
path = .github/actions/translate-on-issue
|
||||
url = https://github.com/xingchun-chen/translation-helper
|
||||
[submodule ".github/actions/auto-assign-action"]
|
||||
path = .github/actions/auto-assign-action
|
||||
url = https://github.com/kentaro-m/auto-assign-action.git
|
||||
branch = 288f36f
|
||||
|
|
|
|||
|
|
@ -1,16 +0,0 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project version="4">
|
||||
<component name="IssueNavigationConfiguration">
|
||||
<option name="links">
|
||||
<list>
|
||||
<IssueNavigationLink>
|
||||
<option name="issueRegexp" value="#(\d+)" />
|
||||
<option name="linkRegexp" value="https://github.com/apache/dolphinscheduler/issues/$1" />
|
||||
</IssueNavigationLink>
|
||||
</list>
|
||||
</option>
|
||||
</component>
|
||||
<component name="VcsDirectoryMappings">
|
||||
<mapping directory="" vcs="Git" />
|
||||
</component>
|
||||
</project>
|
||||
|
|
@ -51,6 +51,5 @@ header:
|
|||
- '**/banner.txt'
|
||||
- '.terraform.lock.hcl'
|
||||
- deploy/kubernetes/dolphinscheduler/README.md.gotmpl
|
||||
- .idea/vcs.xml
|
||||
|
||||
comment: on-failure
|
||||
|
|
|
|||
|
|
@ -124,12 +124,6 @@ Please refer to the [Quick Start in Kubernetes](../../../docs/docs/en/guide/inst
|
|||
| conf.auto | bool | `false` | auto restart, if true, all components will be restarted automatically after the common configuration is updated. if false, you need to restart the components manually. default is false |
|
||||
| conf.common."alert.rpc.port" | int | `50052` | rpc port |
|
||||
| conf.common."appId.collect" | string | `"log"` | way to collect applicationId: log, aop |
|
||||
| conf.common."aws.credentials.provider.type" | string | `"AWSStaticCredentialsProvider"` | |
|
||||
| conf.common."aws.s3.access.key.id" | string | `"minioadmin"` | The AWS access key. if resource.storage.type=S3, and credentials.provider.type is AWSStaticCredentialsProvider. This configuration is required |
|
||||
| conf.common."aws.s3.access.key.secret" | string | `"minioadmin"` | The AWS secret access key. if resource.storage.type=S3, and credentials.provider.type is AWSStaticCredentialsProvider. This configuration is required |
|
||||
| conf.common."aws.s3.bucket.name" | string | `"dolphinscheduler"` | The name of the bucket. You need to create them by yourself. Otherwise, the system cannot start. All buckets in Amazon S3 share a single namespace; ensure the bucket is given a unique name. |
|
||||
| conf.common."aws.s3.endpoint" | string | `"http://minio:9000"` | You need to set this parameter when private cloud s3. If S3 uses public cloud, you only need to set resource.aws.region or set to the endpoint of a public cloud such as S3.cn-north-1.amazonaws.com.cn |
|
||||
| conf.common."aws.s3.region" | string | `"ca-central-1"` | The AWS Region to use. if resource.storage.type=S3, This configuration is required |
|
||||
| conf.common."conda.path" | string | `"/opt/anaconda3/etc/profile.d/conda.sh"` | set path of conda.sh |
|
||||
| conf.common."data-quality.jar.dir" | string | `nil` | data quality option |
|
||||
| conf.common."data.basedir.path" | string | `"/tmp/dolphinscheduler"` | user data local directory path, please make sure the directory exists and have read write permissions |
|
||||
|
|
@ -148,6 +142,11 @@ Please refer to the [Quick Start in Kubernetes](../../../docs/docs/en/guide/inst
|
|||
| conf.common."resource.alibaba.cloud.oss.bucket.name" | string | `"dolphinscheduler"` | oss bucket name, required if you set resource.storage.type=OSS |
|
||||
| conf.common."resource.alibaba.cloud.oss.endpoint" | string | `"https://oss-cn-hangzhou.aliyuncs.com"` | oss bucket endpoint, required if you set resource.storage.type=OSS |
|
||||
| conf.common."resource.alibaba.cloud.region" | string | `"cn-hangzhou"` | alibaba cloud region, required if you set resource.storage.type=OSS |
|
||||
| conf.common."resource.aws.access.key.id" | string | `"minioadmin"` | The AWS access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required |
|
||||
| conf.common."resource.aws.region" | string | `"ca-central-1"` | The AWS Region to use. if resource.storage.type=S3 or use EMR-Task, This configuration is required |
|
||||
| conf.common."resource.aws.s3.bucket.name" | string | `"dolphinscheduler"` | The name of the bucket. You need to create them by yourself. Otherwise, the system cannot start. All buckets in Amazon S3 share a single namespace; ensure the bucket is given a unique name. |
|
||||
| conf.common."resource.aws.s3.endpoint" | string | `"http://minio:9000"` | You need to set this parameter when private cloud s3. If S3 uses public cloud, you only need to set resource.aws.region or set to the endpoint of a public cloud such as S3.cn-north-1.amazonaws.com.cn |
|
||||
| conf.common."resource.aws.secret.access.key" | string | `"minioadmin"` | The AWS secret access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required |
|
||||
| conf.common."resource.azure.client.id" | string | `"minioadmin"` | azure storage account name, required if you set resource.storage.type=ABS |
|
||||
| conf.common."resource.azure.client.secret" | string | `"minioadmin"` | azure storage account key, required if you set resource.storage.type=ABS |
|
||||
| conf.common."resource.azure.subId" | string | `"minioadmin"` | azure storage subId, required if you set resource.storage.type=ABS |
|
||||
|
|
@ -316,6 +315,7 @@ Please refer to the [Quick Start in Kubernetes](../../../docs/docs/en/guide/inst
|
|||
| worker.env.WORKER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_CPU_USAGE_PERCENTAGE_THRESHOLDS | float | `0.7` | Worker max system cpu usage, when the worker's system cpu usage is smaller then this value, worker server can be dispatched tasks. |
|
||||
| worker.env.WORKER_SERVER_LOAD_PROTECTION_MAX_SYSTEM_MEMORY_USAGE_PERCENTAGE_THRESHOLDS | float | `0.7` | Worker max memory usage , when the worker's memory usage is smaller then this value, worker server can be dispatched tasks. |
|
||||
| worker.env.WORKER_TENANT_CONFIG_AUTO_CREATE_TENANT_ENABLED | bool | `true` | tenant corresponds to the user of the system, which is used by the worker to submit the job. If system does not have this user, it will be automatically created after the parameter worker.tenant.auto.create is true. |
|
||||
| worker.env.WORKER_TENANT_CONFIG_DISTRIBUTED_TENANT | bool | `false` | Scenes to be used for distributed users. For example, users created by FreeIpa are stored in LDAP. This parameter only applies to Linux, When this parameter is true, worker.tenant.auto.create has no effect and will not automatically create tenants. |
|
||||
| worker.keda.advanced | object | `{}` | Specify HPA related options |
|
||||
| worker.keda.cooldownPeriod | int | `30` | How many seconds KEDA will wait before scaling to zero. Note that HPA has a separate cooldown period for scale-downs |
|
||||
| worker.keda.enabled | bool | `false` | Enable or disable the Keda component |
|
||||
|
|
|
|||
|
|
@ -37,7 +37,7 @@ data:
|
|||
{{- range $key, $value := index .Values.conf "common" }}
|
||||
{{- if and $.Values.minio.enabled }}
|
||||
{{- if eq $key "resource.storage.type" }}{{ $value = "S3" }}{{- end }}
|
||||
{{- if eq $key "aws.s3.endpoint" }}{{ $value = print "http://" (include "dolphinscheduler.minio.fullname" $) ":9000" }}{{- end }}
|
||||
{{- if eq $key "resource.aws.s3.endpoint" }}{{ $value = print "http://" (include "dolphinscheduler.minio.fullname" $) ":9000" }}{{- end }}
|
||||
{{- end }}
|
||||
{{ $key }}={{ $value }}
|
||||
{{- end }}
|
||||
|
|
|
|||
|
|
@ -250,25 +250,20 @@ conf:
|
|||
# -- resource store on HDFS/S3 path, resource file will store to this base path, self configuration, please make sure the directory exists on hdfs and have read write permissions. "/dolphinscheduler" is recommended
|
||||
resource.storage.upload.base.path: /dolphinscheduler
|
||||
|
||||
# The AWS credentials provider type. support: AWSStaticCredentialsProvider, InstanceProfileCredentialsProvider
|
||||
# AWSStaticCredentialsProvider: use the access key and secret key to authenticate
|
||||
# InstanceProfileCredentialsProvider: use the IAM role to authenticate
|
||||
aws.credentials.provider.type: AWSStaticCredentialsProvider
|
||||
# -- The AWS access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required
|
||||
resource.aws.access.key.id: minioadmin
|
||||
|
||||
# -- The AWS access key. if resource.storage.type=S3, and credentials.provider.type is AWSStaticCredentialsProvider. This configuration is required
|
||||
aws.s3.access.key.id: minioadmin
|
||||
# -- The AWS secret access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required
|
||||
resource.aws.secret.access.key: minioadmin
|
||||
|
||||
# -- The AWS secret access key. if resource.storage.type=S3, and credentials.provider.type is AWSStaticCredentialsProvider. This configuration is required
|
||||
aws.s3.access.key.secret: minioadmin
|
||||
|
||||
# -- The AWS Region to use. if resource.storage.type=S3, This configuration is required
|
||||
aws.s3.region: ca-central-1
|
||||
# -- The AWS Region to use. if resource.storage.type=S3 or use EMR-Task, This configuration is required
|
||||
resource.aws.region: ca-central-1
|
||||
|
||||
# -- The name of the bucket. You need to create them by yourself. Otherwise, the system cannot start. All buckets in Amazon S3 share a single namespace; ensure the bucket is given a unique name.
|
||||
aws.s3.bucket.name: dolphinscheduler
|
||||
resource.aws.s3.bucket.name: dolphinscheduler
|
||||
|
||||
# -- You need to set this parameter when private cloud s3. If S3 uses public cloud, you only need to set resource.aws.region or set to the endpoint of a public cloud such as S3.cn-north-1.amazonaws.com.cn
|
||||
aws.s3.endpoint: http://minio:9000
|
||||
resource.aws.s3.endpoint: http://minio:9000
|
||||
|
||||
# -- alibaba cloud access key id, required if you set resource.storage.type=OSS
|
||||
resource.alibaba.cloud.access.key.id: <your-access-key-id>
|
||||
|
|
@ -677,6 +672,8 @@ worker:
|
|||
WORKER_HOST_WEIGHT: "100"
|
||||
# -- tenant corresponds to the user of the system, which is used by the worker to submit the job. If system does not have this user, it will be automatically created after the parameter worker.tenant.auto.create is true.
|
||||
WORKER_TENANT_CONFIG_AUTO_CREATE_TENANT_ENABLED: true
|
||||
# -- Scenes to be used for distributed users. For example, users created by FreeIpa are stored in LDAP. This parameter only applies to Linux, When this parameter is true, worker.tenant.auto.create has no effect and will not automatically create tenants.
|
||||
WORKER_TENANT_CONFIG_DISTRIBUTED_TENANT: false
|
||||
# -- If set true, will use worker bootstrap user as the tenant to execute task when the tenant is `default`;
|
||||
DEFAULT_TENANT_ENABLED: false
|
||||
|
||||
|
|
|
|||
|
|
@ -145,6 +145,10 @@ export default {
|
|||
title: 'Sqoop',
|
||||
link: '/en-us/docs/dev/user_doc/guide/task/sqoop.html',
|
||||
},
|
||||
{
|
||||
title: 'Pigeon',
|
||||
link: '/en-us/docs/dev/user_doc/guide/task/pigeon.html',
|
||||
},
|
||||
{
|
||||
title: 'Conditions',
|
||||
link: '/en-us/docs/dev/user_doc/guide/task/conditions.html',
|
||||
|
|
@ -428,6 +432,10 @@ export default {
|
|||
title: 'File Manage',
|
||||
link: '/en-us/docs/dev/user_doc/guide/resource/file-manage.html'
|
||||
},
|
||||
{
|
||||
title: 'UDF Manage',
|
||||
link: '/en-us/docs/dev/user_doc/guide/resource/udf-manage.html'
|
||||
},
|
||||
{
|
||||
title: 'Task Group Manage',
|
||||
link: '/en-us/docs/dev/user_doc/guide/resource/task-group.html'
|
||||
|
|
@ -873,6 +881,10 @@ export default {
|
|||
title: 'Sqoop',
|
||||
link: '/zh-cn/docs/dev/user_doc/guide/task/sqoop.html',
|
||||
},
|
||||
{
|
||||
title: 'Pigeon',
|
||||
link: '/zh-cn/docs/dev/user_doc/guide/task/pigeon.html',
|
||||
},
|
||||
{
|
||||
title: 'Conditions',
|
||||
link: '/zh-cn/docs/dev/user_doc/guide/task/conditions.html',
|
||||
|
|
@ -1141,6 +1153,10 @@ export default {
|
|||
title: '文件管理',
|
||||
link: '/zh-cn/docs/dev/user_doc/guide/resource/file-manage.html'
|
||||
},
|
||||
{
|
||||
title: 'UDF 管理',
|
||||
link: '/zh-cn/docs/dev/user_doc/guide/resource/udf-manage.html'
|
||||
},
|
||||
{
|
||||
title: '任务组管理',
|
||||
link: '/zh-cn/docs/dev/user_doc/guide/resource/task-group.html'
|
||||
|
|
|
|||
|
|
@ -191,21 +191,19 @@ The default configuration is as follows:
|
|||
|
||||
Note that DolphinScheduler also supports zookeeper related configuration through `bin/env/dolphinscheduler_env.sh`.
|
||||
|
||||
For ETCD Registry, please see more details
|
||||
on [link](https://github.com/apache/dolphinscheduler/blob/dev/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/README.md).
|
||||
For JDBC Registry, please see more details
|
||||
on [link](https://github.com/apache/dolphinscheduler/blob/dev/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/README.md).
|
||||
For ETCD Registry, please see more details on [link](https://github.com/apache/dolphinscheduler/blob/dev/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/README.md).
|
||||
For JDBC Registry, please see more details on [link](https://github.com/apache/dolphinscheduler/blob/dev/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/README.md).
|
||||
|
||||
### common.properties [hadoop、s3、yarn config properties]
|
||||
|
||||
Currently, common.properties mainly configures Hadoop,s3a related configurations. Configuration file location:
|
||||
|
||||
| Service | Configuration file |
|
||||
|---------------|-----------------------------------------------------------------------|
|
||||
| Master Server | `master-server/conf/common.properties` |
|
||||
| Api Server | `api-server/conf/common.properties`, `api-server/conf/aws.yaml` |
|
||||
| Worker Server | `worker-server/conf/common.properties`, `worker-server/conf/aws.yaml` |
|
||||
| Alert Server | `alert-server/conf/common.properties` |
|
||||
| Service | Configuration file |
|
||||
|---------------|----------------------------------------|
|
||||
| Master Server | `master-server/conf/common.properties` |
|
||||
| Api Server | `api-server/conf/common.properties` |
|
||||
| Worker Server | `worker-server/conf/common.properties` |
|
||||
| Alert Server | `alert-server/conf/common.properties` |
|
||||
|
||||
The default configuration is as follows:
|
||||
|
||||
|
|
@ -214,6 +212,10 @@ The default configuration is as follows:
|
|||
| data.basedir.path | /tmp/dolphinscheduler | local directory used to store temp files |
|
||||
| resource.storage.type | NONE | type of resource files: HDFS, S3, OSS, GCS, ABS, NONE |
|
||||
| resource.upload.path | /dolphinscheduler | storage path of resource files |
|
||||
| aws.access.key.id | minioadmin | access key id of S3 |
|
||||
| aws.secret.access.key | minioadmin | secret access key of S3 |
|
||||
| aws.region | us-east-1 | region of S3 |
|
||||
| aws.s3.endpoint | http://minio:9000 | endpoint of S3 |
|
||||
| hdfs.root.user | hdfs | configure users with corresponding permissions if storage type is HDFS |
|
||||
| fs.defaultFS | hdfs://mycluster:8020 | If resource.storage.type=S3, then the request url would be similar to 's3a://dolphinscheduler'. Otherwise if resource.storage.type=HDFS and hadoop supports HA, copy core-site.xml and hdfs-site.xml into 'conf' directory |
|
||||
| hadoop.security.authentication.startup.state | false | whether hadoop grant kerberos permission |
|
||||
|
|
@ -328,6 +330,7 @@ Location: `worker-server/conf/application.yaml`
|
|||
| worker.registry-disconnect-strategy.max-waiting-time | 100s | Used when the worker disconnect from registry, and the disconnect strategy is waiting, this config means the worker will waiting to reconnect to registry in given times, and after the waiting times, if the worker still cannot connect to registry, will stop itself, if the value is 0s, will wait infinitely |
|
||||
| worker.task-execute-threads-full-policy | REJECT | If REJECT, when the task waiting in the worker reaches exec-threads, it will reject the received task and the Master will redispatch it; If CONTINUE, it will put the task into the worker's execution queue and wait for a free thread to start execution |
|
||||
| worker.tenant-config.auto-create-tenant-enabled | true | tenant corresponds to the user of the system, which is used by the worker to submit the job. If system does not have this user, it will be automatically created after the parameter worker.tenant.auto.create is true. |
|
||||
| worker.tenant-config.distributed-tenant-enabled | false | When this parameter is true, auto-create-tenant-enabled has no effect and will not automatically create tenants |
|
||||
| worker.tenant-config.default-tenant-enabled | false | If set true, will use worker bootstrap user as the tenant to execute task when the tenant is `default`. |
|
||||
|
||||
### Alert Server related configuration
|
||||
|
|
|
|||
|
|
@ -22,6 +22,8 @@ see sql files in `dolphinscheduler/dolphinscheduler-dao/src/main/resources/sql`
|
|||
|
||||
- User can have multiple projects, user project authorization completes the relationship binding using `project_id` and `user_id` in `t_ds_relation_project_user` table.
|
||||
- The `user_id` in the `t_ds_projcet` table represents the user who create the project, and the `user_id` in the `t_ds_relation_project_user` table represents users who have permission to the project.
|
||||
- The `user_id` in the `t_ds_resources` table represents the user who create the resource, and the `user_id` in `t_ds_relation_resources_user` represents the user who has permissions to the resource.
|
||||
- The `user_id` in the `t_ds_udfs` table represents the user who create the UDF, and the `user_id` in the `t_ds_relation_udfs_user` table represents a user who has permission to the UDF.
|
||||
|
||||
### Project - Tenant - ProcessDefinition - Schedule
|
||||
|
||||
|
|
|
|||
|
|
@ -146,6 +146,7 @@ No.|parameter name||type|description |note
|
|||
5| |type |String |database type
|
||||
6| |datasource |Int |datasource id
|
||||
7| |sql |String |query SQL statement
|
||||
8| |udfs | String| udf functions|specify UDF function ids, separate by comma
|
||||
9| |sqlType | String| SQL node type |0 for query and 1 for none-query SQL
|
||||
10| |title |String | mail title
|
||||
11| |receivers |String |receivers
|
||||
|
|
@ -179,6 +180,7 @@ No.|parameter name||type|description |note
|
|||
"type":"MYSQL",
|
||||
"datasource":1,
|
||||
"sql":"select id , namge , age from emp where id = ${id}",
|
||||
"udfs":"",
|
||||
"sqlType":"0",
|
||||
"title":"xxxx@xxx.com",
|
||||
"receivers":"xxxx@xxx.com",
|
||||
|
|
|
|||
|
|
@ -153,7 +153,7 @@ The browser access address [http://localhost:5173](http://localhost:5173) can lo
|
|||
|
||||
#### zookeeper
|
||||
|
||||
Download [ZooKeeper](https://zookeeper.apache.org/releases.html), and extract it.
|
||||
Download [ZooKeeper](https://www.apache.org/dyn/closer.lua/zookeeper/zookeeper-3.6.3), and extract it.
|
||||
|
||||
- Create directory `zkData` and `zkLog`
|
||||
- Go to the zookeeper installation directory, copy configure file `zoo_sample.cfg` to `conf/zoo.cfg`, and change value of dataDir in conf/zoo.cfg to dataDir=./tmp/zookeeper
|
||||
|
|
|
|||
|
|
@ -163,6 +163,9 @@ Resource Management => `http://localhost:8888/#/resource/file`
|
|||
|
||||
```
|
||||
| File Management
|
||||
| udf Management
|
||||
- Resource Management
|
||||
- Function management
|
||||
```
|
||||
|
||||
Data Source Management => `http://localhost:8888/#/datasource/list`
|
||||
|
|
|
|||
|
|
@ -31,7 +31,7 @@ The corresponding relationship between `Pull Request Type` and `Issue Type` is a
|
|||
<tr>
|
||||
<td style="text-align: center;">Bug</td>
|
||||
<td style="text-align: center;">Fix</td>
|
||||
<td style="text-align: center;">[Fix-3333][ui] Fix xxx</td>
|
||||
<td style="text-align: center;">[Fix-3333][server] Fix xxx</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td style="text-align: center;">Improvement</td>
|
||||
|
|
@ -44,24 +44,9 @@ The corresponding relationship between `Pull Request Type` and `Issue Type` is a
|
|||
<td style="text-align: center;">[Test-3333][api] Add the e2e test of xxx</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td style="text-align: center;">Doc</td>
|
||||
<td style="text-align: center;">Doc</td>
|
||||
<td style="text-align: center;">[Doc-3333] Improve xxx</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td style="text-align: center;">E2E</td>
|
||||
<td style="text-align: center;">E2E</td>
|
||||
<td style="text-align: center;">[E2E-3333] Implement xxx</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td style="text-align: center;">CI</td>
|
||||
<td style="text-align: center;">CI</td>
|
||||
<td style="text-align: center;">[CI] Improve xxx</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td style="text-align: center;">Chore</td>
|
||||
<td style="text-align: center;">Chore</td>
|
||||
<td style="text-align: center;">[Chore] Improve xxx</td>
|
||||
<td style="text-align: center;">Sub-Task</td>
|
||||
<td style="text-align: center;">(Parent type corresponding to Sub-Task)</td>
|
||||
<td style="text-align: center;">[Feature-3333][server] Implement xxx</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
|
||||
The purpose of the pseudo-cluster deployment is to deploy the DolphinScheduler service on a single machine. In this mode, DolphinScheduler's master, worker, API server, are all on the same machine.
|
||||
|
||||
If you are a new hand and want to experience DolphinScheduler functions, we recommend you install follow [Standalone deployment](standalone.md). If you want to experience more complete functions and schedule massive tasks, we recommend you install follow [pseudo-cluster deployment](pseudo-cluster.md). If you want to deploy DolphinScheduler in production, we recommend you follow [cluster deployment](cluster.md) or [Kubernetes deployment](kubernetes.md).
|
||||
If you are a new hand and want to experience DolphinScheduler functions, we recommend you install follow [Standalone deployment](standalone.md). If you want to experience more complete functions and schedule massive tasks, we recommend you install follow[pseudo-cluster deployment. If you want to deploy DolphinScheduler in production, we recommend you follow [cluster deployment](cluster.md) or [Kubernetes deployment](kubernetes.md).
|
||||
|
||||
## Preparation
|
||||
|
||||
|
|
|
|||
|
|
@ -36,7 +36,7 @@ Access address `http://localhost:12345/dolphinscheduler/ui` and login DolphinSch
|
|||
|
||||
### Start or Stop Server
|
||||
|
||||
The script `./bin/dolphinscheduler-daemon.sh` can be used not only quickly start standalone, but also to stop the service operation. The following are all the commands:
|
||||
The script `./bin/dolphinscheduler-daemon.sh`can be used not only quickly start standalone, but also to stop the service operation. The following are all the commands:
|
||||
|
||||
```shell
|
||||
# Start Standalone Server
|
||||
|
|
|
|||
|
|
@ -91,11 +91,6 @@ For example, you can get the master metrics by `curl http://localhost:5679/actua
|
|||
- stop: the number of stopped workflow instances
|
||||
- failover: the number of workflow instance fail-overs
|
||||
|
||||
### RPC Related Metrics
|
||||
|
||||
- ds.rpc.client.sync.request.exception.count: (counter) the number of exceptions occurred in sync rpc requests
|
||||
- ds.rpc.client.sync.request.duration.time: (histogram) the time cost of sync rpc requests
|
||||
|
||||
### Master Server Metrics
|
||||
|
||||
- ds.master.overload.count: (counter) the number of times the master overloaded
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ Create a shell task and enter `echo ${dt}` in the script content. In this case,
|
|||
|
||||
### Save the workflow and set global parameters
|
||||
|
||||
Set global parameter: On the workflow definition page, click the plus sign to the right of "Set Global", after filling in the variable name and value, select the appropriate parameter value type, save it.
|
||||
Set global parameter: On the workflow definition page, click the plus sign to the right of "Set Global", after filling in the variable name and value, save it.
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ Project-level parameters are valid for all task nodes under the entire project.
|
|||
|
||||
### Define project-level parameters
|
||||
|
||||
On the project page, click Project Parameters and Create Parameters, and fill in the parameter name and parameter value, select the appropriate parameter value type. As shown below:
|
||||
On the project page, click Project Parameters and Create Parameters, and fill in the parameter name and parameter value. As shown below:
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ Parameters are valid for all task nodes of the entire workflow. It can be config
|
|||
|
||||
## Usage
|
||||
|
||||
Usage of startup parameters is: at the task launch page, click the '+' below the 'Startup Parameter' and fill in the key and value, select the appropriate parameter value type then to save. The workflow will add them into global parameters.
|
||||
Usage of startup parameters is: at the task launch page, click the '+' below the 'Startup Parameter' and fill in the key and value to save. The workflow will add them into global parameters.
|
||||
|
||||
## Example
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
# Resource Center Configuration
|
||||
|
||||
- You could use `Resource Center` to upload text files and other task-related files.
|
||||
- You could use `Resource Center` to upload text files, UDFs and other task-related files.
|
||||
- You could configure `Resource Center` to use distributed file system like [Hadoop](https://hadoop.apache.org/docs/r2.7.0/) (2.6+), [MinIO](https://github.com/minio/minio) cluster or remote storage products like [AWS S3](https://aws.amazon.com/s3/), [Alibaba Cloud OSS](https://www.aliyun.com/product/oss), [Huawei Cloud OBS](https://support.huaweicloud.com/obs/index.html) etc.
|
||||
- You could configure `Resource Center` to use local file system. If you deploy `DolphinScheduler` in `Standalone` mode, you could configure it to use local file system for `Resource Center` without the need of an external `HDFS` system or `S3`.
|
||||
- Furthermore, if you deploy `DolphinScheduler` in `Cluster` mode, you could use [S3FS-FUSE](https://github.com/s3fs-fuse/s3fs-fuse) to mount `S3` or [JINDO-FUSE](https://help.aliyun.com/document_detail/187410.html) to mount `OSS` to your machines and use the local file system for `Resource Center`. In this way, you could operate remote files as if on your local machines.
|
||||
|
|
@ -28,37 +28,74 @@ The configuration you may need to change:
|
|||
|
||||
## connect AWS S3
|
||||
|
||||
if you want to upload resources to `Resource Center` connected to `S3`, you need to configure `api-server/conf/common.properties`, `api-server/conf/aws.yaml` and `worker-server/conf/common.properties`, `worker-server/conf/aws.yaml`. You can refer to the following:
|
||||
if you want to upload resources to `Resource Center` connected to `S3`, you need to configure `api-server/conf/common.properties` and `worker-server/conf/common.properties`. You can refer to the following:
|
||||
|
||||
config the following fields
|
||||
|
||||
```properties
|
||||
......
|
||||
|
||||
resource.storage.type=S3
|
||||
|
||||
......
|
||||
|
||||
resource.aws.access.key.id=aws_access_key_id
|
||||
# The AWS secret access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required
|
||||
resource.aws.secret.access.key=aws_secret_access_key
|
||||
# The AWS Region to use. if resource.storage.type=S3 or use EMR-Task, This configuration is required
|
||||
resource.aws.region=us-west-2
|
||||
# The name of the bucket. You need to create them by yourself. Otherwise, the system cannot start. All buckets in Amazon S3 share a single namespace; ensure the bucket is given a unique name.
|
||||
resource.aws.s3.bucket.name=dolphinscheduler
|
||||
# You need to set this parameter when private cloud s4. If S3 uses public cloud, you only need to set resource.aws.region or set to the endpoint of a public cloud such as S3.cn-north-1.amazonaws.com.cn
|
||||
resource.aws.s3.endpoint=
|
||||
|
||||
......
|
||||
```
|
||||
|
||||
```yaml
|
||||
aws:
|
||||
s3:
|
||||
# The AWS credentials provider type. support: AWSStaticCredentialsProvider, InstanceProfileCredentialsProvider
|
||||
# AWSStaticCredentialsProvider: use the access key and secret key to authenticate
|
||||
# InstanceProfileCredentialsProvider: use the IAM role to authenticate
|
||||
credentials.provider.type: AWSStaticCredentialsProvider
|
||||
access.key.id: <access.key.id>
|
||||
access.key.secret: <access.key.secret>
|
||||
region: <region>
|
||||
bucket.name: <bucket.name>
|
||||
endpoint: <endpoint>
|
||||
## Use HDFS or Remote Object Storage
|
||||
|
||||
```
|
||||
|
||||
## connect OSS S3
|
||||
|
||||
if you want to upload resources to `Resource Center` connected to `OSS`, you need to configure `api-server/conf/common.properties` and `worker-server/conf/common.properties`. You can refer to the following:
|
||||
|
||||
config the following fields
|
||||
After version 3.0.0-alpha, if you want to upload resources to `Resource Center` connected to `HDFS`, you need to configure `api-server/conf/common.properties` and `worker-server/conf/common.properties`.
|
||||
|
||||
```properties
|
||||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
# user data local directory path, please make sure the directory exists and have read write permissions
|
||||
data.basedir.path=/tmp/dolphinscheduler
|
||||
|
||||
# resource view suffixs
|
||||
#resource.view.suffixs=txt,log,sh,bat,conf,cfg,py,java,sql,xml,hql,properties,json,yml,yaml,ini,js
|
||||
|
||||
# resource storage type: LOCAL, HDFS, S3, OSS, GCS, ABS, OBS
|
||||
resource.storage.type=LOCAL
|
||||
# resource store on HDFS/S3/OSS path, resource file will store to this base path, self configuration, please make sure the directory exists on hdfs and have read write permissions. "/dolphinscheduler" is recommended
|
||||
resource.storage.upload.base.path=/tmp/dolphinscheduler
|
||||
|
||||
# The AWS access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required
|
||||
resource.aws.access.key.id=minioadmin
|
||||
# The AWS secret access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required
|
||||
resource.aws.secret.access.key=minioadmin
|
||||
# The AWS Region to use. if resource.storage.type=S3 or use EMR-Task, This configuration is required
|
||||
resource.aws.region=cn-north-1
|
||||
# The name of the bucket. You need to create them by yourself. Otherwise, the system cannot start. All buckets in Amazon S3 share a single namespace; ensure the bucket is given a unique name.
|
||||
resource.aws.s3.bucket.name=dolphinscheduler
|
||||
# You need to set this parameter when private cloud s3. If S3 uses public cloud, you only need to set resource.aws.region or set to the endpoint of a public cloud such as S3.cn-north-1.amazonaws.com.cn
|
||||
resource.aws.s3.endpoint=http://localhost:9000
|
||||
|
||||
# alibaba cloud access key id, required if you set resource.storage.type=OSS
|
||||
resource.alibaba.cloud.access.key.id=<your-access-key-id>
|
||||
# alibaba cloud access key secret, required if you set resource.storage.type=OSS
|
||||
|
|
@ -70,24 +107,89 @@ resource.alibaba.cloud.oss.bucket.name=dolphinscheduler
|
|||
# oss bucket endpoint, required if you set resource.storage.type=OSS
|
||||
resource.alibaba.cloud.oss.endpoint=https://oss-cn-hangzhou.aliyuncs.com
|
||||
|
||||
```
|
||||
|
||||
## connect OBS S3
|
||||
|
||||
if you want to upload resources to `Resource Center` connected to `OBS`, you need to configure `api-server/conf/common.properties` and `worker-server/conf/common.properties`. You can refer to the following:
|
||||
|
||||
config the following fields
|
||||
|
||||
```properties
|
||||
# access key id, required if you set resource.storage.type=OBS
|
||||
# alibaba cloud access key id, required if you set resource.storage.type=OBS
|
||||
resource.huawei.cloud.access.key.id=<your-access-key-id>
|
||||
# access key secret, required if you set resource.storage.type=OBS
|
||||
# alibaba cloud access key secret, required if you set resource.storage.type=OBS
|
||||
resource.huawei.cloud.access.key.secret=<your-access-key-secret>
|
||||
# oss bucket name, required if you set resource.storage.type=OBS
|
||||
resource.huawei.cloud.obs.bucket.name=dolphinscheduler
|
||||
# oss bucket endpoint, required if you set resource.storage.type=OBS
|
||||
resource.huawei.cloud.obs.endpoint=obs.cn-southwest-2.huaweicloud.com
|
||||
|
||||
# if resource.storage.type=HDFS, the user must have the permission to create directories under the HDFS root path
|
||||
resource.hdfs.root.user=hdfs
|
||||
# if resource.storage.type=S3, the value like: s3a://dolphinscheduler; if resource.storage.type=HDFS and namenode HA is enabled, you need to copy core-site.xml and hdfs-site.xml to conf dir
|
||||
resource.hdfs.fs.defaultFS=hdfs://mycluster:8020
|
||||
|
||||
# whether to startup kerberos
|
||||
hadoop.security.authentication.startup.state=false
|
||||
|
||||
# java.security.krb5.conf path
|
||||
java.security.krb5.conf.path=/opt/krb5.conf
|
||||
|
||||
# login user from keytab username
|
||||
login.user.keytab.username=hdfs-mycluster@ESZ.COM
|
||||
|
||||
# login user from keytab path
|
||||
login.user.keytab.path=/opt/hdfs.headless.keytab
|
||||
|
||||
# kerberos expire time, the unit is hour
|
||||
kerberos.expire.time=2
|
||||
|
||||
|
||||
# resourcemanager port, the default value is 8088 if not specified
|
||||
resource.manager.httpaddress.port=8088
|
||||
# if resourcemanager HA is enabled, please set the HA IPs; if resourcemanager is single, keep this value empty
|
||||
yarn.resourcemanager.ha.rm.ids=192.168.xx.xx,192.168.xx.xx
|
||||
# if resourcemanager HA is enabled or not use resourcemanager, please keep the default value; If resourcemanager is single, you only need to replace ds1 to actual resourcemanager hostname
|
||||
yarn.application.status.address=http://ds1:%s/ws/v1/cluster/apps/%s
|
||||
# job history status url when application number threshold is reached(default 10000, maybe it was set to 1000)
|
||||
yarn.job.history.status.address=http://ds1:19888/ws/v1/history/mapreduce/jobs/%s
|
||||
|
||||
# datasource encryption enable
|
||||
datasource.encryption.enable=false
|
||||
|
||||
# datasource encryption salt
|
||||
datasource.encryption.salt=!@#$%^&*
|
||||
|
||||
# data quality jar directory path, it would auto discovery data quality jar from this given dir. You should keep it empty if you do not change anything in
|
||||
# data-quality, it will auto discovery by dolphinscheduler itself. Change it only if you want to use your own data-quality jar and it is not in worker-server
|
||||
# libs directory(but may sure your jar name start with `dolphinscheduler-data-quality`).
|
||||
data-quality.jar.dir=
|
||||
|
||||
#data-quality.error.output.path=/tmp/data-quality-error-data
|
||||
|
||||
# Network IP gets priority, default inner outer
|
||||
|
||||
# Whether hive SQL is executed in the same session
|
||||
support.hive.oneSession=false
|
||||
|
||||
# use sudo or not, if set true, executing user is tenant user and deploy user needs sudo permissions; if set false, executing user is the deploy user and doesn't need sudo permissions
|
||||
sudo.enable=true
|
||||
|
||||
# network interface preferred like eth0, default: empty
|
||||
#dolphin.scheduler.network.interface.preferred=
|
||||
|
||||
# network IP gets priority, default: inner outer
|
||||
#dolphin.scheduler.network.priority.strategy=default
|
||||
|
||||
# system env path
|
||||
#dolphinscheduler.env.path=dolphinscheduler_env.sh
|
||||
|
||||
# development state
|
||||
development.state=false
|
||||
|
||||
# rpc port
|
||||
alert.rpc.port=50052
|
||||
|
||||
# set path of conda.sh
|
||||
conda.path=/opt/anaconda3/etc/profile.d/conda.sh
|
||||
|
||||
# Task resource limit state
|
||||
task.resource.limit.state=false
|
||||
|
||||
# way to collect applicationId: log(original regex match), aop
|
||||
appId.collect: log
|
||||
```
|
||||
|
||||
> **Note:**
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
# Resource Center Introduction
|
||||
|
||||
The Resource Center is typically used for uploading files and task group management. For a stand-alone
|
||||
The Resource Center is typically used for uploading files, UDF functions, and task group management. For a stand-alone
|
||||
environment, you can select the local file directory as the upload folder (**this operation does not require Hadoop or HDFS deployment**).
|
||||
Of course, you can also choose to upload to Hadoop or MinIO cluster. In this case, you need to have Hadoop (2.6+) or MinIO and other related environments.
|
||||
|
|
|
|||
|
|
@ -0,0 +1,45 @@
|
|||
# UDF Manage
|
||||
|
||||
## Resource Management
|
||||
|
||||
- The resource management and file management functions are similar. The difference is that the resource management is the UDF upload function, and the file management uploads the user programs, scripts and configuration files.
|
||||
- It mainly includes the following operations: rename, download, delete, etc.
|
||||
- Upload UDF resources: Same as uploading files.
|
||||
|
||||
## Function Management
|
||||
|
||||
### Create UDF function
|
||||
|
||||
Click `Create UDF Function`, enter the UDF function parameters, select the UDF resource, and click `Submit` to create the UDF function.
|
||||
Currently only temporary UDF functions for HIVE are supported.
|
||||
|
||||
- UDF function name: Enter the name of the UDF function.
|
||||
- Package name Class name: Enter the full path of the UDF function.
|
||||
- UDF resource: Set the resource file corresponding to the created UDF function.
|
||||
|
||||
|
||||
|
||||
## Example
|
||||
|
||||
### Write UDF functions
|
||||
|
||||
Users can customize the desired UDF function according to actual production requirements. Here's a function that appends "HelloWorld" to the end of any string. As shown below:
|
||||
|
||||
|
||||
|
||||
### Configure the UDF function
|
||||
|
||||
Before configuring UDF functions, you need to upload the required function jar package through resource management. Then enter the function management and configure the relevant information. As shown below:
|
||||
|
||||
|
||||
|
||||
### Use UDF functions
|
||||
|
||||
In the process of using UDF functions, users only need to pay attention to the specific function writing, and upload the configuration through the resource center. The system will automatically configure the create function statement, refer to the following: [SqlTask](https://github.com/apache/dolphinscheduler/blob/923f3f38e3271d7f1d22b3abc3497cecb6957e4a/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTask.java#L507-L531)
|
||||
|
||||
Enter the workflow to define an SQL node, the data source type is HIVE, and the data source instance type is HIVE/IMPALA.
|
||||
|
||||
- SQL statement: `select HwUdf("abc");` This function is used in the same way as the built-in functions, and can be accessed directly using the function name.
|
||||
- UDF function: Select the one configured for the resource center.
|
||||
|
||||
|
||||
|
|
@ -103,8 +103,8 @@ public void doPOSTParam()throws Exception{
|
|||
|
||||
## Granted Permissions
|
||||
|
||||
* Granted permissions include project permissions, resource permissions, data source permissions.
|
||||
* Administrators can authorize projects, resources, data sources that ordinary users do not create. Because the authorization methods of projects, resources, data sources are all the same, the project authorization is used as an example to introduce.
|
||||
* Granted permissions include project permissions, resource permissions, data source permissions, and UDF function permissions.
|
||||
* Administrators can authorize projects, resources, data sources, and UDF functions that ordinary users do not create. Because the authorization methods of projects, resources, data sources and UDF functions are all the same, the project authorization is used as an example to introduce.
|
||||
* Note: For projects created by the user, the user has all permissions. Therefore, permission changes to projects created by users themselves are not valid.
|
||||
- The administrator enters the `Security Center -> User Management` page, and clicks the "Authorize" button of the user to be authorized, as shown in the following figure:
|
||||
|
||||
|
|
@ -118,7 +118,7 @@ public void doPOSTParam()throws Exception{
|
|||
|
||||
|
||||
|
||||
- Resources, data sources authorization are the same as project authorization.
|
||||
- Resources, data sources, and UDF function authorization are the same as project authorization.
|
||||
|
||||
## Worker Grouping
|
||||
|
||||
|
|
|
|||
|
|
@ -73,17 +73,14 @@ Parameters of restarting the task by interface
|
|||
|
||||
## Environment to prepare
|
||||
|
||||
Some AWS configuration is required, modify a field in file `aws.yaml`
|
||||
Some AWS configuration is required, modify a field in file `common.properties`
|
||||
|
||||
```yaml
|
||||
dms:
|
||||
# The AWS credentials provider type. support: AWSStaticCredentialsProvider, InstanceProfileCredentialsProvider
|
||||
# AWSStaticCredentialsProvider: use the access key and secret key to authenticate
|
||||
# InstanceProfileCredentialsProvider: use the IAM role to authenticate
|
||||
credentials.provider.type: AWSStaticCredentialsProvider
|
||||
access.key.id: <access.key.id>
|
||||
access.key.secret: <access.key.secret>
|
||||
region: <region>
|
||||
endpoint: <endpoint>
|
||||
# The AWS access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required
|
||||
resource.aws.access.key.id=<YOUR AWS ACCESS KEY>
|
||||
# The AWS secret access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required
|
||||
resource.aws.secret.access.key=<YOUR AWS SECRET KEY>
|
||||
# The AWS Region to use. if resource.storage.type=S3 or use EMR-Task, This configuration is required
|
||||
resource.aws.region=<AWS REGION>
|
||||
```
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,22 @@
|
|||
# Pigeon
|
||||
|
||||
## Overview
|
||||
|
||||
Pigeon is a task used to trigger remote tasks, acquire logs or status by calling remote WebSocket service. It is DolphinScheduler uses a remote WebSocket service to call tasks.
|
||||
|
||||
## Create Task
|
||||
|
||||
- Click `Project Management -> Project Name -> Workflow Definition`, and click the `Create Workflow` button to enter the DAG editing page.
|
||||
- Drag from the toolbar <img src="../../../../img/pigeon.png" width="20"/> to the canvas to create a new Pigeon task.
|
||||
|
||||
## Task Parameters
|
||||
|
||||
[//]: # (TODO: use the commented anchor below once our website template supports this syntax)
|
||||
[//]: # (- Please refer to [DolphinScheduler Task Parameters Appendix](appendix.md#default-task-parameters) `Default Task Parameters` section for default parameters.)
|
||||
|
||||
- Please refer to [DolphinScheduler Task Parameters Appendix](appendix.md) `Default Task Parameters` section for default parameters.
|
||||
|
||||
| **Parameter** | **Description** |
|
||||
|------------------|---------------------------------------|
|
||||
| Target task name | Target task name of this Pigeon node. |
|
||||
|
||||
|
|
@ -35,17 +35,14 @@ The task plugin are shown as follows:
|
|||
|
||||
## Environment to prepare
|
||||
|
||||
Some AWS configuration is required, modify a field in file `aws.yaml`
|
||||
Some AWS configuration is required, modify a field in file `common.properties`
|
||||
|
||||
```yaml
|
||||
sagemaker:
|
||||
# The AWS credentials provider type. support: AWSStaticCredentialsProvider, InstanceProfileCredentialsProvider
|
||||
# AWSStaticCredentialsProvider: use the access key and secret key to authenticate
|
||||
# InstanceProfileCredentialsProvider: use the IAM role to authenticate
|
||||
credentials.provider.type: AWSStaticCredentialsProvider
|
||||
access.key.id: <access.key.id>
|
||||
access.key.secret: <access.key.secret>
|
||||
region: <region>
|
||||
endpoint: <endpoint>
|
||||
# The AWS access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required
|
||||
resource.aws.access.key.id=<YOUR AWS ACCESS KEY>
|
||||
# The AWS secret access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required
|
||||
resource.aws.secret.access.key=<YOUR AWS SECRET KEY>
|
||||
# The AWS Region to use. if resource.storage.type=S3 or use EMR-Task, This configuration is required
|
||||
resource.aws.region=<AWS REGION>
|
||||
```
|
||||
|
||||
|
|
|
|||
|
|
@ -21,11 +21,12 @@ Refer to [datasource-setting](../howto/datasource-setting.md) `DataSource Center
|
|||
- Please refer to [DolphinScheduler Task Parameters Appendix](appendix.md) `Default Task Parameters` section for default parameters.
|
||||
|
||||
| **Parameter** | **Description** |
|
||||
|-------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---|
|
||||
|-------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
|
||||
| Data source | Select the corresponding DataSource. |
|
||||
| SQL type | Supports query and non-query. <ul><li>Query: supports `DML select` type commands, which return a result set. You can specify three templates for email notification as form, attachment or form attachment;</li><li>Non-query: support `DDL` all commands and `DML update, delete, insert` three types of commands;<ul><li>Segmented execution symbol: When the data source does not support executing multiple SQL statements at a time, the symbol for splitting SQL statements is provided to call the data source execution method multiple times. Example: 1. When the Hive data source is selected as the data source, please do not use `;\n` due to Hive JDBC does not support executing multiple SQL statements at one time; 2. When the MySQL data source is selected as the data source, and multi-segment SQL statements are to be executed, this parameter needs to be filled in with a semicolon `;. Because the MySQL data source does not support executing multiple SQL statements at one time.</li></ul></li></ul> |
|
||||
| SQL parameter | The input parameter format is `key1=value1;key2=value2...`. |
|
||||
| SQL statement | SQL statement. | |
|
||||
| SQL statement | SQL statement. |
|
||||
| UDF function | For Hive DataSources, you can refer to UDF functions created in the resource center, but other DataSource do not support UDF functions. |
|
||||
| Custom parameters | SQL task type, and stored procedure is a custom parameter order, to set customized parameter type and data type for the method is the same as the stored procedure task type. The difference is that the custom parameter of the SQL task type replaces the `${variable}` in the SQL statement. |
|
||||
| Pre-SQL | Pre-SQL executes before the SQL statement. |
|
||||
| Post-SQL | Post-SQL executes after the SQL statement. |
|
||||
|
|
@ -56,3 +57,5 @@ Table created in the Pre-SQL, after use in the SQL statement, cleaned in the Pos
|
|||
## Note
|
||||
|
||||
Pay attention to the selection of SQL type. If it is an insert operation, need to change to "Non-Query" type.
|
||||
|
||||
To compatible with long session,UDF function are created by the syntax(CREATE OR REPLACE)
|
||||
|
|
|
|||
|
|
@ -26,8 +26,3 @@ This document records the incompatible updates between each version. You need to
|
|||
|
||||
* Add required field `database` in /datasources/tables && /datasources/tableColumns Api [#14406](https://github.com/apache/dolphinscheduler/pull/14406)
|
||||
|
||||
## 3.3.0
|
||||
|
||||
* Remove the `udf-manage` function from the `resource center` ([#16209])
|
||||
* Remove the `Pigeon` from the `Task Plugin` ([#16218])
|
||||
|
||||
|
|
|
|||
|
|
@ -331,6 +331,7 @@ common.properties配置文件目前主要是配置hadoop/s3/yarn/applicationId
|
|||
| worker.registry-disconnect-strategy.max-waiting-time | 100s | 当Worker与注册中心失联之后重连时间, 之后当strategy为waiting时，该值生效。 该值表示当Worker与注册中心失联时会在给定时间之内进行重连, 在给定时间之内重连失败将会停止自己，在重连时，Worker会丢弃kill正在执行的任务。值为0表示会无限期等待 |
|
||||
| worker.task-execute-threads-full-policy | REJECT | 如果是 REJECT, 当Worker中等待队列中的任务数达到exec-threads时, Worker将会拒绝接下来新接收的任务，Master将会重新分发该任务; 如果是 CONTINUE, Worker将会接收任务，放入等待队列中等待空闲线程去执行该任务 |
|
||||
| worker.tenant-config.auto-create-tenant-enabled | true | 租户对应于系统的用户,由worker提交作业.如果系统没有该用户,则在参数worker.tenant.auto.create为true后自动创建。 |
|
||||
| worker.tenant-config.distributed-tenant-enabled | false | 如果设置为true, auto-create-tenant-enabled 将会不起作用。 |
|
||||
| worker.tenant-config.default-tenant-enabled | false | 如果设置为true, 将会使用worker服务启动用户作为 `default` 租户。 |
|
||||
|
||||
## Alert Server相关配置
|
||||
|
|
|
|||
|
|
@ -20,6 +20,8 @@
|
|||
|
||||
- 一个用户可以有多个项目，用户项目授权通过`t_ds_relation_project_user`表完成project_id和user_id的关系绑定；<br />
|
||||
- `t_ds_projcet`表中的`user_id`表示创建该项目的用户，`t_ds_relation_project_user`表中的`user_id`表示对项目有权限的用户；<br />
|
||||
- `t_ds_resources`表中的`user_id`表示创建该资源的用户，`t_ds_relation_resources_user`中的`user_id`表示对资源有权限的用户；<br />
|
||||
- `t_ds_udfs`表中的`user_id`表示创建该UDF的用户，`t_ds_relation_udfs_user`表中的`user_id`表示对UDF有权限的用户；<br />
|
||||
|
||||
### 项目 - 租户 - 工作流定义 - 定时
|
||||
|
||||
|
|
|
|||
|
|
@ -145,6 +145,7 @@
|
|||
5| |type |String | 数据库类型
|
||||
6| |datasource |Int | 数据源id
|
||||
7| |sql |String | 查询SQL语句
|
||||
8| |udfs | String| udf函数|UDF函数id,以逗号分隔.
|
||||
9| |sqlType | String| SQL节点类型 |0 查询 , 1 非查询
|
||||
10| |title |String | 邮件标题
|
||||
11| |receivers |String | 收件人
|
||||
|
|
@ -178,6 +179,7 @@
|
|||
"type":"MYSQL",
|
||||
"datasource":1,
|
||||
"sql":"select id , namge , age from emp where id = ${id}",
|
||||
"udfs":"",
|
||||
"sqlType":"0",
|
||||
"title":"xxxx@xxx.com",
|
||||
"receivers":"xxxx@xxx.com",
|
||||
|
|
|
|||
|
|
@ -148,7 +148,7 @@ pnpm run dev
|
|||
|
||||
#### zookeeper
|
||||
|
||||
下载 [ZooKeeper](https://zookeeper.apache.org/releases.html)，解压
|
||||
下载 [ZooKeeper](https://www.apache.org/dyn/closer.lua/zookeeper/zookeeper-3.6.3)，解压
|
||||
|
||||
* 在 ZooKeeper 的目录下新建 zkData、zkLog文件夹
|
||||
* 将 conf 目录下的 `zoo_sample.cfg` 文件，复制一份，重命名为 `zoo.cfg`，修改其中数据和日志的配置，如：
|
||||
|
|
|
|||
|
|
@ -163,6 +163,9 @@ npm install node-sass --unsafe-perm #单独安装node-sass依赖
|
|||
|
||||
```
|
||||
| 文件管理
|
||||
| UDF管理
|
||||
- 资源管理
|
||||
- 函数管理
|
||||
```
|
||||
|
||||
数据源管理 => `http://localhost:8888/#/datasource/list`
|
||||
|
|
|
|||
|
|
@ -33,7 +33,7 @@ waste time。
|
|||
<tr>
|
||||
<td style="text-align: center;">Bug</td>
|
||||
<td style="text-align: center;">Fix</td>
|
||||
<td style="text-align: center;">[Fix-3333][ui] Fix xxx</td>
|
||||
<td style="text-align: center;">[Fix-3333][server] Fix xxx</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td style="text-align: center;">Improvement</td>
|
||||
|
|
@ -46,23 +46,9 @@ waste time。
|
|||
<td style="text-align: center;">[Test-3333][api] Add the e2e test of xxx</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td style="text-align: center;">Doc</td>
|
||||
<td style="text-align: center;">Doc</td>
|
||||
<td style="text-align: center;">[Doc-3333] Improve xxx</td>
|
||||
<tr>
|
||||
<td style="text-align: center;">E2E</td>
|
||||
<td style="text-align: center;">E2E</td>
|
||||
<td style="text-align: center;">[E2E-3333] Implement xxx</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td style="text-align: center;">CI</td>
|
||||
<td style="text-align: center;">CI</td>
|
||||
<td style="text-align: center;">[CI] Improve xxx</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td style="text-align: center;">Chore</td>
|
||||
<td style="text-align: center;">Chore</td>
|
||||
<td style="text-align: center;">[Chore] Improve xxx</td>
|
||||
<td style="text-align: center;">Sub-Task</td>
|
||||
<td style="text-align: center;">Sub-Task 对应的父类型</td>
|
||||
<td style="text-align: center;">[Feature-3333][server] Implement xxx</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
|
|
|
|||
|
|
@ -91,11 +91,6 @@ metrics exporter端口`server.port`是在application.yaml里定义的: master: `
|
|||
- stop：停止的工作流实例数量
|
||||
- failover：容错的工作流实例数量
|
||||
|
||||
### RPC相关指标
|
||||
|
||||
- ds.rpc.client.sync.request.exception.count: (counter) 同步rpc请求异常数
|
||||
- ds.rpc.client.sync.request.duration.time: (histogram) 同步rpc请求耗时
|
||||
|
||||
### Master Server指标
|
||||
|
||||
- ds.master.overload.count: (counter) master过载次数
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@
|
|||
|
||||
### 保存工作流，并设置全局参数
|
||||
|
||||
全局参数配置方式如下：在工作流定义页面，点击“设置全局”右边的加号，填写对应的变量名称和对应的值，选择相应的参数值类型，保存即可。如下图所示：
|
||||
全局参数配置方式如下：在工作流定义页面，点击“设置全局”右边的加号，填写对应的变量名称和对应的值，保存即可。如下图所示：
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@
|
|||
|
||||
### 定义项目级别参数
|
||||
|
||||
在项目管理页面，点击项目级别参数，点击创建项目级别参数，填写参数名称和参数值，选择相应的参数值类型。如下图所示：
|
||||
在项目管理页面，点击项目级别参数，点击创建项目级别参数，填写参数名称和参数值。如下图所示：
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@
|
|||
|
||||
## 使用方式
|
||||
|
||||
启动参数配置方式如下：在启动前参数设置界面，点击“启动参数“下面的加号，填写对应的参数名称和对应的值，选择相应的参数值类型，点击确定，工作流会将启动参数加入全局参数中。
|
||||
启动参数配置方式如下：在启动前参数设置界面，点击“启动参数“下面的加号，填写对应的参数名称和对应的值，点击确定，工作流会将启动参数加入全局参数中。
|
||||
|
||||
## 任务样例
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
# 资源中心配置详情
|
||||
|
||||
- 资源中心通常用于上传文件以及任务组管理等操作。
|
||||
- 资源中心通常用于上传文件、UDF 函数，以及任务组管理等操作。
|
||||
- 资源中心可以对接分布式的文件存储系统，如[Hadoop](https://hadoop.apache.org/docs/r2.7.0/)（2.6+）或者[MinIO](https://github.com/minio/minio)集群，也可以对接远端的对象存储，如[AWS S3](https://aws.amazon.com/s3/)或者[阿里云 OSS](https://www.aliyun.com/product/oss)，[华为云 OBS](https://support.huaweicloud.com/obs/index.html) 等。
|
||||
- 资源中心也可以直接对接本地文件系统。在单机模式下，您无需依赖`Hadoop`或`S3`一类的外部存储系统，可以方便地对接本地文件系统进行体验。
|
||||
- 除此之外，对于集群模式下的部署，您可以通过使用[S3FS-FUSE](https://github.com/s3fs-fuse/s3fs-fuse)将`S3`挂载到本地，或者使用[JINDO-FUSE](https://help.aliyun.com/document_detail/187410.html)将`OSS`挂载到本地等，再用资源中心对接本地文件系统方式来操作远端对象存储中的文件。
|
||||
|
|
@ -26,35 +26,77 @@ Dolphinscheduler 资源中心使用本地系统默认是开启的，不需要用
|
|||
|
||||
## 对接AWS S3
|
||||
|
||||
如果需要使用到资源中心的 S3 上传资源，我们需要对以下路径的进行配置：`api-server/conf/common.properties`, `api-server/conf/aws.yaml` 和 `worker-server/conf/common.properties`, `worker-server/conf/aws.yaml`。可参考如下：
|
||||
如果需要使用到资源中心的 S3 上传资源，我们需要对以下路径的进行配置：`api-server/conf/common.properties` 和 `worker-server/conf/common.properties`。可参考如下：
|
||||
|
||||
配置以下字段
|
||||
|
||||
```properties
|
||||
......
|
||||
|
||||
resource.storage.type=S3
|
||||
|
||||
......
|
||||
|
||||
resource.aws.access.key.id=aws_access_key_id
|
||||
# The AWS secret access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required
|
||||
resource.aws.secret.access.key=aws_secret_access_key
|
||||
# The AWS Region to use. if resource.storage.type=S3 or use EMR-Task, This configuration is required
|
||||
resource.aws.region=us-west-2
|
||||
# The name of the bucket. You need to create them by yourself. Otherwise, the system cannot start. All buckets in Amazon S3 share a single namespace; ensure the bucket is given a unique name.
|
||||
resource.aws.s3.bucket.name=dolphinscheduler
|
||||
# You need to set this parameter when private cloud s4. If S3 uses public cloud, you only need to set resource.aws.region or set to the endpoint of a public cloud such as S3.cn-north-1.amazonaws.com.cn
|
||||
resource.aws.s3.endpoint=
|
||||
|
||||
......
|
||||
```
|
||||
|
||||
```yaml
|
||||
aws:
|
||||
s3:
|
||||
# The AWS credentials provider type. support: AWSStaticCredentialsProvider, InstanceProfileCredentialsProvider
|
||||
# AWSStaticCredentialsProvider: use the access key and secret key to authenticate
|
||||
# InstanceProfileCredentialsProvider: use the IAM role to authenticate
|
||||
credentials.provider.type: AWSStaticCredentialsProvider
|
||||
access.key.id: <access.key.id>
|
||||
access.key.secret: <access.key.secret>
|
||||
region: <region>
|
||||
bucket.name: <bucket.name>
|
||||
endpoint: <endpoint>
|
||||
## 对接分布式或远端对象存储
|
||||
|
||||
```
|
||||
当需要使用资源中心进行相关文件的创建或者上传操作时，所有的文件和资源都会被存储在分布式文件系统`HDFS`或者远端的对象存储，如`S3`上。所以需要进行以下配置：
|
||||
|
||||
## 对接阿里云 OSS
|
||||
### 配置 common.properties 文件
|
||||
|
||||
如果需要使用到资源中心的 OSS 上传资源，我们需要对以下路径的进行配置：`api-server/conf/common.properties` 和 `worker-server/conf/common.properties`。可参考如下：
|
||||
在 3.0.0-alpha 版本之后，如果需要使用到资源中心的 HDFS 或 S3 上传资源，我们需要对以下路径的进行配置：`api-server/conf/common.properties` 和 `worker-server/conf/common.properties`。可参考如下：
|
||||
|
||||
```properties
|
||||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
# user data local directory path, please make sure the directory exists and have read write permissions
|
||||
data.basedir.path=/tmp/dolphinscheduler
|
||||
|
||||
# resource storage type: LOCAL, HDFS, S3, OSS, GCS, ABS, OBS
|
||||
resource.storage.type=LOCAL
|
||||
|
||||
# resource store on HDFS/S3/OSS path, resource file will store to this hadoop hdfs path, self configuration,
|
||||
# please make sure the directory exists on hdfs and have read write permissions. "/dolphinscheduler" is recommended
|
||||
resource.storage.upload.base.path=/tmp/dolphinscheduler
|
||||
|
||||
# The AWS access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required
|
||||
resource.aws.access.key.id=minioadmin
|
||||
# The AWS secret access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required
|
||||
resource.aws.secret.access.key=minioadmin
|
||||
# The AWS Region to use. if resource.storage.type=S3 or use EMR-Task, This configuration is required
|
||||
resource.aws.region=cn-north-1
|
||||
# The name of the bucket. You need to create them by yourself. Otherwise, the system cannot start. All buckets in Amazon S3 share a single namespace; ensure the bucket is given a unique name.
|
||||
resource.aws.s3.bucket.name=dolphinscheduler
|
||||
# You need to set this parameter when private cloud s3. If S3 uses public cloud, you only need to set resource.aws.region or set to the endpoint of a public cloud such as S3.cn-north-1.amazonaws.com.cn
|
||||
resource.aws.s3.endpoint=http://localhost:9000
|
||||
|
||||
# alibaba cloud access key id, required if you set resource.storage.type=OSS
|
||||
resource.alibaba.cloud.access.key.id=<your-access-key-id>
|
||||
# alibaba cloud access key secret, required if you set resource.storage.type=OSS
|
||||
|
|
@ -66,22 +108,87 @@ resource.alibaba.cloud.oss.bucket.name=dolphinscheduler
|
|||
# oss bucket endpoint, required if you set resource.storage.type=OSS
|
||||
resource.alibaba.cloud.oss.endpoint=https://oss-cn-hangzhou.aliyuncs.com
|
||||
|
||||
```
|
||||
|
||||
## 对接华为云 OBS
|
||||
|
||||
如果需要使用到资源中心的 OBS 上传资源，我们需要对以下路径的进行配置：`api-server/conf/common.properties` 和 `worker-server/conf/common.properties`。可参考如下：
|
||||
|
||||
```properties
|
||||
# access key id, required if you set resource.storage.type=OBS
|
||||
# alibaba cloud access key id, required if you set resource.storage.type=OBS
|
||||
resource.huawei.cloud.access.key.id=<your-access-key-id>
|
||||
# access key secret, required if you set resource.storage.type=OBS
|
||||
# alibaba cloud access key secret, required if you set resource.storage.type=OBS
|
||||
resource.huawei.cloud.access.key.secret=<your-access-key-secret>
|
||||
# oss bucket name, required if you set resource.storage.type=OBS
|
||||
resource.huawei.cloud.obs.bucket.name=dolphinscheduler
|
||||
# oss bucket endpoint, required if you set resource.storage.type=OBS
|
||||
resource.huawei.cloud.obs.endpoint=obs.cn-southwest-2.huaweicloud.com
|
||||
|
||||
# if resource.storage.type=HDFS, the user must have the permission to create directories under the HDFS root path
|
||||
resource.hdfs.root.user=root
|
||||
# if resource.storage.type=S3, the value like: s3a://dolphinscheduler;
|
||||
# if resource.storage.type=HDFS and namenode HA is enabled, you need to copy core-site.xml and hdfs-site.xml to conf dir
|
||||
resource.hdfs.fs.defaultFS=hdfs://localhost:8020
|
||||
|
||||
# whether to startup kerberos
|
||||
hadoop.security.authentication.startup.state=false
|
||||
|
||||
# java.security.krb5.conf path
|
||||
java.security.krb5.conf.path=/opt/krb5.conf
|
||||
|
||||
# login user from keytab username
|
||||
login.user.keytab.username=hdfs-mycluster@ESZ.COM
|
||||
|
||||
# login user from keytab path
|
||||
login.user.keytab.path=/opt/hdfs.headless.keytab
|
||||
|
||||
# kerberos expire time, the unit is hour
|
||||
kerberos.expire.time=2
|
||||
# resource view suffixs
|
||||
#resource.view.suffixs=txt,log,sh,bat,conf,cfg,py,java,sql,xml,hql,properties,json,yml,yaml,ini,js
|
||||
|
||||
# resourcemanager port, the default value is 8088 if not specified
|
||||
resource.manager.httpaddress.port=8088
|
||||
# if resourcemanager HA is enabled, please set the HA IPs; if resourcemanager is single, keep this value empty
|
||||
yarn.resourcemanager.ha.rm.ids=192.168.xx.xx,192.168.xx.xx
|
||||
# if resourcemanager HA is enabled or not use resourcemanager, please keep the default value;
|
||||
# If resourcemanager is single, you only need to replace ds1 to actual resourcemanager hostname
|
||||
yarn.application.status.address=http://localhost:%s/ds/v1/cluster/apps/%s
|
||||
# job history status url when application number threshold is reached(default 10000, maybe it was set to 1000)
|
||||
yarn.job.history.status.address=http://localhost:19888/ds/v1/history/mapreduce/jobs/%s
|
||||
|
||||
# datasource encryption enable
|
||||
datasource.encryption.enable=false
|
||||
|
||||
# datasource encryption salt
|
||||
datasource.encryption.salt=!@#$%^&*
|
||||
|
||||
# data quality jar directory path, it would auto discovery data quality jar from this given dir. You should keep it empty if you do not change anything in
|
||||
# data-quality, it will auto discovery by dolphinscheduler itself. Change it only if you want to use your own data-quality jar and it is not in worker-server
|
||||
# libs directory(but may sure your jar name start with `dolphinscheduler-data-quality`).
|
||||
data-quality.jar.dir=
|
||||
|
||||
#data-quality.error.output.path=/tmp/data-quality-error-data
|
||||
|
||||
# Network IP gets priority, default inner outer
|
||||
|
||||
# Whether hive SQL is executed in the same session
|
||||
support.hive.oneSession=false
|
||||
|
||||
# use sudo or not, if set true, executing user is tenant user and deploy user needs sudo permissions;
|
||||
# if set false, executing user is the deploy user and doesn't need sudo permissions
|
||||
sudo.enable=true
|
||||
|
||||
# network interface preferred like eth0, default: empty
|
||||
#dolphin.scheduler.network.interface.preferred=
|
||||
|
||||
# network IP gets priority, default: inner outer
|
||||
#dolphin.scheduler.network.priority.strategy=default
|
||||
|
||||
# system env path
|
||||
#dolphinscheduler.env.path=env/dolphinscheduler_env.sh
|
||||
|
||||
# development state
|
||||
development.state=false
|
||||
|
||||
# rpc port
|
||||
alert.rpc.port=50052
|
||||
|
||||
# way to collect applicationId: log(original regex match), aop
|
||||
appId.collect: log
|
||||
```
|
||||
|
||||
> **注意**：
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
# 资源中心简介
|
||||
|
||||
资源中心通常用于上传文件和任务组管理。 对于 standalone 环境，可以选择本地文件目录作为上传文件夹（此操作不需要Hadoop部署）。当然，你也可以
|
||||
资源中心通常用于上传文件、UDF 函数和任务组管理。 对于 standalone 环境，可以选择本地文件目录作为上传文件夹（此操作不需要Hadoop部署）。当然，你也可以
|
||||
选择上传到 Hadoop 或者 MinIO 集群。 在这种情况下，您需要有 Hadoop（2.6+）或 MinIO 等相关环境。
|
||||
|
|
|
|||
|
|
@ -0,0 +1,46 @@
|
|||
# UDF 管理
|
||||
|
||||
- 资源管理和文件管理功能类似，不同之处是资源管理是上传的 UDF 函数，文件管理上传的是用户程序，脚本及配置文件。
|
||||
- 主要包括以下操作：重命名、下载、删除等。
|
||||
* 上传 UDF 资源
|
||||
|
||||
> 和上传文件相同。
|
||||
|
||||
## 函数管理
|
||||
|
||||
* 创建 UDF 函数
|
||||
|
||||
> 点击“创建 UDF 函数”，输入 UDF 函数参数，选择udf资源，点击“提交”，创建 UDF 函数。
|
||||
> 目前只支持 HIVE 的临时 UDF 函数
|
||||
|
||||
- UDF 函数名称：输入 UDF 函数时的名称
|
||||
- 包名类名：输入 UDF 函数的全路径
|
||||
- UDF 资源：设置创建的 UDF 对应的资源文件
|
||||
|
||||
|
||||
|
||||
## 任务样例
|
||||
|
||||
### 编写 UDF 函数
|
||||
|
||||
用户可以根据实际生产需求，自定义想要的 UDF 函数。这里编写一个在任意字符串的末尾添加 "HelloWorld" 的函数。如下图所示：
|
||||
|
||||
|
||||
|
||||
### 配置 UDF 函数
|
||||
|
||||
配置 UDF 函数前，需要先通过资源管理上传所需的函数 jar 包。然后进入函数管理，配置相关信息即可。如下图所示：
|
||||
|
||||
|
||||
|
||||
### 使用 UDF 函数
|
||||
|
||||
在使用 UDF 函数过程中，用户只需关注具体的函数编写，通过资源中心上传配置完成即可。系统会自动配置 create function 语句，参考如下：[SqlTask](https://github.com/apache/dolphinscheduler/blob/923f3f38e3271d7f1d22b3abc3497cecb6957e4a/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTask.java#L507-L531)
|
||||
|
||||
进入工作流定义一个 SQL 节点，数据源类型选择为 HIVE，数据源实例类型为 HIVE/IMPALA。
|
||||
|
||||
- SQL 语句：`select HwUdf("abc");` 该函数与内置函数使用方式一样，直接使用函数名称即可访问。
|
||||
- UDF 函数：选择资源中心所配置的即可。
|
||||
|
||||
|
||||
|
||||
|
|
@ -97,8 +97,8 @@
|
|||
|
||||
## 授予权限
|
||||
|
||||
* 授予权限包括项目权限，数据源权限和k8s命名空间。
|
||||
* 管理员可以对普通用户进行非其创建的项目、数据源k8s命名空间。因为项目、数据源、k8s命名空间授权方式都是一样的，所以以项目授权为例介绍。
|
||||
* 授予权限包括项目权限，资源权限，数据源权限，UDF函数权限，k8s命名空间。
|
||||
* 管理员可以对普通用户进行非其创建的项目、资源、数据源、UDF函数、k8s命名空间。因为项目、资源、数据源、UDF函数、k8s命名空间授权方式都是一样的，所以以项目授权为例介绍。
|
||||
* 注意：对于用户自己创建的项目，该用户默认拥有所有的权限，因此对用户自己创建的项目进行权限变更是无效的。
|
||||
- 管理员进入`安全中心->用户管理页面`，点击需授权用户的“授权”按钮，如下图所示：
|
||||
|
||||
|
|
@ -112,7 +112,7 @@
|
|||
|
||||
|
||||
|
||||
- 数据源授权同项目授权。
|
||||
- 资源、数据源、UDF 函数授权同项目授权。
|
||||
|
||||
## Worker 分组
|
||||
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@
|
|||
|----------|--------------------------------------------------------------------------------------------------------------------------------------|
|
||||
| 任务名称 | 任务的名称，同一个工作流定义中的节点名称不能重复。 |
|
||||
| 运行标志 | 标识这个节点是否需要调度执行，如果不需要执行，可以打开禁止执行开关。 |
|
||||
| 缓存执行 | 标识这个节点是否需要进行缓存，如果缓存，则对于相同标识（相同任务版本，相同任务定义，相同参数传入）的任务进行缓存，运行时若已经存在缓存过的任务时，不再重复执行，直接复用结果。 |
|
||||
| 缓存执行 | 标识这个节点是否需要进行缓存，如果缓存，则对于相同标识（相同任务版本，相同任务定义，相同参数传入）的任务进行缓存，运行时若已经存在缓存过的任务时，不在重复执行，直接复用结果。 |
|
||||
| 描述 | 当前节点的功能描述。 |
|
||||
| 任务优先级 | worker线程数不足时，根据优先级从高到低依次执行任务，优先级一样时根据先到先得原则执行。 |
|
||||
| Worker分组 | 设置分组后，任务会被分配给worker组的机器机执行。若选择Default，则会随机选择一个worker执行。 |
|
||||
|
|
|
|||
|
|
@ -73,17 +73,14 @@ DolphinScheduler 在 启动DMS 任务后，会跟中DMS任务状态，直至DMS
|
|||
|
||||
## 环境配置
|
||||
|
||||
需要进行AWS的一些配置，修改`aws.yml`中的以下配置信息
|
||||
需要进行AWS的一些配置，修改`common.properties`中的以下配置信息
|
||||
|
||||
```yaml
|
||||
dms:
|
||||
# The AWS credentials provider type. support: AWSStaticCredentialsProvider, InstanceProfileCredentialsProvider
|
||||
# AWSStaticCredentialsProvider: use the access key and secret key to authenticate
|
||||
# InstanceProfileCredentialsProvider: use the IAM role to authenticate
|
||||
credentials.provider.type: AWSStaticCredentialsProvider
|
||||
access.key.id: <access.key.id>
|
||||
access.key.secret: <access.key.secret>
|
||||
region: <region>
|
||||
endpoint: <endpoint>
|
||||
# The AWS access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required
|
||||
resource.aws.access.key.id=<YOUR AWS ACCESS KEY>
|
||||
# The AWS secret access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required
|
||||
resource.aws.secret.access.key=<YOUR AWS SECRET KEY>
|
||||
# The AWS Region to use. if resource.storage.type=S3 or use EMR-Task, This configuration is required
|
||||
resource.aws.region=<AWS REGION>
|
||||
```
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,19 @@
|
|||
# Pigeon
|
||||
|
||||
Pigeon任务类型是通过调用远程websocket服务，实现远程任务的触发，状态、日志的获取，是 DolphinScheduler 通用远程 websocket 服务调用任务
|
||||
|
||||
## 创建任务
|
||||
|
||||
拖动工具栏中的<img src="../../../../img/pigeon.png" width="20"/>任务节点到画板中即能完成任务创建
|
||||
|
||||
## 任务参数
|
||||
|
||||
[//]: # (TODO: use the commented anchor below once our website template supports this syntax)
|
||||
[//]: # (- 默认参数说明请参考[DolphinScheduler任务参数附录](appendix.md#默认任务参数)`默认任务参数`一栏。)
|
||||
|
||||
- 默认参数说明请参考[DolphinScheduler任务参数附录](appendix.md)`默认任务参数`一栏。
|
||||
|
||||
| **任务参数** | **描述** |
|
||||
|----------|-------------------|
|
||||
| 目标任务名 | 输入Pigeon任务的目标任务名称 |
|
||||
|
||||
|
|
@ -33,17 +33,14 @@ DolphinScheduler SageMaker 组件的功能:
|
|||
|
||||
## 环境配置
|
||||
|
||||
需要进行AWS的一些配置，修改`aws.yml`中的以下配置信息
|
||||
需要进行AWS的一些配置，修改`common.properties`中的`xxxxx`为你的配置信息
|
||||
|
||||
```yaml
|
||||
sagemaker:
|
||||
# The AWS credentials provider type. support: AWSStaticCredentialsProvider, InstanceProfileCredentialsProvider
|
||||
# AWSStaticCredentialsProvider: use the access key and secret key to authenticate
|
||||
# InstanceProfileCredentialsProvider: use the IAM role to authenticate
|
||||
credentials.provider.type: AWSStaticCredentialsProvider
|
||||
access.key.id: <access.key.id>
|
||||
access.key.secret: <access.key.secret>
|
||||
region: <region>
|
||||
endpoint: <endpoint>
|
||||
# The AWS access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required
|
||||
resource.aws.access.key.id=<YOUR AWS ACCESS KEY>
|
||||
# The AWS secret access key. if resource.storage.type=S3 or use EMR-Task, This configuration is required
|
||||
resource.aws.secret.access.key=<YOUR AWS SECRET KEY>
|
||||
# The AWS Region to use. if resource.storage.type=S3 or use EMR-Task, This configuration is required
|
||||
resource.aws.region=<AWS REGION>
|
||||
```
|
||||
|
||||
|
|
|
|||
|
|
@ -26,6 +26,7 @@ SQL任务类型，用于连接数据库并执行相应SQL。
|
|||
- 默认采用`;\n`作为SQL分隔符,拆分成多段SQL语句执行。Hive的JDBC不支持一次执行多段SQL语句，请不要使用`;\n`。
|
||||
- sql参数：输入参数格式为key1=value1;key2=value2…
|
||||
- sql语句：SQL语句
|
||||
- UDF函数：对于HIVE类型的数据源，可以引用资源中心中创建的UDF函数，其他类型的数据源暂不支持UDF函数。
|
||||
- 自定义参数：SQL任务类型，而存储过程是自定义参数顺序，给方法设置值自定义参数类型和数据类型，同存储过程任务类型一样。区别在于SQL任务类型自定义参数会替换sql语句中${变量}。
|
||||
- 前置sql:前置sql在sql语句之前执行。
|
||||
- 后置sql:后置sql在sql语句之后执行。
|
||||
|
|
@ -56,4 +57,5 @@ SQL任务类型，用于连接数据库并执行相应SQL。
|
|||
## 注意事项
|
||||
|
||||
* 注意SQL类型的选择，如果是INSERT等操作需要选择非查询类型。
|
||||
* 为了兼容长会话情况,UDF函数的创建是通过CREATE OR REPLACE语句
|
||||
|
||||
|
|
|
|||
|
|
@ -24,8 +24,3 @@
|
|||
|
||||
* 在 /datasources/tables && /datasources/tableColumns 接口中添加了必选字段`database` [#14406](https://github.com/apache/dolphinscheduler/pull/14406)
|
||||
|
||||
## 3.3.0
|
||||
|
||||
* 从 `资源中心` 中移除了 `udf-manage` 功能 ([#16209])
|
||||
* 从 `任务插件` 中移除了 `Pigeon` 类型 ([#16218])
|
||||
|
||||
|
|
|
|||
Binary file not shown.
|
Before Width: | Height: | Size: 327 KiB After Width: | Height: | Size: 137 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 338 KiB After Width: | Height: | Size: 166 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 326 KiB After Width: | Height: | Size: 112 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 118 KiB After Width: | Height: | Size: 3.1 MiB |
Binary file not shown.
|
Before Width: | Height: | Size: 196 KiB After Width: | Height: | Size: 346 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 195 KiB After Width: | Height: | Size: 347 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 79 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 16 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 82 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 132 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 1.2 KiB |
|
|
@ -15,13 +15,12 @@
|
|||
~ See the License for the specific language governing permissions and
|
||||
~ limitations under the License.
|
||||
-->
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<parent>
|
||||
<groupId>org.apache.dolphinscheduler</groupId>
|
||||
<artifactId>dolphinscheduler-alert-plugins</artifactId>
|
||||
<version>dev-SNAPSHOT</version>
|
||||
<version>3.2.2</version>
|
||||
</parent>
|
||||
|
||||
<artifactId>dolphinscheduler-alert-aliyunVoice</artifactId>
|
||||
|
|
|
|||
|
|
@ -15,13 +15,12 @@
|
|||
~ See the License for the specific language governing permissions and
|
||||
~ limitations under the License.
|
||||
-->
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<parent>
|
||||
<groupId>org.apache.dolphinscheduler</groupId>
|
||||
<artifactId>dolphinscheduler-alert-plugins</artifactId>
|
||||
<version>dev-SNAPSHOT</version>
|
||||
<version>3.2.2</version>
|
||||
</parent>
|
||||
|
||||
<artifactId>dolphinscheduler-alert-all</artifactId>
|
||||
|
|
|
|||
|
|
@ -17,13 +17,12 @@
|
|||
~ specific language governing permissions and limitations
|
||||
~ under the License.
|
||||
-->
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<parent>
|
||||
<groupId>org.apache.dolphinscheduler</groupId>
|
||||
<artifactId>dolphinscheduler-alert-plugins</artifactId>
|
||||
<version>dev-SNAPSHOT</version>
|
||||
<version>3.2.2</version>
|
||||
</parent>
|
||||
<artifactId>dolphinscheduler-alert-api</artifactId>
|
||||
|
||||
|
|
|
|||
|
|
@ -15,13 +15,12 @@
|
|||
~ See the License for the specific language governing permissions and
|
||||
~ limitations under the License.
|
||||
-->
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<parent>
|
||||
<groupId>org.apache.dolphinscheduler</groupId>
|
||||
<artifactId>dolphinscheduler-alert-plugins</artifactId>
|
||||
<version>dev-SNAPSHOT</version>
|
||||
<version>3.2.2</version>
|
||||
</parent>
|
||||
<artifactId>dolphinscheduler-alert-dingtalk</artifactId>
|
||||
<packaging>jar</packaging>
|
||||
|
|
|
|||
|
|
@ -15,13 +15,12 @@
|
|||
~ See the License for the specific language governing permissions and
|
||||
~ limitations under the License.
|
||||
-->
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<parent>
|
||||
<groupId>org.apache.dolphinscheduler</groupId>
|
||||
<artifactId>dolphinscheduler-alert-plugins</artifactId>
|
||||
<version>dev-SNAPSHOT</version>
|
||||
<version>3.2.2</version>
|
||||
</parent>
|
||||
<artifactId>dolphinscheduler-alert-email</artifactId>
|
||||
<packaging>jar</packaging>
|
||||
|
|
|
|||
|
|
@ -15,13 +15,12 @@
|
|||
~ See the License for the specific language governing permissions and
|
||||
~ limitations under the License.
|
||||
-->
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<parent>
|
||||
<groupId>org.apache.dolphinscheduler</groupId>
|
||||
<artifactId>dolphinscheduler-alert-plugins</artifactId>
|
||||
<version>dev-SNAPSHOT</version>
|
||||
<version>3.2.2</version>
|
||||
</parent>
|
||||
<artifactId>dolphinscheduler-alert-feishu</artifactId>
|
||||
<packaging>jar</packaging>
|
||||
|
|
|
|||
|
|
@ -15,13 +15,12 @@
|
|||
~ See the License for the specific language governing permissions and
|
||||
~ limitations under the License.
|
||||
-->
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<parent>
|
||||
<groupId>org.apache.dolphinscheduler</groupId>
|
||||
<artifactId>dolphinscheduler-alert-plugins</artifactId>
|
||||
<version>dev-SNAPSHOT</version>
|
||||
<version>3.2.2</version>
|
||||
</parent>
|
||||
<artifactId>dolphinscheduler-alert-http</artifactId>
|
||||
<packaging>jar</packaging>
|
||||
|
|
|
|||
|
|
@ -15,13 +15,12 @@
|
|||
~ See the License for the specific language governing permissions and
|
||||
~ limitations under the License.
|
||||
-->
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<parent>
|
||||
<groupId>org.apache.dolphinscheduler</groupId>
|
||||
<artifactId>dolphinscheduler-alert-plugins</artifactId>
|
||||
<version>dev-SNAPSHOT</version>
|
||||
<version>3.2.2</version>
|
||||
</parent>
|
||||
<artifactId>dolphinscheduler-alert-pagerduty</artifactId>
|
||||
<packaging>jar</packaging>
|
||||
|
|
|
|||
|
|
@ -15,13 +15,12 @@
|
|||
~ See the License for the specific language governing permissions and
|
||||
~ limitations under the License.
|
||||
-->
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<parent>
|
||||
<groupId>org.apache.dolphinscheduler</groupId>
|
||||
<artifactId>dolphinscheduler-alert-plugins</artifactId>
|
||||
<version>dev-SNAPSHOT</version>
|
||||
<version>3.2.2</version>
|
||||
</parent>
|
||||
<artifactId>dolphinscheduler-alert-prometheus</artifactId>
|
||||
<packaging>jar</packaging>
|
||||
|
|
|
|||
|
|
@ -15,13 +15,12 @@
|
|||
~ See the License for the specific language governing permissions and
|
||||
~ limitations under the License.
|
||||
-->
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<parent>
|
||||
<groupId>org.apache.dolphinscheduler</groupId>
|
||||
<artifactId>dolphinscheduler-alert-plugins</artifactId>
|
||||
<version>dev-SNAPSHOT</version>
|
||||
<version>3.2.2</version>
|
||||
</parent>
|
||||
<artifactId>dolphinscheduler-alert-script</artifactId>
|
||||
<packaging>jar</packaging>
|
||||
|
|
|
|||
|
|
@ -15,13 +15,12 @@
|
|||
~ See the License for the specific language governing permissions and
|
||||
~ limitations under the License.
|
||||
-->
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<parent>
|
||||
<groupId>org.apache.dolphinscheduler</groupId>
|
||||
<artifactId>dolphinscheduler-alert-plugins</artifactId>
|
||||
<version>dev-SNAPSHOT</version>
|
||||
<version>3.2.2</version>
|
||||
</parent>
|
||||
<artifactId>dolphinscheduler-alert-slack</artifactId>
|
||||
<packaging>jar</packaging>
|
||||
|
|
|
|||
|
|
@ -15,13 +15,12 @@
|
|||
~ See the License for the specific language governing permissions and
|
||||
~ limitations under the License.
|
||||
-->
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<parent>
|
||||
<groupId>org.apache.dolphinscheduler</groupId>
|
||||
<artifactId>dolphinscheduler-alert-plugins</artifactId>
|
||||
<version>dev-SNAPSHOT</version>
|
||||
<version>3.2.2</version>
|
||||
</parent>
|
||||
<artifactId>dolphinscheduler-alert-telegram</artifactId>
|
||||
<packaging>jar</packaging>
|
||||
|
|
|
|||
|
|
@ -15,13 +15,12 @@
|
|||
~ See the License for the specific language governing permissions and
|
||||
~ limitations under the License.
|
||||
-->
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<parent>
|
||||
<groupId>org.apache.dolphinscheduler</groupId>
|
||||
<artifactId>dolphinscheduler-alert-plugins</artifactId>
|
||||
<version>dev-SNAPSHOT</version>
|
||||
<version>3.2.2</version>
|
||||
</parent>
|
||||
<artifactId>dolphinscheduler-alert-webexteams</artifactId>
|
||||
<packaging>jar</packaging>
|
||||
|
|
|
|||
|
|
@ -15,13 +15,12 @@
|
|||
~ See the License for the specific language governing permissions and
|
||||
~ limitations under the License.
|
||||
-->
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<parent>
|
||||
<groupId>org.apache.dolphinscheduler</groupId>
|
||||
<artifactId>dolphinscheduler-alert-plugins</artifactId>
|
||||
<version>dev-SNAPSHOT</version>
|
||||
<version>3.2.2</version>
|
||||
</parent>
|
||||
<artifactId>dolphinscheduler-alert-wechat</artifactId>
|
||||
<packaging>jar</packaging>
|
||||
|
|
|
|||
|
|
@ -15,13 +15,12 @@
|
|||
~ See the License for the specific language governing permissions and
|
||||
~ limitations under the License.
|
||||
-->
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<parent>
|
||||
<groupId>org.apache.dolphinscheduler</groupId>
|
||||
<artifactId>dolphinscheduler-alert</artifactId>
|
||||
<version>dev-SNAPSHOT</version>
|
||||
<version>3.2.2</version>
|
||||
</parent>
|
||||
<artifactId>dolphinscheduler-alert-plugins</artifactId>
|
||||
<packaging>pom</packaging>
|
||||
|
|
|
|||
|
|
@ -15,13 +15,12 @@
|
|||
~ See the License for the specific language governing permissions and
|
||||
~ limitations under the License.
|
||||
-->
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<parent>
|
||||
<groupId>org.apache.dolphinscheduler</groupId>
|
||||
<artifactId>dolphinscheduler-alert</artifactId>
|
||||
<version>dev-SNAPSHOT</version>
|
||||
<version>3.2.2</version>
|
||||
</parent>
|
||||
<artifactId>dolphinscheduler-alert-server</artifactId>
|
||||
<packaging>jar</packaging>
|
||||
|
|
|
|||
|
|
@ -56,13 +56,6 @@
|
|||
</includes>
|
||||
<outputDirectory>conf</outputDirectory>
|
||||
</fileSet>
|
||||
<fileSet>
|
||||
<directory>${basedir}/../dolphinscheduler-authentication/dolphinscheduler-aws-authentication/src/main/resources</directory>
|
||||
<includes>
|
||||
<include>**/*.yaml</include>
|
||||
</includes>
|
||||
<outputDirectory>conf</outputDirectory>
|
||||
</fileSet>
|
||||
</fileSets>
|
||||
<dependencySets>
|
||||
<dependencySet>
|
||||
|
|
|
|||
|
|
@ -24,7 +24,6 @@
|
|||
-XX:+PrintGCDetails
|
||||
-Xloggc:gc.log
|
||||
|
||||
-XX:-OmitStackTraceInFastThrow
|
||||
-XX:+ExitOnOutOfMemoryError
|
||||
-XX:+HeapDumpOnOutOfMemoryError
|
||||
-XX:HeapDumpPath=dump.hprof
|
||||
|
|
|
|||
|
|
@ -82,12 +82,12 @@ registry:
|
|||
namespace: dolphinscheduler
|
||||
connect-string: localhost:2181
|
||||
retry-policy:
|
||||
base-sleep-time: 1s
|
||||
max-sleep: 3s
|
||||
base-sleep-time: 60ms
|
||||
max-sleep: 300ms
|
||||
max-retries: 5
|
||||
session-timeout: 60s
|
||||
connection-timeout: 15s
|
||||
block-until-connected: 15s
|
||||
session-timeout: 30s
|
||||
connection-timeout: 9s
|
||||
block-until-connected: 600ms
|
||||
digest: ~
|
||||
|
||||
metrics:
|
||||
|
|
|
|||
|
|
@ -17,13 +17,12 @@
|
|||
~ specific language governing permissions and limitations
|
||||
~ under the License.
|
||||
-->
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<parent>
|
||||
<groupId>org.apache.dolphinscheduler</groupId>
|
||||
<artifactId>dolphinscheduler</artifactId>
|
||||
<version>dev-SNAPSHOT</version>
|
||||
<version>3.2.2</version>
|
||||
</parent>
|
||||
<artifactId>dolphinscheduler-alert</artifactId>
|
||||
<packaging>pom</packaging>
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue