natas_hw
/
dolphinscheduler0701
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'dev' into add_ldap_admin_group

This commit is contained in:
xiangzihao 2024-06-26 20:09:51 +08:00 committed by GitHub
parent 99756422c6 c7a0994fb0
commit 5397f1544b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
386 changed files with 9175 additions and 10798 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.asf.yaml
View File

@ -46,7 +46,8 @@ github:
- E2E
- Docs
- Frontend Build
# - "Mergeable: milestone-label-check"
- "Mergeable: milestone-label-check"
- "Title Validator"
required_pull_request_reviews:
dismiss_stale_reviews: true
required_approving_review_count: 2

45
.github/CODEOWNERS vendored
View File

@ -16,30 +16,39 @@
#
/.github/ @SbloodyS
/deploy/ @caishunfeng
/dolphinscheduler-alert/ @caishunfeng
/dolphinscheduler-e2e/ @SbloodyS
/dolphinscheduler-api-test/ @SbloodyS
/dolphinscheduler-registry/ @caishunfeng @ruanwenjun
/deploy/ @SbloodyS
/dolphinscheduler-alert/ @SbloodyS
/dolphinscheduler-api/ @caishunfeng @SbloodyS
/dolphinscheduler-dao/ @caishunfeng @SbloodyS
/dolphinscheduler-dao/src/main/resources/sql/ @EricGao888
/dolphinscheduler-common/ @caishunfeng
/dolphinscheduler-standalone-server/ @caishunfeng
/dolphinscheduler-datasource-plugin/ @caishunfeng
/dolphinscheduler-dist/ @caishunfeng
/dolphinscheduler-meter/ @caishunfeng @ruanwenjun @EricGao888
/dolphinscheduler-scheduler-plugin/ @caishunfeng
/dolphinscheduler-api-test/ @SbloodyS
/dolphinscheduler-authentication/ @ruanwenjun
/dolphinscheduler-bom/ @ruanwenjun
/dolphinscheduler-common/ @SbloodyS
/dolphinscheduler-dao/ @SbloodyS @ruanwenjun
/dolphinscheduler-dao-plugin/ @SbloodyS @ruanwenjun
/dolphinscheduler-data-quality/ @SbloodyS
/dolphinscheduler-datasource-plugin/ @SbloodyS
/dolphinscheduler-dist/ @SbloodyS
/dolphinscheduler-e2e/ @SbloodyS
/dolphinscheduler-extract/ @ruanwenjun
/dolphinscheduler-master/ @caishunfeng @SbloodyS @ruanwenjun
/dolphinscheduler-worker/ @caishunfeng @SbloodyS @ruanwenjun
/dolphinscheduler-meter/ @ruanwenjun @EricGao888
/dolphinscheduler-microbench/ @SbloodyS @ruanwenjun
/dolphinscheduler-registry/ @ruanwenjun
/dolphinscheduler-scheduler-plugin/ @ruanwenjun
/dolphinscheduler-service/ @caishunfeng
/dolphinscheduler-extract/ @caishunfeng @ruanwenjun
/dolphinscheduler-spi/ @caishunfeng
/dolphinscheduler-standalone-server/ @SbloodyS
/dolphinscheduler-storage-plugin/ @ruanwenjun @SbloodyS
/dolphinscheduler-task-plugin/ @caishunfeng @SbloodyS @zhuangchong
/dolphinscheduler-tools/ @caishunfeng @SbloodyS @EricGao888
/script/ @caishunfeng @SbloodyS @EricGao888
/dolphinscheduler-ui/ @songjianet @Amy0104
/docs/ @EricGao888
/dolphinscheduler-worker/ @caishunfeng @SbloodyS @ruanwenjun
/dolphinscheduler-yarn-aop/ @ruanwenjun
/dolphinscheduler-dao/src/main/resources/sql/ @EricGao888
/script/ @SbloodyS @EricGao888
/docs/ @EricGao888 @SbloodyS
/licenses/ @EricGao888
/images/ @EricGao888
/style/ @caishunfeng
/style/ @SbloodyS @EricGao888

3
.github/PULL_REQUEST_TEMPLATE.md vendored
View File

@ -31,4 +31,7 @@ This change added tests and can be verified as follows:
(or)
## Pull Request Notice
[Pull Request Notice](https://github.com/apache/dolphinscheduler/blob/dev/docs/docs/en/contribute/join/pull-request.md)
If your pull request contain incompatible change, you should also add it to `docs/docs/en/guide/upgrede/incompatible.md`

1
.github/actions/auto-assign-action vendored Submodule

@ -0,0 +1 @@
Subproject commit 288f36f8cea1dded676b9c4477a64ff91e50c104

22
.github/actions/auto-assign/auto-assign.yml vendored Normal file
View File

@ -0,0 +1,22 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Set addAssignees to 'author' to set the PR creator as the assignee.
addAssignees: author
# Enable it to run on drafts
runOnDraft: true

25
.github/actions/labeler/labeler.yml vendored
View File

@ -18,37 +18,54 @@
backend:
- 'dolphinscheduler-alert/**/*'
- 'dolphinscheduler-api/**/*'
- 'dolphinscheduler-authentication/**/*'
- 'dolphinscheduler-bom/**/*'
- 'dolphinscheduler-common/**/*'
- 'dolphinscheduler-dao/**/*'
- 'dolphinscheduler-dao-plugin/**/*'
- 'dolphinscheduler-data-quality/**/*'
- 'dolphinscheduler-datasource-plugin/**/*'
- 'dolphinscheduler-dist/**/*'
- 'dolphinscheduler-master/**/*'
- 'dolphinscheduler-registry/**/*'
- 'dolphinscheduler-extract/**/*'
- 'dolphinscheduler-master/**/*'
- 'dolphinscheduler-meter/**/*'
- 'dolphinscheduler-microbench/**/*'
- 'dolphinscheduler-registry/**/*'
- 'dolphinscheduler-scheduler-plugin/**/*'
- 'dolphinscheduler-service/**/*'
- 'dolphinscheduler-spi/**/*'
- 'dolphinscheduler-standalone-server/**/*'
- 'dolphinscheduler-storage-plugin/**/*'
- 'dolphinscheduler-task-plugin/**/*'
- 'dolphinscheduler-tools/**/*'
- 'dolphinscheduler-worker/**/*'
- 'dolphinscheduler-yarn-aop/**/*'
- 'script/**/*'
- '.github/workflows/backend.yml'
document:
- 'docs/**/*'
- '**/*.md'
CI&CD:
- any: ['.github/**/*']
docker:
- any: ['.deploy/**/*']
- any: ['deploy/docker/**/*']
UI:
- any: ['dolphinscheduler-ui/**/*']
- any: ['.github/workflows/frontend.yml']
e2e:
- any: ['dolphinscheduler-e2e/**/*']
- any: ['dolphinscheduler-api-test/**/*']
- any: ['.github/workflows/e2e.yml']
- any: ['.github/workflows/api-test.yml']
- any: ['.github/workflows/e2e-k8s.yml']
test:
- any: ['dolphinscheduler-api-test/**/*']
- any: ['**/test/**/*']
kubernetes:
- any: ['deploy/kubernetes/**/*']

72
.github/boring-cyborg.yml vendored Normal file
View File

@ -0,0 +1,72 @@
################################################################################
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Details: https://github.com/apps/boring-cyborg
###### IssueLink Adder #################################################################################################
# Insert Issue (Jira/Github etc) link in PR description based on the Issue ID in PR title.
insertIssueLinkInPrDescription:
# specify the placeholder for the issue link that should be present in the description
descriptionIssuePlaceholderRegexp: "^fix: (.*)$"
matchers:
# you can have several matches - for different types of issues
# only the first matching entry is replaced
jiraIssueMatch:
# specify the regexp of issue id that you can find in the title of the PR
# the match groups can be used to build the issue id (${1}, ${2}, etc.).
titleIssueIdRegexp: \[(Feature|Fix|Improvement|Test|E2E|Doc-[0-9]+)\]
# the issue link to be added. ${1}, ${2} ... are replaced with the match groups from the
# title match (remember to use quotes)
descriptionIssueLink: "[${1}](https://github.com/apache/dolphinschedule/${1}/)"
###### Title Validator #################################################################################################
# Verifies if commit/PR titles match the regexp specified
verifyTitles:
# Regular expression that should be matched by titles of commits or PR
titleRegexp: ^\[DSIP-[0-9]+\].*$|^\[Feature-[0-9]+\].*$|^\[Fix-[0-9]+\].*$|^\[Doc-[0-9]+\].*$|^\[Improvement-[0-9]+\].*$|^\[Test-[0-9]+\].*$|^\[E2E-[0-9]+\].*$|^\[CI].*$|^\[Chore].*$
# If set to true, it will always check the PR title (as opposed to the individual commits).
alwaysUsePrTitle: true
# If set to true, it will only check the commit in case there is a single commit.
# In case of multiple commits it will check PR title.
# This reflects the standard behaviour of Github that for `Squash & Merge` GitHub
# uses the PR title rather than commit messages for the squashed commit ¯\_(ツ)_/¯
# For single-commit PRs it takes the squashed commit message from the commit as expected.
#
# If set to false it will check all commit messages. This is useful when you do not squash commits at merge.
validateEitherPrOrSingleCommitTitle: true
# The title the GitHub status should appear from.
statusTitle: "Title Validator"
# A custom message to be displayed when the title passes validation.
successMessage: "Title Validation successful!"
# A custom message to be displayed when the title fails validation.
# Allows insertion of ${type} (commit/PR), ${title} (the title validated) and ${regex} (the titleRegexp above).
failureMessage: "Wrong ${type} title: ${title}"
# Various Flags to control behaviour of the "Labeler"
labelerFlags:
# If this flag is changed to 'false', labels would only be added when the PR is first created
# and not when existing PR is updated.
# The default is 'true' which means the labels would be added when PR is updated even if they
# were removed by the user
labelOnPRUpdates: true
# Comment to be posted to welcome users when they open their first PR
firstPRWelcomeComment: >
Thanks for opening this pull request! Please check out our contributing guidelines. (https://github.com/apache/dolphinscheduler/blob/dev/docs/docs/en/contribute/join/pull-request.md)
# Comment to be posted to congratulate user on their first merged PR
firstPRMergeComment: >
Awesome work, congrats on your first merged pull request!

87
.github/workflows/mergeable.yml vendored
View File

@ -14,65 +14,50 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
---
#version: 2
on:
pull_request:
types:
- opened
- reopened
- synchronize
- labeled
- unlabeled
- milestoned
- demilestoned
name: "Mergeable"
jobs:
result:
name: "Mergeable: milestone-label-check"
runs-on: ubuntu-latest
timeout-minutes: 30
timeout-minutes: 10
steps:
- name: Status
run: |
echo "Temporary skipping this check"
#mergeable:
# # we can not use `pull_request.*` which including event `pull_request.labeled`, according to https://github.com/mergeability/mergeable/issues/643,
# # otherwise mergeable will keep add or remove label endless, we just need this CI act like the default behavior as
# # GitHub action workflow `pull_requests` https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request like,
# # which only trigger runs when a pull_request event's activity type is opened, synchronize, or reopened
# - when: pull_request.opened, pull_request.reopened, pull_request.synchronize
# name: sync-sql-ddl
# validate:
# # Sql files must change synchronize
# - do: dependent
# files:
# - 'dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_h2.sql'
# - 'dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql'
# - 'dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql'
# message: 'Sql files not change synchronize'
# # Add labels 'sql not sync' and comment to reviewers if Sql files not change synchronize
# fail:
# - do: comment
# payload:
# body: >
# :warning: This PR do not change database DDL synchronize.
# leave_old_comment: false
# - do: labels
# add: 'sql not sync'
# # Remove labels 'sql not sync' if pass
# pass:
# - do: labels
# delete: 'sql not sync'
#
# - when: pull_request.*
# name: milestone-label-check
# validate:
# - do: milestone
# no_empty:
# enabled: false # Cannot be empty when true.
# message: 'Milestone is required and cannot be empty.'
# - do: label
# and:
# - must_include:
# regex: 'feature|bug|improvement|document|chore|revert'
# message: 'Label must include one of the following: `feature`, `bug`, `improvement`, `document`, `chore`, `revert`'
# - must_include:
# regex: 'ready-to-merge'
# message: 'Please check if there are PRs that already have a `ready-to-merge` label and can be merged, if exists please merge them first.'
- name: Check milestone and labels
uses: actions/github-script@v7
with:
script: |
const { data } = await github.request("GET /repos/{owner}/{repo}/pulls/{pr}", {
owner: context.repo.owner,
repo: context.repo.repo,
pr: context.payload.pull_request.number
});
if (data.milestone) {
core.info(`This pull request has a milestone: ${data.milestone.title}`);
} else {
core.setFailed(`A maintainer needs to set the milestone for this pull request.`);
}
let labeledFlag = false;
for (const label of data.labels) {
if (['feature', 'bug', 'improvement', 'document', 'chore', 'DSIP', 'CI&CD', 'revert'].includes(label.name)) {
core.info(`This pull request has a valid label: ${label.name}`);
labeledFlag = true;
break;
}
}
if (!labeledFlag) {
core.setFailed(`A maintainer needs to set a valid label for this pull request.`);
}

12
.github/workflows/pull-request-robot.yml → .github/workflows/pull-request-target-robot.yml vendored
View File

@ -15,20 +15,21 @@
# limitations under the License.
#
name: "pull-request-robot"
name: "pull-request-target-robot"
on:
pull_request_target:
jobs:
labelRobot:
pullRequestTargetRobot:
permissions:
contents: read
pull-requests: write
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
- name: "Checkout ${{ github.ref }}"
uses: actions/checkout@v2
uses: actions/checkout@v4
with:
persist-credentials: false
submodules: true
@ -39,3 +40,8 @@ jobs:
repo-token: "${{ secrets.GITHUB_TOKEN }}"
configuration-path: .github/actions/labeler/labeler.yml
sync-labels: true
- name: "Auto assign in pull request"
uses: ./.github/actions/auto-assign-action
with:
configuration-path: .github/actions/auto-assign/auto-assign.yml

1
.gitignore vendored
View File

@ -7,6 +7,7 @@
.target
.idea/
!.idea/icon.png
!.idea/vcs.xml
.run/
target/
dist/

4
.gitmodules vendored
View File

@ -21,3 +21,7 @@
[submodule ".github/actions/translate-on-issue"]
path = .github/actions/translate-on-issue
url = https://github.com/xingchun-chen/translation-helper
[submodule ".github/actions/auto-assign-action"]
path = .github/actions/auto-assign-action
url = https://github.com/kentaro-m/auto-assign-action.git
branch = 288f36f

16
.idea/vcs.xml Normal file
View File

@ -0,0 +1,16 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="IssueNavigationConfiguration">
<option name="links">
<list>
<IssueNavigationLink>
<option name="issueRegexp" value="#(\d+)" />
<option name="linkRegexp" value="https://github.com/apache/dolphinscheduler/issues/$1" />
</IssueNavigationLink>
</list>
</option>
</component>
<component name="VcsDirectoryMappings">
<mapping directory="" vcs="Git" />
</component>
</project>

1
.licenserc.yaml
View File

@ -51,5 +51,6 @@ header:
- '**/banner.txt'
- '.terraform.lock.hcl'
- deploy/kubernetes/dolphinscheduler/README.md.gotmpl
- .idea/vcs.xml
comment: on-failure

9
deploy/kubernetes/dolphinscheduler/README.md
View File

@ -14,6 +14,8 @@ Please refer to the [Quick Start in Kubernetes](../../../docs/docs/en/guide/inst
|-----|------|---------|-------------|
| alert.affinity | object | `{}` | Affinity is a group of affinity scheduling rules. If specified, the pod's scheduling constraints. More info: [node-affinity](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#node-affinity) |
| alert.annotations | object | `{}` | You can use annotations to attach arbitrary non-identifying metadata to objects. Clients such as tools and libraries can retrieve this metadata. |
| alert.customizedConfig | object | `{}` | configure aligned with https://github.com/apache/dolphinscheduler/blob/dev/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/resources/application.yaml |
| alert.enableCustomizedConfig | bool | `false` | enable configure custom config |
| alert.enabled | bool | `true` | Enable or disable the Alert-Server component |
| alert.env.JAVA_OPTS | string | `"-Xms512m -Xmx512m -Xmn256m"` | The jvm options for alert server |
| alert.livenessProbe | object | `{"enabled":true,"failureThreshold":"3","initialDelaySeconds":"30","periodSeconds":"30","successThreshold":"1","timeoutSeconds":"5"}` | Periodic probe of container liveness. Container will be restarted if the probe fails. More info: [container-probes](https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#container-probes) |
@ -52,6 +54,8 @@ Please refer to the [Quick Start in Kubernetes](../../../docs/docs/en/guide/inst
| alert.tolerations | list | `[]` | Tolerations are appended (excluding duplicates) to pods running with this RuntimeClass during admission, effectively unioning the set of nodes tolerated by the pod and the RuntimeClass. |
| api.affinity | object | `{}` | Affinity is a group of affinity scheduling rules. If specified, the pod's scheduling constraints. More info: [node-affinity](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#node-affinity) |
| api.annotations | object | `{}` | You can use annotations to attach arbitrary non-identifying metadata to objects. Clients such as tools and libraries can retrieve this metadata. |
| api.customizedConfig | object | `{}` | configure aligned with https://github.com/apache/dolphinscheduler/blob/dev/dolphinscheduler-api/src/main/resources/application.yaml |
| api.enableCustomizedConfig | bool | `false` | enable configure custom config |
| api.enabled | bool | `true` | Enable or disable the API-Server component |
| api.env.JAVA_OPTS | string | `"-Xms512m -Xmx512m -Xmn256m"` | The jvm options for api server |
| api.livenessProbe | object | `{"enabled":true,"failureThreshold":"3","initialDelaySeconds":"30","periodSeconds":"30","successThreshold":"1","timeoutSeconds":"5"}` | Periodic probe of container liveness. Container will be restarted if the probe fails. More info: [container-probes](https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#container-probes) |
@ -191,6 +195,8 @@ Please refer to the [Quick Start in Kubernetes](../../../docs/docs/en/guide/inst
| initImage.pullPolicy | string | `"IfNotPresent"` | Image pull policy. Options: Always, Never, IfNotPresent |
| master.affinity | object | `{}` | Affinity is a group of affinity scheduling rules. If specified, the pod's scheduling constraints. More info: [node-affinity](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#node-affinity) |
| master.annotations | object | `{}` | You can use annotations to attach arbitrary non-identifying metadata to objects. Clients such as tools and libraries can retrieve this metadata. |
| master.customizedConfig | object | `{}` | configure aligned with https://github.com/apache/dolphinscheduler/blob/dev/dolphinscheduler-master/src/main/resources/application.yaml |
| master.enableCustomizedConfig | bool | `false` | enable configure custom config |
| master.enabled | bool | `true` | Enable or disable the Master component |
| master.env.JAVA_OPTS | string | `"-Xms1g -Xmx1g -Xmn512m"` | The jvm options for master server |
| master.env.MASTER_DISPATCH_TASK_NUM | string | `"3"` | Master dispatch task number per batch |
@ -297,6 +303,8 @@ Please refer to the [Quick Start in Kubernetes](../../../docs/docs/en/guide/inst
| timezone | string | `"Asia/Shanghai"` | World time and date for cities in all time zones |
| worker.affinity | object | `{}` | Affinity is a group of affinity scheduling rules. If specified, the pod's scheduling constraints. More info: [node-affinity](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#node-affinity) |
| worker.annotations | object | `{}` | You can use annotations to attach arbitrary non-identifying metadata to objects. Clients such as tools and libraries can retrieve this metadata. |
| worker.customizedConfig | object | `{}` | configure aligned with https://github.com/apache/dolphinscheduler/blob/dev/dolphinscheduler-worker/src/main/resources/application.yaml |
| worker.enableCustomizedConfig | bool | `false` | enable configure custom config |
| worker.enabled | bool | `true` | Enable or disable the Worker component |
| worker.env.DEFAULT_TENANT_ENABLED | bool | `false` | If set true, will use worker bootstrap user as the tenant to execute task when the tenant is `default`; |
| worker.env.WORKER_EXEC_THREADS | string | `"100"` | Worker execute thread number to limit task instances |
@ -315,7 +323,6 @@ Please refer to the [Quick Start in Kubernetes](../../../docs/docs/en/guide/inst
| worker.keda.minReplicaCount | int | `0` | Minimum number of workers created by keda |
| worker.keda.namespaceLabels | object | `{}` | Keda namespace labels |
| worker.keda.pollingInterval | int | `5` | How often KEDA polls the DolphinScheduler DB to report new scale requests to the HPA |
| worker.livenessProbe | object | `{"enabled":true,"failureThreshold":"3","initialDelaySeconds":"30","periodSeconds":"30","successThreshold":"1","timeoutSeconds":"5"}` | Periodic probe of container liveness. Container will be restarted if the probe fails. More info: [container-probes](https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#container-probes) |
| worker.livenessProbe.enabled | bool | `true` | Turn on and off liveness probe |
| worker.livenessProbe.failureThreshold | string | `"3"` | Minimum consecutive failures for the probe |
| worker.livenessProbe.initialDelaySeconds | string | `"30"` | Delay before liveness probe is initiated |

97
deploy/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-alert.yaml
View File

@ -14,7 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
{{- if and .Values.alert.enabled }}
{{- if .Values.alert.enableCustomizedConfig }}
apiVersion: v1
kind: ConfigMap
metadata:
@ -23,93 +23,8 @@ metadata:
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-alert
{{- include "dolphinscheduler.alert.labels" . | nindent 4 }}
data:
application.yaml: |
spring:
profiles:
active: {{ .Values.datasource.profile }}
jackson:
time-zone: UTC
date-format: "yyyy-MM-dd HH:mm:ss"
banner:
charset: UTF-8
datasource:
driver-class-name: org.postgresql.Driver
url: jdbc:postgresql://127.0.0.1:5432/dolphinscheduler
username: root
password: root
hikari:
connection-test-query: select 1
pool-name: DolphinScheduler
# Mybatis-plus configuration, you don't need to change it
mybatis-plus:
mapper-locations: classpath:org/apache/dolphinscheduler/dao/mapper/*Mapper.xml
type-aliases-package: org.apache.dolphinscheduler.dao.entity
configuration:
cache-enabled: false
call-setters-on-nulls: true
map-underscore-to-camel-case: true
jdbc-type-for-null: NULL
global-config:
db-config:
id-type: auto
banner: false
server:
port: 50053
management:
endpoints:
web:
exposure:
include: health,metrics,prometheus
endpoint:
health:
enabled: true
show-details: always
health:
db:
enabled: true
defaults:
enabled: false
metrics:
tags:
application: ${spring.application.name}
alert:
port: 50052
# Mark each alert of alert server if late after x milliseconds as failed.
# Define value is (0 = infinite), and alert server would be waiting alert result.
wait-timeout: 0
max-heartbeat-interval: 60s
query_alert_threshold: 100
registry:
type: zookeeper
zookeeper:
namespace: dolphinscheduler
connect-string: localhost:2181
retry-policy:
base-sleep-time: 60ms
max-sleep: 300ms
max-retries: 5
session-timeout: 30s
connection-timeout: 9s
block-until-connected: 600ms
digest: ~
metrics:
enabled: true
# Override by profile
---
spring:
config:
activate:
on-profile: mysql
datasource:
driver-class-name: com.mysql.cj.jdbc.Driver
url: jdbc:mysql://127.0.0.1:3306/dolphinscheduler
username: root
password: root
{{- end }}
{{- range $path, $config := .Values.alert.customizedConfig }}
{{ $path }}: |
{{ $config | indent 4 -}}
{{- end -}}
{{- end -}}

240
deploy/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-api.yaml
View File

@ -14,7 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
{{- if and .Values.api.enabled }}
{{- if .Values.api.enableCustomizedConfig }}
apiVersion: v1
kind: ConfigMap
metadata:
@ -23,236 +23,8 @@ metadata:
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-api
{{- include "dolphinscheduler.api.labels" . | nindent 4 }}
data:
application.yaml: |
server:
port: 12345
servlet:
session:
timeout: 120m
context-path: /dolphinscheduler/
compression:
enabled: true
mime-types: text/html,text/xml,text/plain,text/css,text/javascript,application/javascript,application/json,application/xml
jetty:
max-http-form-post-size: 5000000
accesslog:
enabled: true
custom-format: '%{client}a - %u %t "%r" %s %O %{ms}Tms'
spring:
profiles:
active: {{ .Values.datasource.profile }}
banner:
charset: UTF-8
jackson:
time-zone: UTC
date-format: "yyyy-MM-dd HH:mm:ss"
servlet:
multipart:
max-file-size: 1024MB
max-request-size: 1024MB
messages:
basename: i18n/messages
datasource:
driver-class-name: org.postgresql.Driver
url: jdbc:postgresql://127.0.0.1:5432/dolphinscheduler
username: root
password: root
hikari:
connection-test-query: select 1
pool-name: DolphinScheduler
quartz:
auto-startup: false
job-store-type: jdbc
jdbc:
initialize-schema: never
properties:
org.quartz.jobStore.isClustered: true
org.quartz.jobStore.class: org.springframework.scheduling.quartz.LocalDataSourceJobStore
org.quartz.scheduler.instanceId: AUTO
org.quartz.jobStore.tablePrefix: QRTZ_
org.quartz.jobStore.acquireTriggersWithinLock: true
org.quartz.scheduler.instanceName: DolphinScheduler
org.quartz.threadPool.class: org.apache.dolphinscheduler.scheduler.quartz.QuartzZeroSizeThreadPool
org.quartz.jobStore.useProperties: false
org.quartz.jobStore.misfireThreshold: 60000
org.quartz.scheduler.makeSchedulerThreadDaemon: true
org.quartz.jobStore.driverDelegateClass: org.quartz.impl.jdbcjobstore.PostgreSQLDelegate
org.quartz.jobStore.clusterCheckinInterval: 5000
org.quartz.scheduler.batchTriggerAcquisitionMaxCount: 1
mvc:
pathmatch:
matching-strategy: ANT_PATH_MATCHER
static-path-pattern: /static/**
springdoc:
swagger-ui:
path: /swagger-ui.html
packages-to-scan: org.apache.dolphinscheduler.api
# Mybatis-plus configuration, you don't need to change it
mybatis-plus:
mapper-locations: classpath:org/apache/dolphinscheduler/dao/mapper/*Mapper.xml
type-aliases-package: org.apache.dolphinscheduler.dao.entity
configuration:
cache-enabled: false
call-setters-on-nulls: true
map-underscore-to-camel-case: true
jdbc-type-for-null: NULL
global-config:
db-config:
id-type: auto
banner: false
management:
endpoints:
web:
exposure:
include: health,metrics,prometheus
endpoint:
health:
enabled: true
show-details: always
health:
db:
enabled: true
defaults:
enabled: false
metrics:
tags:
application: ${spring.application.name}
registry:
type: zookeeper
zookeeper:
namespace: dolphinscheduler
connect-string: localhost:2181
retry-policy:
base-sleep-time: 60ms
max-sleep: 300ms
max-retries: 5
session-timeout: 60s
connection-timeout: 15s
block-until-connected: 15s
digest: ~
api:
audit-enable: false
# Traffic control, if you turn on this config, the maximum number of request/s will be limited.
# global max request number per second
# default tenant-level max request number
traffic-control:
global-switch: false
max-global-qps-rate: 300
tenant-switch: false
default-tenant-qps-rate: 10
#customize-tenant-qps-rate:
# eg.
#tenant1: 11
#tenant2: 20
python-gateway:
# Weather enable python gateway server or not. The default value is false.
enabled: false
# Authentication token for connection from python api to python gateway server. Should be changed the default value
# when you deploy in public network.
auth-token: jwUDzpLsNKEFER4*a8gruBH_GsAurNxU7A@Xc
# The address of Python gateway server start. Set its value to `0.0.0.0` if your Python API run in different
# between Python gateway server. It could be be specific to other address like `127.0.0.1` or `localhost`
gateway-server-address: 0.0.0.0
# The port of Python gateway server start. Define which port you could connect to Python gateway server from
# Python API side.
gateway-server-port: 25333
# The address of Python callback client.
python-address: 127.0.0.1
# The port of Python callback client.
python-port: 25334
# Close connection of socket server if no other request accept after x milliseconds. Define value is (0 = infinite),
# and socket server would never close even though no requests accept
connect-timeout: 0
# Close each active connection of socket server if python program not active after x milliseconds. Define value is
# (0 = infinite), and socket server would never close even though no requests accept
read-timeout: 0
metrics:
enabled: true
security:
authentication:
# Authentication types (supported types: PASSWORD,LDAP,CASDOOR_SSO)
type: PASSWORD
# IF you set type `LDAP`, below config will be effective
ldap:
# ldap server config
urls: ldap://ldap.forumsys.com:389/
base-dn: dc=example,dc=com
username: cn=read-only-admin,dc=example,dc=com
password: password
user:
# admin userId when you use LDAP login
admin: read-only-admin
identity-attribute: uid
email-attribute: mail
# action when ldap user is not exist (supported types: CREATE,DENY)
not-exist-action: CREATE
ssl:
enable: false
# jks file absolute path && password
trust-store: "/ldapkeystore.jks"
trust-store-password: "password"
casdoor:
user:
admin: ""
oauth2:
enable: false
provider:
github:
authorizationUri: ""
redirectUri: ""
clientId: ""
clientSecret: ""
tokenUri: ""
userInfoUri: ""
callbackUrl: ""
iconUri: ""
provider: github
google:
authorizationUri: ""
redirectUri: ""
clientId: ""
clientSecret: ""
tokenUri: ""
userInfoUri: ""
callbackUrl: ""
iconUri: ""
provider: google
casdoor:
# Your Casdoor server url
endpoint: ""
client-id: ""
client-secret: ""
# The certificate may be multi-line, you can use `|-` for ease
certificate: ""
# Your organization name added in Casdoor
organization-name: ""
# Your application name added in Casdoor
application-name: ""
# Doplhinscheduler login url
redirect-url: ""
# Override by profile
---
spring:
config:
activate:
on-profile: mysql
datasource:
driver-class-name: com.mysql.cj.jdbc.Driver
url: jdbc:mysql://127.0.0.1:3306/dolphinscheduler
username: root
password: root
quartz:
properties:
org.quartz.jobStore.driverDelegateClass: org.quartz.impl.jdbcjobstore.StdJDBCDelegate
{{- end }}
{{- range $path, $config := .Values.api.customizedConfig }}
{{ $path }}: |
{{ $config | indent 4 -}}
{{- end -}}
{{- end -}}

155
deploy/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-master.yaml
View File

@ -14,7 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
{{- if and .Values.master.enabled }}
{{- if .Values.master.enableCustomizedConfig }}
apiVersion: v1
kind: ConfigMap
metadata:
@ -23,151 +23,8 @@ metadata:
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-master
{{- include "dolphinscheduler.master.labels" . | nindent 4 }}
data:
application.yaml: |
spring:
profiles:
active: {{ .Values.datasource.profile }}
banner:
charset: UTF-8
jackson:
time-zone: UTC
date-format: "yyyy-MM-dd HH:mm:ss"
datasource:
driver-class-name: org.postgresql.Driver
url: jdbc:postgresql://127.0.0.1:5432/dolphinscheduler
username: root
password: root
hikari:
connection-test-query: select 1
pool-name: DolphinScheduler
quartz:
job-store-type: jdbc
jdbc:
initialize-schema: never
properties:
org.quartz.threadPool.threadPriority: 5
org.quartz.jobStore.isClustered: true
org.quartz.jobStore.class: org.springframework.scheduling.quartz.LocalDataSourceJobStore
org.quartz.scheduler.instanceId: AUTO
org.quartz.jobStore.tablePrefix: QRTZ_
org.quartz.jobStore.acquireTriggersWithinLock: true
org.quartz.scheduler.instanceName: DolphinScheduler
org.quartz.threadPool.class: org.quartz.simpl.SimpleThreadPool
org.quartz.jobStore.useProperties: false
org.quartz.threadPool.makeThreadsDaemons: true
org.quartz.threadPool.threadCount: 25
org.quartz.jobStore.misfireThreshold: 60000
org.quartz.scheduler.batchTriggerAcquisitionMaxCount: 1
org.quartz.scheduler.makeSchedulerThreadDaemon: true
org.quartz.jobStore.driverDelegateClass: org.quartz.impl.jdbcjobstore.PostgreSQLDelegate
org.quartz.jobStore.clusterCheckinInterval: 5000
# Mybatis-plus configuration, you don't need to change it
mybatis-plus:
mapper-locations: classpath:org/apache/dolphinscheduler/dao/mapper/*Mapper.xml
type-aliases-package: org.apache.dolphinscheduler.dao.entity
configuration:
cache-enabled: false
call-setters-on-nulls: true
map-underscore-to-camel-case: true
jdbc-type-for-null: NULL
global-config:
db-config:
id-type: auto
banner: false
registry:
type: zookeeper
zookeeper:
namespace: dolphinscheduler
connect-string: localhost:2181
retry-policy:
base-sleep-time: 60ms
max-sleep: 300ms
max-retries: 5
session-timeout: 30s
connection-timeout: 9s
block-until-connected: 600ms
digest: ~
master:
listen-port: 5678
# master fetch command num
fetch-command-num: 10
# master prepare execute thread number to limit handle commands in parallel
pre-exec-threads: 10
# master execute thread number to limit process instances in parallel
exec-threads: 100
# master dispatch task number per batch, if all the tasks dispatch failed in a batch, will sleep 1s.
dispatch-task-number: 3
# master host selector to select a suitable worker, default value: LowerWeight. Optional values include random, round_robin, lower_weight
host-selector: lower_weight
# master heartbeat interval
max-heartbeat-interval: 10s
# master commit task retry times
task-commit-retry-times: 5
# master commit task interval
task-commit-interval: 1s
state-wheel-interval: 5s
server-load-protection:
# If set true, will open master overload protection
enabled: true
# Master max system cpu usage, when the master's system cpu usage is smaller then this value, master server can execute workflow.
max-system-cpu-usage-percentage-thresholds: 0.7
# Master max jvm cpu usage, when the master's jvm cpu usage is smaller then this value, master server can execute workflow.
max-jvm-cpu-usage-percentage-thresholds: 0.7
# Master max System memory usage , when the master's system memory usage is smaller then this value, master server can execute workflow.
max-system-memory-usage-percentage-thresholds: 0.7
# Master max disk usage , when the master's disk usage is smaller then this value, master server can execute workflow.
max-disk-usage-percentage-thresholds: 0.7
# failover interval, the unit is minute
failover-interval: 10m
# kill yarn / k8s application when failover taskInstance, default true
kill-application-when-task-failover: true
registry-disconnect-strategy:
# The disconnect strategy: stop, waiting
strategy: waiting
# The max waiting time to reconnect to registry if you set the strategy to waiting
max-waiting-time: 100s
worker-group-refresh-interval: 10s
server:
port: 5679
management:
endpoints:
web:
exposure:
include: health,metrics,prometheus
endpoint:
health:
enabled: true
show-details: always
health:
db:
enabled: true
defaults:
enabled: false
metrics:
tags:
application: ${spring.application.name}
metrics:
enabled: true
# Override by profile
---
spring:
config:
activate:
on-profile: mysql
datasource:
driver-class-name: com.mysql.cj.jdbc.Driver
url: jdbc:mysql://127.0.0.1:3306/dolphinscheduler
username: root
password: root
quartz:
properties:
org.quartz.jobStore.driverDelegateClass: org.quartz.impl.jdbcjobstore.StdJDBCDelegate
{{- end }}
{{- range $path, $config := .Values.master.customizedConfig }}
{{ $path }}: |
{{ $config | indent 4 -}}
{{- end -}}
{{- end -}}

90
deploy/kubernetes/dolphinscheduler/templates/configmap-dolphinscheduler-worker.yaml
View File

@ -14,7 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
{{- if and .Values.worker.enabled }}
{{- if .Values.worker.enableCustomizedConfig }}
apiVersion: v1
kind: ConfigMap
metadata:
@ -23,86 +23,8 @@ metadata:
app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-worker
{{- include "dolphinscheduler.worker.labels" . | nindent 4 }}
data:
application.yaml: |
spring:
banner:
charset: UTF-8
jackson:
time-zone: UTC
date-format: "yyyy-MM-dd HH:mm:ss"
autoconfigure:
exclude:
- org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration
registry:
type: zookeeper
zookeeper:
namespace: dolphinscheduler
connect-string: localhost:2181
retry-policy:
base-sleep-time: 60ms
max-sleep: 300ms
max-retries: 5
session-timeout: 30s
connection-timeout: 9s
block-until-connected: 600ms
digest: ~
worker:
# worker listener port
listen-port: 1234
# worker execute thread number to limit task instances in parallel
exec-threads: 100
# worker heartbeat interval
max-heartbeat-interval: 10s
# worker host weight to dispatch tasks, default value 100
host-weight: 100
server-load-protection:
# If set true, will open worker overload protection
enabled: true
# Worker max system cpu usage, when the worker's system cpu usage is smaller then this value, worker server can be dispatched tasks.
max-system-cpu-usage-percentage-thresholds: 0.7
# Worker max jvm cpu usage, when the worker's jvm cpu usage is smaller then this value, worker server can be dispatched tasks.
max-jvm-cpu-usage-percentage-thresholds: 0.7
# Worker max System memory usage , when the master's system memory usage is smaller then this value, master server can execute workflow.
max-system-memory-usage-percentage-thresholds: 0.7
# Worker max disk usage , when the worker's disk usage is smaller then this value, worker server can be dispatched tasks.
max-disk-usage-percentage-thresholds: 0.7
registry-disconnect-strategy:
# The disconnect strategy: stop, waiting
strategy: waiting
# The max waiting time to reconnect to registry if you set the strategy to waiting
max-waiting-time: 100s
task-execute-threads-full-policy: REJECT
tenant-config:
# tenant corresponds to the user of the system, which is used by the worker to submit the job. If system does not have this user, it will be automatically created after the parameter worker.tenant.auto.create is true.
auto-create-tenant-enabled: true
# Scenes to be used for distributed users. For example, users created by FreeIpa are stored in LDAP. This parameter only applies to Linux, When this parameter is true, auto-create-tenant-enabled has no effect and will not automatically create tenants.
distributed-tenant-enabled: false
# If set true, will use worker bootstrap user as the tenant to execute task when the tenant is `default`.
default-tenant-enabled: false
server:
port: 1235
management:
endpoints:
web:
exposure:
include: health,metrics,prometheus
endpoint:
health:
enabled: true
show-details: always
health:
db:
enabled: true
defaults:
enabled: false
metrics:
tags:
application: ${spring.application.name}
metrics:
enabled: true
{{- end }}
{{- range $path, $config := .Values.worker.customizedConfig }}
{{ $path }}: |
{{ $config | indent 4 -}}
{{- end -}}
{{- end -}}

4
deploy/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-alert.yaml
View File

@ -115,9 +115,11 @@ spec:
- name: config-volume
mountPath: /opt/dolphinscheduler/conf/common.properties
subPath: common.properties
{{- if .Values.alert.enableCustomizedConfig }}
- name: alert-config-volume
mountPath: /opt/dolphinscheduler/conf/application.yaml
subPath: application.yaml
{{- end }}
volumes:
- name: {{ include "dolphinscheduler.fullname" . }}-alert
{{- if .Values.alert.persistentVolumeClaim.enabled }}
@ -129,7 +131,9 @@ spec:
- name: config-volume
configMap:
name: {{ include "dolphinscheduler.fullname" . }}-configs
{{- if .Values.alert.enableCustomizedConfig }}
- name: alert-config-volume
configMap:
name: {{ include "dolphinscheduler.fullname" . }}-alert
{{- end }}
{{- end }}

4
deploy/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml
View File

@ -121,9 +121,11 @@ spec:
mountPath: /opt/dolphinscheduler/conf/task-type-config.yaml
subPath: task-type-config.yaml
{{- end }}
{{- if .Values.api.enableCustomizedConfig }}
- name: api-config-volume
mountPath: /opt/dolphinscheduler/conf/application.yaml
subPath: application.yaml
{{- end }}
{{- include "dolphinscheduler.sharedStorage.volumeMount" . | nindent 12 }}
{{- include "dolphinscheduler.fsFileResource.volumeMount" . | nindent 12 }}
{{- include "dolphinscheduler.ldap.ssl.volumeMount" . | nindent 12 }}
@ -139,9 +141,11 @@ spec:
- name: config-volume
configMap:
name: {{ include "dolphinscheduler.fullname" . }}-configs
{{- if .Values.api.enableCustomizedConfig }}
- name: api-config-volume
configMap:
name: {{ include "dolphinscheduler.fullname" . }}-api
{{- end }}
{{- include "dolphinscheduler.sharedStorage.volume" . | nindent 8 }}
{{- include "dolphinscheduler.fsFileResource.volume" . | nindent 8 }}
{{- include "dolphinscheduler.ldap.ssl.volume" . | nindent 8 }}

4
deploy/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml
View File

@ -109,9 +109,11 @@ spec:
volumeMounts:
- mountPath: "/opt/dolphinscheduler/logs"
name: {{ include "dolphinscheduler.fullname" . }}-master
{{- if .Values.master.enableCustomizedConfig }}
- name: master-config-volume
mountPath: /opt/dolphinscheduler/conf/application.yaml
subPath: application.yaml
{{- end }}
{{- include "dolphinscheduler.sharedStorage.volumeMount" . | nindent 12 }}
- name: config-volume
mountPath: /opt/dolphinscheduler/conf/common.properties
@ -125,9 +127,11 @@ spec:
{{- else }}
emptyDir: {}
{{- end }}
{{- if .Values.master.enableCustomizedConfig }}
- name: master-config-volume
configMap:
name: {{ include "dolphinscheduler.fullname" . }}-master
{{- end }}
{{- include "dolphinscheduler.sharedStorage.volume" . | nindent 8 }}
- name: config-volume
configMap:

4
deploy/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml
View File

@ -111,9 +111,11 @@ spec:
name: {{ include "dolphinscheduler.fullname" . }}-worker-data
- mountPath: "/opt/dolphinscheduler/logs"
name: {{ include "dolphinscheduler.fullname" . }}-worker-logs
{{- if .Values.worker.enableCustomizedConfig }}
- name: worker-config-volume
mountPath: /opt/dolphinscheduler/conf/application.yaml
subPath: application.yaml
{{- end }}
- name: config-volume
mountPath: /opt/dolphinscheduler/conf/common.properties
subPath: common.properties
@ -142,9 +144,11 @@ spec:
- name: {{ include "dolphinscheduler.fullname" . }}-worker-logs
emptyDir: {}
{{- end }}
{{- if .Values.worker.enableCustomizedConfig }}
- name: worker-config-volume
configMap:
name: {{ include "dolphinscheduler.fullname" . }}-worker
{{- end }}
- name: config-volume
configMap:
name: {{ include "dolphinscheduler.fullname" . }}-configs

53
deploy/kubernetes/dolphinscheduler/values.yaml
View File

@ -449,7 +449,19 @@ master:
# requests:
# memory: "2Gi"
# cpu: "500m"
# -- enable configure custom config
enableCustomizedConfig: false
# -- configure aligned with https://github.com/apache/dolphinscheduler/blob/dev/dolphinscheduler-master/src/main/resources/application.yaml
customizedConfig: { }
# customizedConfig:
# application.yaml: |
# profiles:
# active: postgresql
# banner:
# charset: UTF-8
# jackson:
# time-zone: UTC
# date-format: "yyyy-MM-dd HH:mm:ss"
# -- Periodic probe of container liveness. Container will be restarted if the probe fails.
# More info: [container-probes](https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#container-probes)
livenessProbe:
@ -578,6 +590,17 @@ worker:
# -- Periodic probe of container liveness. Container will be restarted if the probe fails.
# More info: [container-probes](https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#container-probes)
# -- enable configure custom config
enableCustomizedConfig: false
# -- configure aligned with https://github.com/apache/dolphinscheduler/blob/dev/dolphinscheduler-worker/src/main/resources/application.yaml
customizedConfig: { }
# customizedConfig:
# application.yaml: |
# banner:
# charset: UTF-8
# jackson:
# time-zone: UTC
# date-format: "yyyy-MM-dd HH:mm:ss"
livenessProbe:
# -- Turn on and off liveness probe
enabled: true
@ -740,7 +763,19 @@ alert:
# requests:
# memory: "1Gi"
# cpu: "500m"
# -- enable configure custom config
enableCustomizedConfig: false
# -- configure aligned with https://github.com/apache/dolphinscheduler/blob/dev/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/resources/application.yaml
customizedConfig: { }
# customizedConfig:
# application.yaml: |
# profiles:
# active: postgresql
# banner:
# charset: UTF-8
# jackson:
# time-zone: UTC
# date-format: "yyyy-MM-dd HH:mm:ss"
# -- Periodic probe of container liveness. Container will be restarted if the probe fails.
# More info: [container-probes](https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#container-probes)
livenessProbe:
@ -840,7 +875,19 @@ api:
# requests:
# memory: "1Gi"
# cpu: "500m"
# -- enable configure custom config
enableCustomizedConfig: false
# -- configure aligned with https://github.com/apache/dolphinscheduler/blob/dev/dolphinscheduler-api/src/main/resources/application.yaml
customizedConfig: { }
# customizedConfig:
# application.yaml: |
# profiles:
# active: postgresql
# banner:
# charset: UTF-8
# jackson:
# time-zone: UTC
# date-format: "yyyy-MM-dd HH:mm:ss"
# -- Periodic probe of container liveness. Container will be restarted if the probe fails.
# More info: [container-probes](https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#container-probes)
livenessProbe:

23
docs/docs/en/contribute/join/pull-request.md
View File

@ -31,7 +31,7 @@ The corresponding relationship between `Pull Request Type` and `Issue Type` is a
<tr>
<td style="text-align: center;">Bug</td>
<td style="text-align: center;">Fix</td>
<td style="text-align: center;">[Fix-3333][server] Fix xxx</td>
<td style="text-align: center;">[Fix-3333][ui] Fix xxx</td>
</tr>
<tr>
<td style="text-align: center;">Improvement</td>
@ -44,9 +44,24 @@ The corresponding relationship between `Pull Request Type` and `Issue Type` is a
<td style="text-align: center;">[Test-3333][api] Add the e2e test of xxx</td>
</tr>
<tr>
<td style="text-align: center;">Sub-Task</td>
<td style="text-align: center;">(Parent type corresponding to Sub-Task)</td>
<td style="text-align: center;">[Feature-3333][server] Implement xxx</td>
<td style="text-align: center;">Doc</td>
<td style="text-align: center;">Doc</td>
<td style="text-align: center;">[Doc-3333] Improve xxx</td>
</tr>
<tr>
<td style="text-align: center;">E2E</td>
<td style="text-align: center;">E2E</td>
<td style="text-align: center;">[E2E-3333] Implement xxx</td>
</tr>
<tr>
<td style="text-align: center;">CI</td>
<td style="text-align: center;">CI</td>
<td style="text-align: center;">[CI] Improve xxx</td>
</tr>
<tr>
<td style="text-align: center;">Chore</td>
<td style="text-align: center;">Chore</td>
<td style="text-align: center;">[Chore] Improve xxx</td>
</tr>
</tbody>
</table>

16
docs/docs/en/guide/installation/cluster.md
View File

@ -14,21 +14,7 @@ Configure all the configurations refer to [pseudo-cluster deployment](pseudo-clu
### Modify Configuration
This step differs quite a lot from [pseudo-cluster deployment](pseudo-cluster.md), because the deployment script transfers the required resources for installation to each deployment machine by using `scp`. So we only need to modify the configuration of the machine that runs `install.sh` script and configurations will dispatch to cluster by `scp`. The configuration file is under the path `bin/env/install_env.sh`, here we only need to modify section **INSTALL MACHINE**, **DolphinScheduler ENV, Database, Registry Server** and keep other sections the same as [pseudo-cluster deployment](pseudo-cluster.md), the following describes the parameters that must be modified:
```shell
# ---------------------------------------------------------
# INSTALL MACHINE
# ---------------------------------------------------------
# Using IP or machine hostname for the server going to deploy master, worker, API server, the IP of the server
# If you using a hostname, make sure machines could connect each other by hostname
# As below, the hostname of the machine deploying DolphinScheduler is ds1, ds2, ds3, ds4, ds5, where ds1, ds2 install the master server, ds3, ds4, and ds5 installs worker server, the alert server is installed in ds4, and the API server is installed in ds5
ips="ds1,ds2,ds3,ds4,ds5"
masters="ds1,ds2"
workers="ds3:default,ds4:default,ds5:default"
alertServer="ds4"
apiServers="ds5"
```
This step differs quite a lot from [pseudo-cluster deployment](pseudo-cluster.md), please use `scp` or other methods to distribute the configuration files to each machine, then modify the configuration files.
## Start and Login DolphinScheduler

32
docs/docs/en/guide/installation/pseudo-cluster.md
View File

@ -71,31 +71,7 @@ Go to the ZooKeeper installation directory, copy configure file `zoo_sample.cfg`
## Modify Configuration
After completing the preparation of the basic environment, you need to modify the configuration file according to the
environment you used. Change the environment configurations via `export <ENV_NAME>=<VALUE>`. The configuration files are located in directory `bin/env` as `install_env.sh` and `dolphinscheduler_env.sh`.
### Modify `install_env.sh`
File `install_env.sh` describes which machines will be installed DolphinScheduler and what server will be installed on
each machine. You could find this file in the path `bin/env/install_env.sh` and the detail of the configuration as below.
```shell
# ---------------------------------------------------------
# INSTALL MACHINE
# ---------------------------------------------------------
# Due to the master, worker, and API server being deployed on a single node, the IP of the server is the machine IP or localhost
ips="localhost"
sshPort="22"
masters="localhost"
workers="localhost:default"
alertServer="localhost"
apiServers="localhost"
# DolphinScheduler installation path, it will auto-create if not exists
installPath=~/dolphinscheduler
# Deploy user, use the user you create in section **Configure machine SSH password-free login**
deployUser="dolphinscheduler"
```
environment you used. Change the environment configurations via `export <ENV_NAME>=<VALUE>`. The configuration files are located in directory `bin/env` as `dolphinscheduler_env.sh`.
### Modify `dolphinscheduler_env.sh`
@ -146,11 +122,7 @@ Follow the instructions in [datasource-setting](../howto/datasource-setting.md)
## Start DolphinScheduler
Use **deployment user** you created above, running the following command to complete the deployment, and the server log will be stored in the logs folder.
```shell
bash ./bin/install.sh
```
Use **deployment user** you created above, running the command to complete the deployment, and the server log will be stored in the logs folder.
> **_Note:_** For the first time deployment, there maybe occur five times of `sh: bin/dolphinscheduler-daemon.sh: No such file or directory` in the terminal,
> this is non-important information that you can ignore.

5
docs/docs/en/guide/metrics/metrics.md
View File

@ -91,6 +91,11 @@ For example, you can get the master metrics by `curl http://localhost:5679/actua
- stop: the number of stopped workflow instances
- failover: the number of workflow instance fail-overs
### RPC Related Metrics
- ds.rpc.client.sync.request.exception.count: (counter) the number of exceptions occurred in sync rpc requests
- ds.rpc.client.sync.request.duration.time: (histogram) the time cost of sync rpc requests
### Master Server Metrics
- ds.master.overload.count: (counter) the number of times the master overloaded

22
docs/docs/zh/contribute/join/pull-request.md
View File

@ -33,7 +33,7 @@ waste time。
<tr>
<td style="text-align: center;">Bug</td>
<td style="text-align: center;">Fix</td>
<td style="text-align: center;">[Fix-3333][server] Fix xxx</td>
<td style="text-align: center;">[Fix-3333][ui] Fix xxx</td>
</tr>
<tr>
<td style="text-align: center;">Improvement</td>
@ -46,9 +46,23 @@ waste time。
<td style="text-align: center;">[Test-3333][api] Add the e2e test of xxx</td>
</tr>
<tr>
<td style="text-align: center;">Sub-Task</td>
<td style="text-align: center;">Sub-Task 对应的父类型</td>
<td style="text-align: center;">[Feature-3333][server] Implement xxx</td>
<td style="text-align: center;">Doc</td>
<td style="text-align: center;">Doc</td>
<td style="text-align: center;">[Doc-3333] Improve xxx</td>
<tr>
<td style="text-align: center;">E2E</td>
<td style="text-align: center;">E2E</td>
<td style="text-align: center;">[E2E-3333] Implement xxx</td>
</tr>
<tr>
<td style="text-align: center;">CI</td>
<td style="text-align: center;">CI</td>
<td style="text-align: center;">[CI] Improve xxx</td>
</tr>
<tr>
<td style="text-align: center;">Chore</td>
<td style="text-align: center;">Chore</td>
<td style="text-align: center;">[Chore] Improve xxx</td>
</tr>
</tbody>
</table>

16
docs/docs/zh/guide/installation/cluster.md
View File

@ -14,21 +14,7 @@
### 修改相关配置
这个是与[伪集群部署](pseudo-cluster.md)差异较大的一步，因为部署脚本会通过 `scp` 的方式将安装需要的资源传输到各个机器上，所以这一步我们仅需要修改运行`install.sh`脚本的所在机器的配置即可。配置文件在路径在`bin/env/install_env.sh`下，此处我们仅需修改**INSTALL MACHINE**，**DolphinScheduler ENV、Database、Registry Server**与伪集群部署保持一致，下面对必须修改参数进行说明
```shell
# ---------------------------------------------------------
# INSTALL MACHINE
# ---------------------------------------------------------
# 需要配置master、worker、API server，所在服务器的IP均为机器IP或者localhost
# 如果是配置hostname的话，需要保证机器间可以通过hostname相互链接
# 如下图所示，部署 DolphinScheduler 机器的 hostname 为 ds1,ds2,ds3,ds4,ds5，其中 ds1,ds2 安装 master 服务，ds3,ds4,ds5安装 worker 服务，alert server安装在ds4中，api server 安装在ds5中
ips="ds1,ds2,ds3,ds4,ds5"
masters="ds1,ds2"
workers="ds3:default,ds4:default,ds5:default"
alertServer="ds4"
apiServers="ds5"
```
这个是与[伪集群部署](pseudo-cluster.md)差异较大的一步，请使用 scp 等方式将配置文件分发到各台机器上，然后修改配置文件
## 启动 DolphinScheduler && 登录 DolphinScheduler && 启停服务

31
docs/docs/zh/guide/installation/pseudo-cluster.md
View File

@ -70,30 +70,7 @@ chmod 600 ~/.ssh/authorized_keys
## 修改相关配置
完成基础环境的准备后，需要根据你的机器环境修改配置文件。配置文件可以在目录 `bin/env` 中找到，他们分别是 并命名为 `install_env.sh` 和 `dolphinscheduler_env.sh`。
### 修改 `install_env.sh` 文件
文件 `install_env.sh` 描述了哪些机器将被安装 DolphinScheduler 以及每台机器对应安装哪些服务。您可以在路径 `bin/env/install_env.sh` 中找到此文件，可通过以下方式更改 env 变量,export <ENV_NAME>=<VALUE>，配置详情如下。
```shell
# ---------------------------------------------------------
# INSTALL MACHINE
# ---------------------------------------------------------
# Due to the master, worker, and API server being deployed on a single node, the IP of the server is the machine IP or localhost
ips="localhost"
sshPort="22"
masters="localhost"
workers="localhost:default"
alertServer="localhost"
apiServers="localhost"
# DolphinScheduler installation path, it will auto-create if not exists
installPath=~/dolphinscheduler
# Deploy user, use the user you create in section **Configure machine SSH password-free login**
deployUser="dolphinscheduler"
```
完成基础环境的准备后，需要根据你的机器环境修改配置文件。配置文件可以在目录 `bin/env/dolphinscheduler_env.sh` 中找到。
### 修改 `dolphinscheduler_env.sh` 文件
@ -141,11 +118,7 @@ export PATH=$HADOOP_HOME/bin:$SPARK_HOME/bin:$PYTHON_LAUNCHER:$JAVA_HOME/bin:$HI
## 启动 DolphinScheduler
使用上面创建的**部署用户**运行以下命令完成部署，部署后的运行日志将存放在 logs 文件夹内
```shell
bash ./bin/install.sh
```
使用上面创建的**部署用户**运行命令完成部署，部署后的运行日志将存放在 logs 文件夹内
> **_注意:_** 第一次部署的话，可能出现 5 次`sh: bin/dolphinscheduler-daemon.sh: No such file or directory`相关信息，此为非重要信息直接忽略即可

5
docs/docs/zh/guide/metrics/metrics.md
View File

@ -91,6 +91,11 @@ metrics exporter端口`server.port`是在application.yaml里定义的: master: `
- stop：停止的工作流实例数量
- failover：容错的工作流实例数量
### RPC相关指标
- ds.rpc.client.sync.request.exception.count: (counter) 同步rpc请求异常数
- ds.rpc.client.sync.request.duration.time: (histogram) 同步rpc请求耗时
### Master Server指标
- ds.master.overload.count: (counter) master过载次数

10
dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/resources/application.yaml
View File

@ -82,12 +82,12 @@ registry:
namespace: dolphinscheduler
connect-string: localhost:2181
retry-policy:
base-sleep-time: 60ms
max-sleep: 300ms
base-sleep-time: 1s
max-sleep: 3s
max-retries: 5
session-timeout: 30s
connection-timeout: 9s
block-until-connected: 600ms
session-timeout: 60s
connection-timeout: 15s
block-until-connected: 15s
digest: ~
metrics:

20
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/ApiApplicationServer.java
View File

@ -19,22 +19,14 @@ package org.apache.dolphinscheduler.api;
import org.apache.dolphinscheduler.api.metrics.ApiServerMetrics;
import org.apache.dolphinscheduler.common.CommonConfiguration;
import org.apache.dolphinscheduler.common.enums.PluginType;
import org.apache.dolphinscheduler.common.thread.DefaultUncaughtExceptionHandler;
import org.apache.dolphinscheduler.dao.DaoConfiguration;
import org.apache.dolphinscheduler.dao.PluginDao;
import org.apache.dolphinscheduler.dao.entity.PluginDefine;
import org.apache.dolphinscheduler.plugin.datasource.api.plugin.DataSourceProcessorProvider;
import org.apache.dolphinscheduler.plugin.storage.api.StorageConfiguration;
import org.apache.dolphinscheduler.plugin.task.api.TaskChannelFactory;
import org.apache.dolphinscheduler.plugin.task.api.TaskPluginManager;
import org.apache.dolphinscheduler.registry.api.RegistryConfiguration;
import org.apache.dolphinscheduler.service.ServiceConfiguration;
import org.apache.dolphinscheduler.spi.params.PluginParamsTransfer;
import org.apache.dolphinscheduler.spi.params.base.PluginParams;
import java.util.List;
import java.util.Map;
import lombok.extern.slf4j.Slf4j;
@ -68,17 +60,7 @@ public class ApiApplicationServer {
@EventListener
public void run(ApplicationReadyEvent readyEvent) {
log.info("Received spring application context ready event will load taskPlugin and write to DB");
// install task plugin
TaskPluginManager.loadPlugin();
DataSourceProcessorProvider.initialize();
for (Map.Entry<String, TaskChannelFactory> entry : TaskPluginManager.getTaskChannelFactoryMap().entrySet()) {
String taskPluginName = entry.getKey();
TaskChannelFactory taskChannelFactory = entry.getValue();
List<PluginParams> params = taskChannelFactory.getParams();
String paramsJson = PluginParamsTransfer.transferParamsToJson(params);
PluginDefine pluginDefine = new PluginDefine(taskPluginName, PluginType.TASK.getDesc(), paramsJson);
pluginDao.addOrUpdatePluginDefine(pluginDefine);
}
TaskPluginManager.loadTaskPlugin();
}
}

529
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java
View File

@ -28,42 +28,50 @@ import static org.apache.dolphinscheduler.api.enums.Status.QUERY_DATASOURCE_BY_T
import static org.apache.dolphinscheduler.api.enums.Status.QUERY_RESOURCES_LIST_ERROR;
import static org.apache.dolphinscheduler.api.enums.Status.QUERY_RESOURCES_LIST_PAGING;
import static org.apache.dolphinscheduler.api.enums.Status.QUERY_UDF_FUNCTION_LIST_PAGING_ERROR;
import static org.apache.dolphinscheduler.api.enums.Status.RESOURCE_FILE_IS_EMPTY;
import static org.apache.dolphinscheduler.api.enums.Status.RESOURCE_NOT_EXIST;
import static org.apache.dolphinscheduler.api.enums.Status.UPDATE_RESOURCE_ERROR;
import static org.apache.dolphinscheduler.api.enums.Status.UPDATE_UDF_FUNCTION_ERROR;
import static org.apache.dolphinscheduler.api.enums.Status.VERIFY_RESOURCE_BY_NAME_AND_TYPE_ERROR;
import static org.apache.dolphinscheduler.api.enums.Status.VERIFY_UDF_FUNCTION_NAME_ERROR;
import static org.apache.dolphinscheduler.api.enums.Status.VIEW_RESOURCE_FILE_ON_LINE_ERROR;
import static org.apache.dolphinscheduler.api.enums.Status.VIEW_UDF_FUNCTION_ERROR;
import org.apache.dolphinscheduler.api.audit.OperatorLog;
import org.apache.dolphinscheduler.api.audit.enums.AuditType;
import org.apache.dolphinscheduler.api.dto.resources.DeleteDataTransferResponse;
import org.apache.dolphinscheduler.api.dto.resources.CreateDirectoryRequest;
import org.apache.dolphinscheduler.api.dto.resources.CreateFileFromContentRequest;
import org.apache.dolphinscheduler.api.dto.resources.CreateFileRequest;
import org.apache.dolphinscheduler.api.dto.resources.DeleteResourceRequest;
import org.apache.dolphinscheduler.api.dto.resources.DownloadFileRequest;
import org.apache.dolphinscheduler.api.dto.resources.FetchFileContentRequest;
import org.apache.dolphinscheduler.api.dto.resources.PagingResourceItemRequest;
import org.apache.dolphinscheduler.api.dto.resources.RenameDirectoryRequest;
import org.apache.dolphinscheduler.api.dto.resources.RenameFileRequest;
import org.apache.dolphinscheduler.api.dto.resources.ResourceComponent;
import org.apache.dolphinscheduler.api.dto.resources.UpdateFileFromContentRequest;
import org.apache.dolphinscheduler.api.dto.resources.UpdateFileRequest;
import org.apache.dolphinscheduler.api.exceptions.ApiException;
import org.apache.dolphinscheduler.api.service.ResourcesService;
import org.apache.dolphinscheduler.api.service.UdfFuncService;
import org.apache.dolphinscheduler.api.utils.PageInfo;
import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.api.vo.ResourceItemVO;
import org.apache.dolphinscheduler.api.vo.resources.FetchFileContentResponse;
import org.apache.dolphinscheduler.common.constants.Constants;
import org.apache.dolphinscheduler.common.enums.ProgramType;
import org.apache.dolphinscheduler.common.enums.UdfType;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.plugin.storage.api.StorageEntity;
import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils;
import org.apache.dolphinscheduler.spi.enums.ResourceType;
import org.apache.commons.lang3.StringUtils;
import java.util.Map;
import java.util.List;
import javax.servlet.http.HttpServletResponse;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.io.Resource;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
@ -77,15 +85,14 @@ import org.springframework.web.bind.annotation.ResponseStatus;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.multipart.MultipartFile;
import com.google.common.io.Files;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.Parameter;
import io.swagger.v3.oas.annotations.Parameters;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.tags.Tag;
/**
* resources controller
*/
@Tag(name = "RESOURCES_TAG")
@RestController
@RequestMapping("resources")
@ -94,41 +101,34 @@ public class ResourcesController extends BaseController {
@Autowired
private ResourcesService resourceService;
@Autowired
private UdfFuncService udfFuncService;
/**
* @param loginUser login user
* @param type type
* @param alias alias
* @param pid parent id
* @param currentDir current directory
* @return create result code
*/
@Operation(summary = "createDirectory", description = "CREATE_RESOURCE_NOTES")
@Parameters({
@Parameter(name = "type", description = "RESOURCE_TYPE", required = true, schema = @Schema(implementation = ResourceType.class)),
@Parameter(name = "name", description = "RESOURCE_NAME", required = true, schema = @Schema(implementation = String.class)),
@Parameter(name = "pid", description = "RESOURCE_PID", required = true, schema = @Schema(implementation = int.class, example = "10")),
@Parameter(name = "currentDir", description = "RESOURCE_CURRENT_DIR", required = true, schema = @Schema(implementation = String.class))})
@PostMapping(value = "/directory")
@ApiException(CREATE_RESOURCE_ERROR)
@OperatorLog(auditType = AuditType.FOLDER_CREATE)
public Result<Object> createDirectory(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
@RequestParam(value = "type") ResourceType type,
@RequestParam(value = "name") String alias,
@RequestParam(value = "pid") int pid,
@RequestParam(value = "currentDir") String currentDir) {
// todo verify the directory name
return resourceService.createDirectory(loginUser, alias, type, pid, currentDir);
public Result<Void> createDirectory(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
@RequestParam(value = "type") ResourceType type,
@RequestParam(value = "name") String directoryName,
@RequestParam(value = "currentDir") String parentDirectory) {
CreateDirectoryRequest createDirectoryRequest = CreateDirectoryRequest.builder()
.loginUser(loginUser)
.directoryName(directoryName)
.type(type)
.parentAbsoluteDirectory(parentDirectory)
.build();
resourceService.createDirectory(createDirectoryRequest);
return Result.success(null);
}
/**
* create resource
*
* @return create result code
*/
@Operation(summary = "createResource", description = "CREATE_RESOURCE_NOTES")
@Operation(summary = "uploadFile", description = "CREATE_FILE")
@Parameters({
@Parameter(name = "type", description = "RESOURCE_TYPE", required = true, schema = @Schema(implementation = ResourceType.class)),
@Parameter(name = "name", description = "RESOURCE_NAME", required = true, schema = @Schema(implementation = String.class)),
@ -137,224 +137,24 @@ public class ResourcesController extends BaseController {
@PostMapping()
@ApiException(CREATE_RESOURCE_ERROR)
@OperatorLog(auditType = AuditType.FILE_CREATE)
public Result<Object> createResource(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
@RequestParam(value = "type") ResourceType type,
@RequestParam(value = "name") String alias,
@RequestParam("file") MultipartFile file,
@RequestParam(value = "currentDir") String currentDir) {
// todo verify the file name
return resourceService.uploadResource(loginUser, alias, type, file, currentDir);
public Result<Void> createFile(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
@RequestParam(value = "type") ResourceType type,
@RequestParam(value = "name") String fileName,
@RequestParam("file") MultipartFile file,
@RequestParam(value = "currentDir") String parentDirectoryAbsolutePath) {
CreateFileRequest uploadFileRequest = CreateFileRequest.builder()
.loginUser(loginUser)
.fileName(fileName)
.file(file)
.type(type)
.parentAbsoluteDirectory(parentDirectoryAbsolutePath)
.build();
resourceService.createFile(uploadFileRequest);
return Result.success();
}
/**
* update resource
*
* @param loginUser login user
* @param alias alias
* @param type resource type
* @param file resource file
* @return update result code
*/
@Operation(summary = "updateResource", description = "UPDATE_RESOURCE_NOTES")
@Parameters({
@Parameter(name = "fullName", description = "RESOURCE_FULLNAME", required = true, schema = @Schema(implementation = String.class)),
@Parameter(name = "tenantCode", description = "TENANT_CODE", required = true, schema = @Schema(implementation = String.class)),
@Parameter(name = "type", description = "RESOURCE_TYPE", required = true, schema = @Schema(implementation = ResourceType.class)),
@Parameter(name = "name", description = "RESOURCE_NAME", required = true, schema = @Schema(implementation = String.class)),
@Parameter(name = "file", description = "RESOURCE_FILE", required = true, schema = @Schema(implementation = MultipartFile.class))})
@PutMapping()
@ApiException(UPDATE_RESOURCE_ERROR)
@OperatorLog(auditType = AuditType.FILE_UPDATE)
public Result<Object> updateResource(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
@RequestParam(value = "fullName") String fullName,
@RequestParam(value = "tenantCode", required = false) String tenantCode,
@RequestParam(value = "type") ResourceType type,
@RequestParam(value = "name") String alias,
@RequestParam(value = "file", required = false) MultipartFile file) {
return resourceService.updateResource(loginUser, fullName, tenantCode, alias, type, file);
}
/**
* query resources list
*
* @param loginUser login user
* @param type resource type
* @return resource list
*/
@Operation(summary = "queryResourceList", description = "QUERY_RESOURCE_LIST_NOTES")
@Parameters({
@Parameter(name = "type", description = "RESOURCE_TYPE", required = true, schema = @Schema(implementation = ResourceType.class)),
@Parameter(name = "fullName", description = "RESOURCE_FULLNAME", required = true, schema = @Schema(implementation = String.class))})
@GetMapping(value = "/list")
@ResponseStatus(HttpStatus.OK)
@ApiException(QUERY_RESOURCES_LIST_ERROR)
public Result<Object> queryResourceList(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
@RequestParam(value = "type") ResourceType type,
@RequestParam(value = "fullName") String fullName) {
Map<String, Object> result = resourceService.queryResourceList(loginUser, type, fullName);
return returnDataList(result);
}
/**
* query resources list paging
*
* @param loginUser login user
* @param type resource type
* @param searchVal search value
* @param pageNo page number
* @param pageSize page size
* @return resource list page
*/
@Operation(summary = "queryResourceListPaging", description = "QUERY_RESOURCE_LIST_PAGING_NOTES")
@Parameters({
@Parameter(name = "type", description = "RESOURCE_TYPE", required = true, schema = @Schema(implementation = ResourceType.class)),
@Parameter(name = "fullName", description = "RESOURCE_FULLNAME", required = true, schema = @Schema(implementation = String.class, example = "bucket_name/tenant_name/type/ds")),
@Parameter(name = "searchVal", description = "SEARCH_VAL", schema = @Schema(implementation = String.class)),
@Parameter(name = "pageNo", description = "PAGE_NO", required = true, schema = @Schema(implementation = int.class, example = "1")),
@Parameter(name = "pageSize", description = "PAGE_SIZE", required = true, schema = @Schema(implementation = int.class, example = "20"))})
@GetMapping()
@ResponseStatus(HttpStatus.OK)
@ApiException(QUERY_RESOURCES_LIST_PAGING)
public Result<PageInfo<StorageEntity>> queryResourceListPaging(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
@RequestParam(value = "fullName") String fullName,
@RequestParam(value = "tenantCode") String tenantCode,
@RequestParam(value = "type") ResourceType type,
@RequestParam("pageNo") Integer pageNo,
@RequestParam(value = "searchVal", required = false) String searchVal,
@RequestParam("pageSize") Integer pageSize) {
checkPageParams(pageNo, pageSize);
searchVal = ParameterUtils.handleEscapes(searchVal);
return resourceService.queryResourceListPaging(loginUser, fullName, tenantCode, type, searchVal, pageNo,
pageSize);
}
/**
* delete resource
*
* @param loginUser login user
* @return delete result code
*/
@Operation(summary = "deleteResource", description = "DELETE_RESOURCE_BY_ID_NOTES")
@Parameters({
@Parameter(name = "fullName", description = "RESOURCE_FULLNAME", required = true, schema = @Schema(implementation = String.class, example = "test/"))})
@DeleteMapping()
@ResponseStatus(HttpStatus.OK)
@ApiException(DELETE_RESOURCE_ERROR)
@OperatorLog(auditType = AuditType.FILE_DELETE)
public Result<Object> deleteResource(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
@RequestParam(value = "fullName") String fullName,
@RequestParam(value = "tenantCode", required = false) String tenantCode) throws Exception {
return resourceService.delete(loginUser, fullName, tenantCode);
}
/**
* delete DATA_TRANSFER data
*
* @param loginUser login user
* @return delete result code
*/
@Operation(summary = "deleteDataTransferData", description = "Delete the N days ago data of DATA_TRANSFER ")
@Parameters({
@Parameter(name = "days", description = "N days ago", required = true, schema = @Schema(implementation = Integer.class))})
@DeleteMapping(value = "/data-transfer")
@ResponseStatus(HttpStatus.OK)
@ApiException(DELETE_RESOURCE_ERROR)
public DeleteDataTransferResponse deleteDataTransferData(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
@RequestParam(value = "days") Integer days) {
return resourceService.deleteDataTransferData(loginUser, days);
}
/**
* verify resource by alias and type
*
* @param loginUser login user
* @param fullName resource full name
* @param type resource type
* @return true if the resource name not exists, otherwise return false
*/
@Operation(summary = "verifyResourceName", description = "VERIFY_RESOURCE_NAME_NOTES")
@Parameters({
@Parameter(name = "type", description = "RESOURCE_TYPE", required = true, schema = @Schema(implementation = ResourceType.class)),
@Parameter(name = "fullName", description = "RESOURCE_FULL_NAME", required = true, schema = @Schema(implementation = String.class))})
@GetMapping(value = "/verify-name")
@ResponseStatus(HttpStatus.OK)
@ApiException(VERIFY_RESOURCE_BY_NAME_AND_TYPE_ERROR)
public Result<Object> verifyResourceName(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
@RequestParam(value = "fullName") String fullName,
@RequestParam(value = "type") ResourceType type) {
return resourceService.verifyResourceName(fullName, type, loginUser);
}
/**
* query resources by type
*
* @param loginUser login user
* @param type resource type
* @return resource list
*/
@Operation(summary = "queryResourceByProgramType", description = "QUERY_RESOURCE_LIST_NOTES")
@Parameters({
@Parameter(name = "type", description = "RESOURCE_TYPE", required = true, schema = @Schema(implementation = ResourceType.class))})
@GetMapping(value = "/query-by-type")
@ResponseStatus(HttpStatus.OK)
@ApiException(QUERY_RESOURCES_LIST_ERROR)
public Result<Object> queryResourceJarList(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
@RequestParam(value = "type") ResourceType type,
@RequestParam(value = "programType", required = false) ProgramType programType) {
return resourceService.queryResourceByProgramType(loginUser, type, programType);
}
/**
* query resource by file name and type
*
* @param loginUser login user
* @param fileName resource full name
* @param tenantCode tenantCode of the owner of the resource
* @param type resource type
* @return true if the resource name not exists, otherwise return false
*/
@Operation(summary = "queryResourceByFileName", description = "QUERY_BY_RESOURCE_FILE_NAME")
@Parameters({
@Parameter(name = "type", description = "RESOURCE_TYPE", required = true, schema = @Schema(implementation = ResourceType.class)),
@Parameter(name = "fileName", description = "RESOURCE_FILE_NAME", required = true, schema = @Schema(implementation = String.class)),
@Parameter(name = "tenantCode", description = "TENANT_CODE", required = true, schema = @Schema(implementation = String.class)),})
@GetMapping(value = "/query-file-name")
@ResponseStatus(HttpStatus.OK)
@ApiException(RESOURCE_NOT_EXIST)
public Result<Object> queryResourceByFileName(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
@RequestParam(value = "fileName", required = false) String fileName,
@RequestParam(value = "tenantCode", required = false) String tenantCode,
@RequestParam(value = "type") ResourceType type) {
return resourceService.queryResourceByFileName(loginUser, fileName, type, tenantCode);
}
/**
* view resource file online
*
* @param loginUser login user
* @param skipLineNum skip line number
* @param limit limit
* @return resource content
*/
@Operation(summary = "viewResource", description = "VIEW_RESOURCE_BY_ID_NOTES")
@Parameters({
@Parameter(name = "fullName", description = "RESOURCE_FULL_NAME", required = true, schema = @Schema(implementation = String.class, example = "tenant/1.png")),
@Parameter(name = "tenantCode", description = "TENANT_CODE", required = true, schema = @Schema(implementation = String.class)),
@Parameter(name = "skipLineNum", description = "SKIP_LINE_NUM", required = true, schema = @Schema(implementation = int.class, example = "100")),
@Parameter(name = "limit", description = "LIMIT", required = true, schema = @Schema(implementation = int.class, example = "100"))})
@GetMapping(value = "/view")
@ApiException(VIEW_RESOURCE_FILE_ON_LINE_ERROR)
public Result viewResource(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
@RequestParam(value = "skipLineNum") int skipLineNum,
@RequestParam(value = "limit") int limit,
@RequestParam(value = "fullName") String fullName,
@RequestParam(value = "tenantCode") String tenantCode) {
return resourceService.readResource(loginUser, fullName, tenantCode, skipLineNum, limit);
}
@Operation(summary = "onlineCreateResource", description = "ONLINE_CREATE_RESOURCE_NOTES")
@Operation(summary = "createFileFromContent", description = "ONLINE_CREATE_RESOURCE_NOTES")
@Parameters({
@Parameter(name = "type", description = "RESOURCE_TYPE", required = true, schema = @Schema(implementation = ResourceType.class)),
@Parameter(name = "fileName", description = "RESOURCE_NAME", required = true, schema = @Schema(implementation = String.class)),
@ -365,27 +165,24 @@ public class ResourcesController extends BaseController {
@PostMapping(value = "/online-create")
@ApiException(CREATE_RESOURCE_FILE_ON_LINE_ERROR)
@OperatorLog(auditType = AuditType.FILE_CREATE)
public Result createResourceFile(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
@RequestParam(value = "type") ResourceType type,
@RequestParam(value = "fileName") String fileName,
@RequestParam(value = "suffix") String fileSuffix,
@RequestParam(value = "content") String content,
@RequestParam(value = "currentDir") String currentDir) {
if (StringUtils.isEmpty(content)) {
log.error("resource file contents are not allowed to be empty");
return error(RESOURCE_FILE_IS_EMPTY.getCode(), RESOURCE_FILE_IS_EMPTY.getMsg());
}
return resourceService.createResourceFile(loginUser, type, fileName, fileSuffix, content, currentDir);
public Result<Void> createFileFromContent(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
@RequestParam(value = "type") ResourceType type,
@RequestParam(value = "fileName") String fileName,
@RequestParam(value = "suffix") String fileSuffix,
@RequestParam(value = "content") String fileContent,
@RequestParam(value = "currentDir") String fileParentDirectoryAbsolutePath) {
CreateFileFromContentRequest createFileFromContentRequest = CreateFileFromContentRequest.builder()
.loginUser(loginUser)
.fileName(fileName + "." + fileSuffix)
.fileContent(fileContent)
.type(type)
.parentAbsoluteDirectory(fileParentDirectoryAbsolutePath)
.build();
resourceService.createFileFromContent(createFileFromContentRequest);
return Result.success();
}
/**
* edit resource file online
*
* @param loginUser login user
* @param content content
* @return update result code
*/
@Operation(summary = "updateResourceContent", description = "UPDATE_RESOURCE_NOTES")
@Operation(summary = "updateFileContent", description = "UPDATE_RESOURCE_NOTES")
@Parameters({
@Parameter(name = "content", description = "CONTENT", required = true, schema = @Schema(implementation = String.class)),
@Parameter(name = "fullName", description = "FULL_NAME", required = true, schema = @Schema(implementation = String.class)),
@ -393,52 +190,170 @@ public class ResourcesController extends BaseController {
@PutMapping(value = "/update-content")
@ApiException(EDIT_RESOURCE_FILE_ON_LINE_ERROR)
@OperatorLog(auditType = AuditType.FILE_UPDATE)
public Result updateResourceContent(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
@RequestParam(value = "fullName") String fullName,
@RequestParam(value = "tenantCode") String tenantCode,
@RequestParam(value = "content") String content) {
if (StringUtils.isEmpty(content)) {
log.error("The resource file contents are not allowed to be empty");
return error(RESOURCE_FILE_IS_EMPTY.getCode(), RESOURCE_FILE_IS_EMPTY.getMsg());
}
return resourceService.updateResourceContent(loginUser, fullName, tenantCode, content);
public Result<Void> updateFileContent(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
@RequestParam(value = "fullName") String fileAbsolutePath,
@RequestParam(value = "content") String fileContent) {
UpdateFileFromContentRequest updateFileContentRequest = UpdateFileFromContentRequest.builder()
.loginUser(loginUser)
.fileContent(fileContent)
.fileAbsolutePath(fileAbsolutePath)
.build();
resourceService.updateFileFromContent(updateFileContentRequest);
return Result.success();
}
@Operation(summary = "updateResource", description = "UPDATE_RESOURCE_NOTES")
@Parameters({
@Parameter(name = "fullName", description = "RESOURCE_FULLNAME", required = true, schema = @Schema(implementation = String.class)),
@Parameter(name = "tenantCode", description = "TENANT_CODE", required = true, schema = @Schema(implementation = String.class)),
@Parameter(name = "type", description = "RESOURCE_TYPE", required = true, schema = @Schema(implementation = ResourceType.class)),
@Parameter(name = "name", description = "RESOURCE_NAME", required = true, schema = @Schema(implementation = String.class)),
@Parameter(name = "file", description = "RESOURCE_FILE", required = true, schema = @Schema(implementation = MultipartFile.class))})
@PutMapping()
@ApiException(UPDATE_RESOURCE_ERROR)
@OperatorLog(auditType = AuditType.FILE_UPDATE)
public Result<Void> updateResource(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
@RequestParam(value = "fullName") String resourceAbsolutePath,
@RequestParam(value = "name") String resourceName,
@RequestParam(value = "file", required = false) MultipartFile file) {
if (StringUtils.isEmpty(Files.getFileExtension(resourceName))) {
RenameDirectoryRequest renameDirectoryRequest = RenameDirectoryRequest.builder()
.loginUser(loginUser)
.directoryAbsolutePath(resourceAbsolutePath)
.newDirectoryName(resourceName)
.build();
resourceService.renameDirectory(renameDirectoryRequest);
return Result.success();
}
if (file == null) {
RenameFileRequest renameFileRequest = RenameFileRequest.builder()
.loginUser(loginUser)
.fileAbsolutePath(resourceAbsolutePath)
.newFileName(resourceName)
.build();
resourceService.renameFile(renameFileRequest);
return Result.success();
}
UpdateFileRequest updateFileRequest = UpdateFileRequest.builder()
.loginUser(loginUser)
.fileAbsolutePath(resourceAbsolutePath)
.file(file)
.build();
resourceService.updateFile(updateFileRequest);
return Result.success();
}
@Operation(summary = "pagingResourceItemRequest", description = "PAGING_RESOURCE_ITEM_LIST")
@Parameters({
@Parameter(name = "type", description = "RESOURCE_TYPE", required = true, schema = @Schema(implementation = ResourceType.class)),
@Parameter(name = "fullName", description = "RESOURCE_FULLNAME", required = true, schema = @Schema(implementation = String.class, example = "bucket_name/tenant_name/type/ds")),
@Parameter(name = "searchVal", description = "SEARCH_VAL", schema = @Schema(implementation = String.class)),
@Parameter(name = "pageNo", description = "PAGE_NO", required = true, schema = @Schema(implementation = int.class, example = "1")),
@Parameter(name = "pageSize", description = "PAGE_SIZE", required = true, schema = @Schema(implementation = int.class, example = "20"))})
@GetMapping()
@ResponseStatus(HttpStatus.OK)
@ApiException(QUERY_RESOURCES_LIST_PAGING)
public Result<PageInfo<ResourceItemVO>> pagingResourceItemRequest(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
@RequestParam(value = "fullName") String resourceAbsolutePath,
@RequestParam(value = "type") ResourceType resourceType,
@RequestParam(value = "searchVal", required = false) String resourceNameKeyWord,
@RequestParam("pageNo") Integer pageNo,
@RequestParam("pageSize") Integer pageSize) {
PagingResourceItemRequest pagingResourceItemRequest = PagingResourceItemRequest.builder()
.loginUser(loginUser)
.resourceAbsolutePath(resourceAbsolutePath)
.resourceType(resourceType)
.resourceNameKeyWord(StringUtils.trim(ParameterUtils.handleEscapes(resourceNameKeyWord)))
.pageNo(pageNo)
.pageSize(pageSize)
.build();
pagingResourceItemRequest.checkPageNoAndPageSize();
return Result.success(resourceService.pagingResourceItem(pagingResourceItemRequest));
}
// todo: this api is used for udf, we should remove it
@Operation(summary = "queryResourceList", description = "QUERY_RESOURCE_LIST_NOTES")
@Parameters({
@Parameter(name = "type", description = "RESOURCE_TYPE", required = true, schema = @Schema(implementation = ResourceType.class)),
@Parameter(name = "fullName", description = "RESOURCE_FULLNAME", required = true, schema = @Schema(implementation = String.class))})
@GetMapping(value = "/list")
@ResponseStatus(HttpStatus.OK)
@ApiException(QUERY_RESOURCES_LIST_ERROR)
public Result<List<ResourceComponent>> queryResourceList(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
@RequestParam(value = "type") ResourceType type) {
return Result.success(resourceService.queryResourceFiles(loginUser, type));
}
@Operation(summary = "deleteResource", description = "DELETE_RESOURCE_BY_ID_NOTES")
@Parameters({
@Parameter(name = "fullName", description = "RESOURCE_FULLNAME", required = true, schema = @Schema(implementation = String.class, example = "file:////tmp/dolphinscheduler/storage/default/resources/demo.sql"))
})
@DeleteMapping()
@ResponseStatus(HttpStatus.OK)
@ApiException(DELETE_RESOURCE_ERROR)
@OperatorLog(auditType = AuditType.FILE_DELETE)
public Result<Void> deleteResource(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
@RequestParam(value = "fullName") String resourceAbsolutePath) {
DeleteResourceRequest deleteResourceRequest = DeleteResourceRequest.builder()
.loginUser(loginUser)
.resourceAbsolutePath(resourceAbsolutePath)
.build();
resourceService.delete(deleteResourceRequest);
return Result.success();
}
@Operation(summary = "queryResourceFileList", description = "QUERY_RESOURCE_FILE_LIST_NOTES")
@Parameters({
@Parameter(name = "type", description = "RESOURCE_TYPE", required = true, schema = @Schema(implementation = ResourceType.class))})
@GetMapping(value = "/query-by-type")
@ResponseStatus(HttpStatus.OK)
@ApiException(QUERY_RESOURCES_LIST_ERROR)
public Result<List<ResourceComponent>> queryResourceFileList(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
@RequestParam(value = "type") ResourceType type) {
return Result.success(resourceService.queryResourceFiles(loginUser, type));
}
@Operation(summary = "viewResource", description = "VIEW_RESOURCE_BY_ID_NOTES")
@Parameters({
@Parameter(name = "fullName", description = "RESOURCE_FULL_NAME", required = true, schema = @Schema(implementation = String.class, example = "tenant/1.png")),
@Parameter(name = "tenantCode", description = "TENANT_CODE", required = true, schema = @Schema(implementation = String.class)),
@Parameter(name = "skipLineNum", description = "SKIP_LINE_NUM", required = true, schema = @Schema(implementation = int.class, example = "100")),
@Parameter(name = "limit", description = "LIMIT", required = true, schema = @Schema(implementation = int.class, example = "100"))})
@GetMapping(value = "/view")
@ApiException(VIEW_RESOURCE_FILE_ON_LINE_ERROR)
public Result<FetchFileContentResponse> viewResource(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
@RequestParam(value = "fullName") String resourceAbsoluteFilePath,
@RequestParam(value = "skipLineNum") int skipLineNum,
@RequestParam(value = "limit") int limit) {
FetchFileContentRequest fetchFileContentRequest = FetchFileContentRequest.builder()
.loginUser(loginUser)
.resourceFileAbsolutePath(resourceAbsoluteFilePath)
.limit(limit == -1 ? Integer.MAX_VALUE : skipLineNum)
.skipLineNum(skipLineNum)
.build();
return Result.success(resourceService.fetchResourceFileContent(fetchFileContentRequest));
}
/**
* download resource file
*
* @param loginUser login user
* @return resource content
*/
@Operation(summary = "downloadResource", description = "DOWNLOAD_RESOURCE_NOTES")
@Parameters({
@Parameter(name = "fullName", description = "RESOURCE_FULLNAME", required = true, schema = @Schema(implementation = String.class, example = "test/"))})
@GetMapping(value = "/download")
@ResponseBody
@ApiException(DOWNLOAD_RESOURCE_FILE_ERROR)
public ResponseEntity downloadResource(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
@RequestParam(value = "fullName") String fullName) throws Exception {
Resource file = resourceService.downloadResource(loginUser, fullName);
if (file == null) {
return ResponseEntity.status(HttpStatus.BAD_REQUEST).body(RESOURCE_NOT_EXIST.getMsg());
}
return ResponseEntity.ok()
.header(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=\"" + file.getFilename() + "\"")
.body(file);
public void downloadResource(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
HttpServletResponse response,
@RequestParam(value = "fullName") String fileAbsolutePath) {
DownloadFileRequest downloadFileRequest = DownloadFileRequest.builder()
.loginUser(loginUser)
.fileAbsolutePath(fileAbsolutePath)
.build();
resourceService.downloadResource(response, downloadFileRequest);
}
/**
* create udf function
*
* @param loginUser login user
* @param type udf type
* @param funcName function name
* @param argTypes argument types
* @param database database
* @param description description
* @param className class name
* @return create result code
*/
@Operation(summary = "createUdfFunc", description = "CREATE_UDF_FUNCTION_NOTES")
@Parameters({
@Parameter(name = "type", description = "UDF_TYPE", required = true, schema = @Schema(implementation = UdfType.class)),
@ -612,8 +527,8 @@ public class ResourcesController extends BaseController {
@GetMapping(value = "/base-dir")
@ResponseStatus(HttpStatus.OK)
@ApiException(RESOURCE_NOT_EXIST)
public Result<Object> queryResourceBaseDir(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
public Result<String> queryResourceBaseDir(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
@RequestParam(value = "type") ResourceType type) {
return resourceService.queryResourceBaseDir(loginUser, type);
return Result.success(resourceService.queryResourceBaseDir(loginUser, type));
}
}

34
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/AbstractResourceCreateRequest.java Normal file
View File

@ -0,0 +1,34 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.dto.resources;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.spi.enums.ResourceType;
import lombok.Data;
import lombok.experimental.SuperBuilder;
@Data
@SuperBuilder
public abstract class AbstractResourceCreateRequest {
private User loginUser;
private String parentAbsoluteDirectory;
private ResourceType type;
}

20
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/BlockingOpportunity.java → dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/AbstractResourceDto.java
View File

@ -15,21 +15,15 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.enums;
package org.apache.dolphinscheduler.api.dto.resources;
public enum BlockingOpportunity {
import lombok.Data;
import lombok.experimental.SuperBuilder;
BLOCKING_ON_SUCCESS("BlockingOnSuccess"),
BLOCKING_ON_FAILED("BlockingOnFailed");
@Data
@SuperBuilder
public abstract class AbstractResourceDto {
private final String desc;
BlockingOpportunity(String desc) {
this.desc = desc;
}
public String getDesc() {
return desc;
}
private String resourceAbsolutePath;
}

37
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/CreateDirectoryDto.java Normal file
View File

@ -0,0 +1,37 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.dto.resources;
import org.apache.dolphinscheduler.dao.entity.User;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
public class CreateDirectoryDto {
private User loginUser;
private String directoryAbsolutePath;
}

41
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/CreateDirectoryRequest.java Normal file
View File

@ -0,0 +1,41 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.dto.resources;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.spi.enums.ResourceType;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class CreateDirectoryRequest {
private User loginUser;
private ResourceType type;
private String parentAbsoluteDirectory;
private String directoryName;
}

48
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/WorkerServer.java → dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/CreateFileDto.java
View File

@ -15,47 +15,27 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.dao.entity;
package org.apache.dolphinscheduler.api.dto.resources;
import java.util.Date;
import org.apache.dolphinscheduler.dao.entity.User;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import org.springframework.web.multipart.MultipartFile;
@Data
public class WorkerServer {
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class CreateFileDto {
/**
* id
*/
private int id;
private User loginUser;
/**
* host
*/
private String host;
private String fileAbsolutePath;
/**
* port
*/
private int port;
private MultipartFile file;
/**
* zookeeper directory
*/
private String zkDirectory;
/**
* resource info
*/
private String resInfo;
/**
* create time
*/
private Date createTime;
/**
* last heart beat time
*/
private Date lastHeartbeatTime;
}

39
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/CreateFileFromContentDto.java Normal file
View File

@ -0,0 +1,39 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.dto.resources;
import org.apache.dolphinscheduler.dao.entity.User;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class CreateFileFromContentDto {
private User loginUser;
private String fileAbsolutePath;
private String fileContent;
}

40
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/CreateFileFromContentRequest.java Normal file
View File

@ -0,0 +1,40 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.dto.resources;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.spi.enums.ResourceType;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class CreateFileFromContentRequest {
private User loginUser;
private ResourceType type;
private String parentAbsoluteDirectory;
private String fileName;
private String fileContent;
}

41
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/CreateFileRequest.java Normal file
View File

@ -0,0 +1,41 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.dto.resources;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.spi.enums.ResourceType;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import org.springframework.web.multipart.MultipartFile;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class CreateFileRequest {
private User loginUser;
private ResourceType type;
private String parentAbsoluteDirectory;
private String fileName;
private MultipartFile file;
}

36
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/DeleteResourceDto.java Normal file
View File

@ -0,0 +1,36 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.dto.resources;
import org.apache.dolphinscheduler.dao.entity.User;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class DeleteResourceDto {
private User loginUser;
private String resourceAbsolutePath;
}

35
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/DeleteResourceRequest.java Normal file
View File

@ -0,0 +1,35 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.dto.resources;
import org.apache.dolphinscheduler.dao.entity.User;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
public class DeleteResourceRequest {
private User loginUser;
private String resourceAbsolutePath;
}

36
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/DownloadFileDto.java Normal file
View File

@ -0,0 +1,36 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.dto.resources;
import org.apache.dolphinscheduler.dao.entity.User;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class DownloadFileDto {
private User loginUser;
private String fileAbsolutePath;
}

37
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/DownloadFileRequest.java Normal file
View File

@ -0,0 +1,37 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.dto.resources;
import org.apache.dolphinscheduler.dao.entity.User;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
public class DownloadFileRequest {
private User loginUser;
private String fileAbsolutePath;
}

38
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/FetchFileContentDto.java Normal file
View File

@ -0,0 +1,38 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.dto.resources;
import org.apache.dolphinscheduler.dao.entity.User;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
public class FetchFileContentDto {
private User loginUser;
private String resourceFileAbsolutePath;
private int skipLineNum;
private int limit;
}

37
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/FetchFileContentRequest.java Normal file
View File

@ -0,0 +1,37 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.dto.resources;
import org.apache.dolphinscheduler.dao.entity.User;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class FetchFileContentRequest {
private User loginUser;
private String resourceFileAbsolutePath;
private int skipLineNum;
private int limit;
}

58
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/PagingResourceItemRequest.java Normal file
View File

@ -0,0 +1,58 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.dto.resources;
import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.exceptions.ServiceException;
import org.apache.dolphinscheduler.common.constants.Constants;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.spi.enums.ResourceType;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
public class PagingResourceItemRequest {
private User loginUser;
private String resourceAbsolutePath;
private ResourceType resourceType;
private String resourceNameKeyWord;
Integer pageNo;
Integer pageSize;
public void checkPageNoAndPageSize() {
if (pageNo <= 0) {
throw new ServiceException(Status.REQUEST_PARAMS_NOT_VALID_ERROR, Constants.PAGE_NUMBER);
}
if (pageSize <= 0) {
throw new ServiceException(Status.REQUEST_PARAMS_NOT_VALID_ERROR, Constants.PAGE_SIZE);
}
}
}

34
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/QueryResourceDto.java Normal file
View File

@ -0,0 +1,34 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.dto.resources;
import java.util.List;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class QueryResourceDto {
private List<String> resourceAbsolutePaths;
}

39
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/RenameDirectoryDto.java Normal file
View File

@ -0,0 +1,39 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.dto.resources;
import org.apache.dolphinscheduler.dao.entity.User;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class RenameDirectoryDto {
private User loginUser;
private String originDirectoryAbsolutePath;
private String targetDirectoryAbsolutePath;
}

39
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/RenameDirectoryRequest.java Normal file
View File

@ -0,0 +1,39 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.dto.resources;
import org.apache.dolphinscheduler.dao.entity.User;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class RenameDirectoryRequest {
private User loginUser;
private String directoryAbsolutePath;
private String newDirectoryName;
}

38
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/RenameFileDto.java Normal file
View File

@ -0,0 +1,38 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.dto.resources;
import org.apache.dolphinscheduler.dao.entity.User;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class RenameFileDto {
private User loginUser;
private String originFileAbsolutePath;
private String targetFileAbsolutePath;
}

39
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/RenameFileRequest.java Normal file
View File

@ -0,0 +1,39 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.dto.resources;
import org.apache.dolphinscheduler.dao.entity.User;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class RenameFileRequest {
private User loginUser;
private String fileAbsolutePath;
private String newFileName;
}

31
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/ResourceComponent.java
View File

@ -25,34 +25,13 @@ import java.util.List;
import lombok.Data;
import lombok.NoArgsConstructor;
import com.fasterxml.jackson.annotation.JsonPropertyOrder;
/**
* resource component
*/
@Data
@NoArgsConstructor
@JsonPropertyOrder({"id", "pid", "name", "fullName", "description", "isDirctory", "children", "type"})
public abstract class ResourceComponent {
public ResourceComponent(int id, String pid, String name, String fullName, String description, boolean isDirctory) {
this.id = id;
this.pid = pid;
this.name = name;
this.fullName = fullName;
this.isDirctory = isDirctory;
int directoryFlag = isDirctory ? 1 : 0;
this.idValue = String.format("%s_%s", id, directoryFlag);
}
/**
* id
*/
protected int id;
/**
* parent id
*/
protected String pid;
/**
* name
*/
@ -73,10 +52,7 @@ public abstract class ResourceComponent {
* is directory
*/
protected boolean isDirctory;
/**
* id value
*/
protected String idValue;
/**
* resoruce type
*/
@ -88,14 +64,11 @@ public abstract class ResourceComponent {
/**
* add resource component
*
* @param resourceComponent resource component
*/
public void add(ResourceComponent resourceComponent) {
children.add(resourceComponent);
}
public void setIdValue(int id, boolean isDirctory) {
int directoryFlag = isDirctory ? 1 : 0;
this.idValue = String.format("%s_%s", id, directoryFlag);
}
}

41
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/UpdateFileDto.java Normal file
View File

@ -0,0 +1,41 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.dto.resources;
import org.apache.dolphinscheduler.dao.entity.User;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import org.springframework.web.multipart.MultipartFile;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class UpdateFileDto {
private User loginUser;
private String fileAbsolutePath;
private MultipartFile file;
}

39
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/UpdateFileFromContentDto.java Normal file
View File

@ -0,0 +1,39 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.dto.resources;
import org.apache.dolphinscheduler.dao.entity.User;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class UpdateFileFromContentDto {
private User loginUser;
private String fileAbsolutePath;
private String fileContent;
}

39
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/UpdateFileFromContentRequest.java Normal file
View File

@ -0,0 +1,39 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.dto.resources;
import org.apache.dolphinscheduler.dao.entity.User;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class UpdateFileFromContentRequest {
private User loginUser;
private String fileAbsolutePath;
private String fileContent;
}

41
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/UpdateFileRequest.java Normal file
View File

@ -0,0 +1,41 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.dto.resources;
import org.apache.dolphinscheduler.dao.entity.User;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import org.springframework.web.multipart.MultipartFile;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class UpdateFileRequest {
private User loginUser;
private String fileAbsolutePath;
private MultipartFile file;
}

6
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/visitor/ResourceTreeVisitor.java
View File

@ -122,12 +122,8 @@ public class ResourceTreeVisitor implements Visitor {
tempResourceComponent = new FileLeaf();
}
tempResourceComponent.setName(resource.getAlias());
// tempResourceComponent.setFullName(resource.getFullName().replaceFirst("/",""));
tempResourceComponent.setName(resource.getFileName());
tempResourceComponent.setFullName(resource.getFullName());
tempResourceComponent.setId(resource.getId());
tempResourceComponent.setPid(resource.getPfullName());
tempResourceComponent.setIdValue(resource.getId(), resource.isDirectory());
tempResourceComponent.setType(resource.getType());
return tempResourceComponent;
}

4
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/task/TaskCreateRequest.java
View File

@ -19,11 +19,11 @@ package org.apache.dolphinscheduler.api.dto.task;
import static org.apache.dolphinscheduler.common.constants.Constants.VERSION_FIRST;
import org.apache.dolphinscheduler.common.constants.Constants;
import org.apache.dolphinscheduler.common.enums.Flag;
import org.apache.dolphinscheduler.common.enums.Priority;
import org.apache.dolphinscheduler.common.enums.TimeoutFlag;
import org.apache.dolphinscheduler.dao.entity.TaskDefinition;
import org.apache.dolphinscheduler.dao.utils.WorkerGroupUtils;
import org.apache.dolphinscheduler.plugin.task.api.enums.TaskTimeoutStrategy;
import java.util.Date;
@ -107,7 +107,7 @@ public class TaskCreateRequest {
taskDefinition.setProjectCode(this.projectCode);
taskDefinition.setTaskType(this.taskType);
taskDefinition.setTaskParams(this.taskParams);
taskDefinition.setWorkerGroup(this.workerGroup == null ? Constants.DEFAULT_WORKER_GROUP : this.workerGroup);
taskDefinition.setWorkerGroup(WorkerGroupUtils.getWorkerGroupOrDefault(workerGroup));
taskDefinition.setEnvironmentCode(this.environmentCode);
taskDefinition.setFailRetryTimes(this.failRetryTimes);
taskDefinition.setFailRetryInterval(this.failRetryInterval);

4
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/exceptions/ServiceException.java
View File

@ -52,4 +52,8 @@ public class ServiceException extends RuntimeException {
this.code = code;
}
public ServiceException(String message, Exception exception) {
this(Status.INTERNAL_SERVER_ERROR_ARGS, message, exception);
}
}

91
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/python/PythonGateway.java
View File

@ -31,14 +31,12 @@ import org.apache.dolphinscheduler.api.service.SchedulerService;
import org.apache.dolphinscheduler.api.service.TaskDefinitionService;
import org.apache.dolphinscheduler.api.service.TenantService;
import org.apache.dolphinscheduler.api.service.UsersService;
import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.constants.Constants;
import org.apache.dolphinscheduler.common.enums.ComplementDependentMode;
import org.apache.dolphinscheduler.common.enums.ExecutionOrder;
import org.apache.dolphinscheduler.common.enums.FailureStrategy;
import org.apache.dolphinscheduler.common.enums.Priority;
import org.apache.dolphinscheduler.common.enums.ProcessExecutionTypeEnum;
import org.apache.dolphinscheduler.common.enums.ProgramType;
import org.apache.dolphinscheduler.common.enums.ReleaseState;
import org.apache.dolphinscheduler.common.enums.RunMode;
import org.apache.dolphinscheduler.common.enums.TaskDependType;
@ -216,22 +214,22 @@ public class PythonGateway {
* If workflow do not exists in Project=`projectCode` would create a new one
* If workflow already exists in Project=`projectCode` would update it
*
* @param userName user name who create or update workflow
* @param projectName project name which workflow belongs to
* @param name workflow name
* @param description description
* @param globalParams global params
* @param schedule schedule for workflow, will not set schedule if null,
* and if would always fresh exists schedule if not null
* @param onlineSchedule Whether set the workflow's schedule to online state
* @param warningType warning type
* @param warningGroupId warning group id
* @param timeout timeout for workflow working, if running time longer than timeout,
* task will mark as fail
* @param workerGroup run task in which worker group
* @param taskRelationJson relation json for nodes
* @param userName user name who create or update workflow
* @param projectName project name which workflow belongs to
* @param name workflow name
* @param description description
* @param globalParams global params
* @param schedule schedule for workflow, will not set schedule if null,
* and if would always fresh exists schedule if not null
* @param onlineSchedule Whether set the workflow's schedule to online state
* @param warningType warning type
* @param warningGroupId warning group id
* @param timeout timeout for workflow working, if running time longer than timeout,
* task will mark as fail
* @param workerGroup run task in which worker group
* @param taskRelationJson relation json for nodes
* @param taskDefinitionJson taskDefinitionJson
* @param otherParamsJson otherParamsJson handle other params
* @param otherParamsJson otherParamsJson handle other params
* @return create result code
*/
public Long createOrUpdateWorkflow(String userName,
@ -300,8 +298,8 @@ public class PythonGateway {
/**
* get workflow
*
* @param user user who create or update schedule
* @param projectCode project which workflow belongs to
* @param user user who create or update schedule
* @param projectCode project which workflow belongs to
* @param workflowName workflow name
*/
private ProcessDefinition getWorkflow(User user, long projectCode, String workflowName) {
@ -327,13 +325,13 @@ public class PythonGateway {
* It would always use latest schedule define in workflow-as-code, and set schedule online when
* it's not null
*
* @param user user who create or update schedule
* @param projectCode project which workflow belongs to
* @param workflowCode workflow code
* @param schedule schedule expression
* @param user user who create or update schedule
* @param projectCode project which workflow belongs to
* @param workflowCode workflow code
* @param schedule schedule expression
* @param onlineSchedule Whether set the workflow's schedule to online state
* @param workerGroup work group
* @param warningType warning type
* @param workerGroup work group
* @param warningType warning type
* @param warningGroupId warning group id
*/
private void createOrUpdateSchedule(User user,
@ -512,7 +510,7 @@ public class PythonGateway {
* it will return the datasource match the type.
*
* @param datasourceName datasource name of datasource
* @param type datasource type
* @param type datasource type
*/
public DataSource getDatasource(String datasourceName, String type) {
@ -545,8 +543,8 @@ public class PythonGateway {
* Get workflow object by given workflow name. It returns map contain workflow id, name, code.
* Useful in Python API create subProcess task which need workflow information.
*
* @param userName user who create or update schedule
* @param projectName project name which workflow belongs to
* @param userName user who create or update schedule
* @param projectName project name which workflow belongs to
* @param workflowName workflow name
*/
public Map<String, Object> getWorkflowInfo(String userName, String projectName,
@ -577,9 +575,9 @@ public class PythonGateway {
* Get project, workflow, task code.
* Useful in Python API create dependent task which need workflow information.
*
* @param projectName project name which workflow belongs to
* @param projectName project name which workflow belongs to
* @param workflowName workflow name
* @param taskName task name
* @param taskName task name
*/
public Map<String, Object> getDependentInfo(String projectName, String workflowName, String taskName) {
Map<String, Object> result = new HashMap<>();
@ -614,25 +612,22 @@ public class PythonGateway {
* Get resource by given program type and full name. It returns map contain resource id, name.
* Useful in Python API create flink or spark task which need workflow information.
*
* @param programType program type one of SCALA, JAVA and PYTHON
* @param fullName full name of the resource
* @param fullName full name of the resource
*/
public Map<String, Object> getResourcesFileInfo(String programType, String fullName) {
public Map<String, Object> getResourcesFileInfo(String fullName) {
Map<String, Object> result = new HashMap<>();
Result<Object> resources = resourceService.queryResourceByProgramType(dummyAdminUser, ResourceType.FILE,
ProgramType.valueOf(programType));
List<ResourceComponent> resourcesComponent = (List<ResourceComponent>) resources.getData();
List<ResourceComponent> namedResources =
resourcesComponent.stream().filter(s -> fullName.equals(s.getFullName())).collect(Collectors.toList());
List<ResourceComponent> resourceComponents =
resourceService.queryResourceFiles(dummyAdminUser, ResourceType.FILE);
List<ResourceComponent> namedResources = resourceComponents.stream()
.filter(s -> fullName.equals(s.getFullName()))
.collect(Collectors.toList());
if (CollectionUtils.isEmpty(namedResources)) {
String msg =
String.format("Can not find valid resource by program type %s and name %s", programType, fullName);
String msg = String.format("Can not find valid resource by name %s", fullName);
log.error(msg);
throw new IllegalArgumentException(msg);
}
result.put("id", namedResources.get(0).getId());
result.put("name", namedResources.get(0).getName());
return result;
}
@ -671,20 +666,6 @@ public class PythonGateway {
return PythonGateway.class.getPackage().getImplementationVersion();
}
/**
* create or update resource.
* If the folder is not already created, it will be
*
* @param userName user who create or update resource
* @param fullName The fullname of resource.Includes path and suffix.
* @param resourceContent content of resource
* @return StorageEntity object which contains necessary information about resource
*/
public StorageEntity createOrUpdateResource(String userName, String fullName,
String resourceContent) throws Exception {
return resourceService.createOrUpdateResource(userName, fullName, resourceContent);
}
@PostConstruct
public void init() {
if (apiConfig.getPythonGateway().isEnabled()) {

201
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java
View File

@ -17,181 +17,99 @@
package org.apache.dolphinscheduler.api.service;
import org.apache.dolphinscheduler.api.dto.resources.DeleteDataTransferResponse;
import org.apache.dolphinscheduler.api.dto.resources.CreateDirectoryRequest;
import org.apache.dolphinscheduler.api.dto.resources.CreateFileFromContentRequest;
import org.apache.dolphinscheduler.api.dto.resources.CreateFileRequest;
import org.apache.dolphinscheduler.api.dto.resources.DeleteResourceRequest;
import org.apache.dolphinscheduler.api.dto.resources.DownloadFileRequest;
import org.apache.dolphinscheduler.api.dto.resources.FetchFileContentRequest;
import org.apache.dolphinscheduler.api.dto.resources.PagingResourceItemRequest;
import org.apache.dolphinscheduler.api.dto.resources.RenameDirectoryRequest;
import org.apache.dolphinscheduler.api.dto.resources.RenameFileRequest;
import org.apache.dolphinscheduler.api.dto.resources.ResourceComponent;
import org.apache.dolphinscheduler.api.dto.resources.UpdateFileFromContentRequest;
import org.apache.dolphinscheduler.api.dto.resources.UpdateFileRequest;
import org.apache.dolphinscheduler.api.utils.PageInfo;
import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.enums.ProgramType;
import org.apache.dolphinscheduler.api.vo.ResourceItemVO;
import org.apache.dolphinscheduler.api.vo.resources.FetchFileContentResponse;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.plugin.storage.api.StorageEntity;
import org.apache.dolphinscheduler.spi.enums.ResourceType;
import java.io.IOException;
import java.util.Map;
import java.util.List;
import org.springframework.web.multipart.MultipartFile;
import javax.servlet.http.HttpServletResponse;
/**
* resources service
*/
public interface ResourcesService {
/**
* create directory
*
* @param loginUser login user
* @param name alias
* @param type type
* @param pid parent id
* @param currentDir current directory
* @return create directory result
* Create a new directory in the resource storage, if the directory already exists will throw exception
*/
Result<Object> createDirectory(User loginUser,
String name,
ResourceType type,
int pid,
String currentDir);
void createDirectory(CreateDirectoryRequest createDirectoryRequest);
/**
* create resource
*
* @param loginUser login user
* @param name alias
* @param type type
* @param file file
* @param currentDir current directory
* @return create result code
* Rename the directory in the resource storage, if the origin directory not exists or the new directory already exists will throw exception.
* <p> If the origin directory is empty will only update the directory name.
* <p> If the origin directory is not empty will move all the files and directories to the new directory.
* <p> After update the origin directory will be deleted.
*/
Result<Object> uploadResource(User loginUser,
String name,
ResourceType type,
MultipartFile file,
String currentDir);
void renameDirectory(RenameDirectoryRequest renameDirectoryRequest);
/**
* update resource
* @param loginUser login user
* @param name name
* @param type resource type
* @param file resource file
* @return update result code
* Upload a new file to the resource storage, if the file already exists will throw exception
*/
Result<Object> updateResource(User loginUser,
String fullName,
String tenantCode,
String name,
ResourceType type,
MultipartFile file);
void createFile(CreateFileRequest createFileRequest);
/**
* query resources list paging
*
* @param loginUser login user
* @param type resource type
* @param searchVal search value
* @param pageNo page number
* @param pageSize page size
* @return resource list page
* Update the file in the resource storage, if the origin file not exists or the new file already exists will throw exception.
* <p> If the new file is empty will only update the file name.
* <p> If the new file is not empty will update the file content and name.
* <p> After update the origin file will be deleted.
*/
Result<PageInfo<StorageEntity>> queryResourceListPaging(User loginUser, String fullName, String resTenantCode,
ResourceType type, String searchVal, Integer pageNo,
Integer pageSize);
void updateFile(UpdateFileRequest updateFileRequest);
/**
* query resource list
*
* @param loginUser login user
* @param type resource type
* @return resource list
* Rename the file in the resource storage, if the origin file not exists or the new file already exists will throw exception.
*/
Map<String, Object> queryResourceList(User loginUser, ResourceType type, String fullName);
void renameFile(RenameFileRequest renameFileRequest);
/**
* query resource list by program type
*
* @param loginUser login user
* @param type resource type
* @return resource list
* Create a new file in the resource storage, if the file already exists will throw exception.
* Different with {@link ResourcesService#createFile(CreateFileRequest)} this method will create a new file with the given content.
*/
Result<Object> queryResourceByProgramType(User loginUser, ResourceType type, ProgramType programType);
void createFileFromContent(CreateFileFromContentRequest createFileFromContentRequest);
/**
* delete resource
*
* @param loginUser login user
* @return delete result code
* @throws IOException exception
* Update the file content.
*/
Result<Object> delete(User loginUser, String fullName, String tenantCode) throws IOException;
void updateFileFromContent(UpdateFileFromContentRequest updateFileContentRequest);
/**
* verify resource by name and type
* @param loginUser login user
* @param fullName resource full name
* @param type resource type
* @return true if the resource name not exists, otherwise return false
* Paging query resource items.
* <p>If the login user is not admin will only query the resource items that under the user's tenant.
* <p>If the login user is admin and {@link PagingResourceItemRequest##resourceAbsolutePath} is null will return all the resource items.
*/
Result<Object> verifyResourceName(String fullName, ResourceType type, User loginUser);
PageInfo<ResourceItemVO> pagingResourceItem(PagingResourceItemRequest pagingResourceItemRequest);
/**
* verify resource by file name
* @param fileName resource file name
* @param type resource type
* @return true if the resource file name, otherwise return false
* Query the resource file items by the given resource type and program type.
*/
Result<Object> queryResourceByFileName(User loginUser, String fileName, ResourceType type, String resTenantCode);
List<ResourceComponent> queryResourceFiles(User loginUser, ResourceType type);
/**
* view resource file online
*
* @param skipLineNum skip line number
* @param limit limit
* @param fullName fullName
* @return resource content
* Delete the resource item.
* <p>If the resource item is a directory will delete all the files and directories under the directory.
* <p>If the resource item is a file will delete the file.
* <p>If the resource item not exists will throw exception.
*/
Result<Object> readResource(User loginUser, String fullName, String tenantCode, int skipLineNum, int limit);
void delete(DeleteResourceRequest deleteResourceRequest);
/**
* create resource file online
*
* @param loginUser login user
* @param type resource type
* @param fileName file name
* @param fileSuffix file suffix
* @param content content
* @return create result code
* Fetch the file content.
*/
Result<Object> createResourceFile(User loginUser, ResourceType type, String fileName, String fileSuffix,
String content, String currentDirectory);
FetchFileContentResponse fetchResourceFileContent(FetchFileContentRequest fetchFileContentRequest);
/**
* create or update resource.
* If the folder is not already created, it will be ignored and directly create the new file
*
* @param userName user who create or update resource
* @param fullName The fullname of resource.Includes path and suffix.
* @param resourceContent content of resource
*/
StorageEntity createOrUpdateResource(String userName, String fullName, String resourceContent) throws Exception;
/**
* updateProcessInstance resource
*
* @param loginUser login user
* @param fullName full name
* @param tenantCode tenantCode
* @param content content
* @return update result cod
*/
Result<Object> updateResourceContent(User loginUser, String fullName, String tenantCode,
String content);
/**
* download file
*
* @return resource content
* @throws IOException exception
*/
org.springframework.core.io.Resource downloadResource(User loginUser, String fullName) throws IOException;
void downloadResource(HttpServletResponse response, DownloadFileRequest downloadFileRequest);
/**
* Get resource by given resource type and file name.
@ -202,21 +120,6 @@ public interface ResourcesService {
*/
StorageEntity queryFileStatus(String userName, String fileName) throws Exception;
/**
* delete DATA_TRANSFER data in resource center
*
* @param loginUser user who query resource
* @param days number of days
*/
DeleteDataTransferResponse deleteDataTransferData(User loginUser, Integer days);
/**
* get resource base dir
*
* @param loginUser login user
* @param type resource type
* @return
*/
Result<Object> queryResourceBaseDir(User loginUser, ResourceType type);
String queryResourceBaseDir(User loginUser, ResourceType type);
}

8
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/WorkerGroupService.java
View File

@ -18,7 +18,6 @@
package org.apache.dolphinscheduler.api.service;
import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.dao.entity.TaskInstance;
import org.apache.dolphinscheduler.dao.entity.User;
import java.util.List;
@ -77,13 +76,6 @@ public interface WorkerGroupService {
*/
Map<String, Object> getWorkerAddressList();
/**
* Get task instance's worker group
* @param taskInstance task instance
* @return worker group
*/
String getTaskWorkerGroup(TaskInstance taskInstance);
/**
* Query worker group by process definition codes
* @param processDefinitionCodeList processDefinitionCodeList

4
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ExecutorServiceImpl.java
View File

@ -89,8 +89,8 @@ import org.apache.dolphinscheduler.extract.master.dto.WorkflowExecuteDto;
import org.apache.dolphinscheduler.extract.master.transportor.StreamingTaskTriggerRequest;
import org.apache.dolphinscheduler.extract.master.transportor.StreamingTaskTriggerResponse;
import org.apache.dolphinscheduler.extract.master.transportor.WorkflowInstanceStateChangeEvent;
import org.apache.dolphinscheduler.plugin.task.api.TaskConstants;
import org.apache.dolphinscheduler.plugin.task.api.model.Property;
import org.apache.dolphinscheduler.plugin.task.api.utils.TaskTypeUtils;
import org.apache.dolphinscheduler.registry.api.enums.RegistryNodeType;
import org.apache.dolphinscheduler.service.command.CommandService;
import org.apache.dolphinscheduler.service.cron.CronUtils;
@ -361,7 +361,7 @@ public class ExecutorServiceImpl extends BaseServiceImpl implements ExecutorServ
// find out the process definition code
Set<Long> processDefinitionCodeSet = new HashSet<>();
taskDefinitions.stream()
.filter(task -> TaskConstants.TASK_TYPE_SUB_PROCESS.equalsIgnoreCase(task.getTaskType())).forEach(
.filter(task -> TaskTypeUtils.isSubWorkflowTask(task.getTaskType())).forEach(
taskDefinition -> processDefinitionCodeSet.add(Long.valueOf(
JSONUtils.getNodeString(taskDefinition.getTaskParams(),
CMD_PARAM_SUB_PROCESS_DEFINE_CODE))));

28
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java
View File

@ -34,13 +34,12 @@ import static org.apache.dolphinscheduler.api.enums.Status.PROCESS_DEFINE_NOT_EX
import static org.apache.dolphinscheduler.common.constants.CommandKeyConstants.CMD_PARAM_SUB_PROCESS_DEFINE_CODE;
import static org.apache.dolphinscheduler.common.constants.Constants.COPY_SUFFIX;
import static org.apache.dolphinscheduler.common.constants.Constants.DATA_LIST;
import static org.apache.dolphinscheduler.common.constants.Constants.DEFAULT_WORKER_GROUP;
import static org.apache.dolphinscheduler.common.constants.Constants.GLOBAL_PARAMS;
import static org.apache.dolphinscheduler.common.constants.Constants.IMPORT_SUFFIX;
import static org.apache.dolphinscheduler.common.constants.Constants.LOCAL_PARAMS;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.LOCAL_PARAMS_LIST;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_SQL;
import static org.apache.dolphinscheduler.plugin.task.api.TaskPluginManager.checkTaskParameters;
import org.apache.dolphinscheduler.api.dto.DagDataSchedule;
import org.apache.dolphinscheduler.api.dto.treeview.Instance;
@ -109,12 +108,13 @@ import org.apache.dolphinscheduler.dao.model.PageListingResult;
import org.apache.dolphinscheduler.dao.repository.ProcessDefinitionDao;
import org.apache.dolphinscheduler.dao.repository.ProcessDefinitionLogDao;
import org.apache.dolphinscheduler.dao.repository.TaskDefinitionLogDao;
import org.apache.dolphinscheduler.plugin.task.api.TaskPluginManager;
import org.apache.dolphinscheduler.dao.utils.WorkerGroupUtils;
import org.apache.dolphinscheduler.plugin.task.api.enums.SqlType;
import org.apache.dolphinscheduler.plugin.task.api.enums.TaskTimeoutStrategy;
import org.apache.dolphinscheduler.plugin.task.api.model.Property;
import org.apache.dolphinscheduler.plugin.task.api.parameters.ParametersNode;
import org.apache.dolphinscheduler.plugin.task.api.parameters.SqlParameters;
import org.apache.dolphinscheduler.plugin.task.api.utils.TaskTypeUtils;
import org.apache.dolphinscheduler.plugin.task.sql.SqlTaskChannelFactory;
import org.apache.dolphinscheduler.service.alert.ListenerEventAlertManager;
import org.apache.dolphinscheduler.service.model.TaskNode;
import org.apache.dolphinscheduler.service.process.ProcessService;
@ -421,11 +421,7 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro
throw new ServiceException(Status.DATA_IS_NOT_VALID, taskDefinitionJson);
}
for (TaskDefinitionLog taskDefinitionLog : taskDefinitionLogs) {
if (!TaskPluginManager.checkTaskParameters(ParametersNode.builder()
.taskType(taskDefinitionLog.getTaskType())
.taskParams(taskDefinitionLog.getTaskParams())
.dependence(taskDefinitionLog.getDependence())
.build())) {
if (!checkTaskParameters(taskDefinitionLog.getTaskType(), taskDefinitionLog.getTaskParams())) {
log.error(
"Generate task definition list failed, the given task definition parameter is invalided, taskName: {}, taskDefinition: {}",
taskDefinitionLog.getName(), taskDefinitionLog);
@ -1386,11 +1382,11 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro
sqlParameters.setLocalParams(Collections.emptyList());
taskDefinition.setTaskParams(JSONUtils.toJsonString(sqlParameters));
taskDefinition.setCode(CodeGenerateUtils.genCode());
taskDefinition.setTaskType(TASK_TYPE_SQL);
taskDefinition.setTaskType(SqlTaskChannelFactory.NAME);
taskDefinition.setFailRetryTimes(0);
taskDefinition.setFailRetryInterval(0);
taskDefinition.setTimeoutFlag(TimeoutFlag.CLOSE);
taskDefinition.setWorkerGroup(DEFAULT_WORKER_GROUP);
taskDefinition.setWorkerGroup(WorkerGroupUtils.getDefaultWorkerGroup());
taskDefinition.setTaskPriority(Priority.MEDIUM);
taskDefinition.setEnvironmentCode(-1);
taskDefinition.setTimeout(0);
@ -1615,13 +1611,7 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro
// check whether the process definition json is normal
for (TaskNode taskNode : taskNodes) {
if (!TaskPluginManager.checkTaskParameters(ParametersNode.builder()
.taskType(taskNode.getType())
.taskParams(taskNode.getTaskParams())
.dependence(taskNode.getDependence())
.switchResult(taskNode.getSwitchResult())
.build())) {
log.error("Task node {} parameter invalid.", taskNode.getName());
if (!checkTaskParameters(taskNode.getType(), taskNode.getParams())) {
putMsg(result, Status.PROCESS_NODE_S_PARAMETER_INVALID, taskNode.getName());
return result;
}
@ -1891,7 +1881,7 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro
long subProcessCode = 0L;
// if process is sub process, the return sub id, or sub id=0
if (taskInstance.isSubProcess()) {
if (TaskTypeUtils.isSubWorkflowTask(taskInstance.getTaskType())) {
TaskDefinition taskDefinition = taskDefinitionMap.get(taskInstance.getTaskCode());
subProcessCode = Long.parseLong(JSONUtils.parseObject(
taskDefinition.getTaskParams()).path(CMD_PARAM_SUB_PROCESS_DEFINE_CODE).asText());

20
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessInstanceServiceImpl.java
View File

@ -26,8 +26,7 @@ import static org.apache.dolphinscheduler.common.constants.Constants.GLOBAL_PARA
import static org.apache.dolphinscheduler.common.constants.Constants.LOCAL_PARAMS;
import static org.apache.dolphinscheduler.common.constants.Constants.PROCESS_INSTANCE_STATE;
import static org.apache.dolphinscheduler.common.constants.Constants.TASK_LIST;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_DEPENDENT;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_SUB_PROCESS;
import static org.apache.dolphinscheduler.plugin.task.api.TaskPluginManager.checkTaskParameters;
import org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant;
import org.apache.dolphinscheduler.api.dto.DynamicSubWorkflowDto;
@ -77,11 +76,10 @@ import org.apache.dolphinscheduler.dao.repository.ProcessInstanceDao;
import org.apache.dolphinscheduler.dao.repository.ProcessInstanceMapDao;
import org.apache.dolphinscheduler.dao.repository.TaskInstanceDao;
import org.apache.dolphinscheduler.dao.utils.WorkflowUtils;
import org.apache.dolphinscheduler.plugin.task.api.TaskPluginManager;
import org.apache.dolphinscheduler.plugin.task.api.enums.DependResult;
import org.apache.dolphinscheduler.plugin.task.api.model.Property;
import org.apache.dolphinscheduler.plugin.task.api.parameters.ParametersNode;
import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils;
import org.apache.dolphinscheduler.plugin.task.api.utils.TaskTypeUtils;
import org.apache.dolphinscheduler.service.expand.CuringParamsService;
import org.apache.dolphinscheduler.service.model.TaskNode;
import org.apache.dolphinscheduler.service.process.ProcessService;
@ -498,7 +496,7 @@ public class ProcessInstanceServiceImpl extends BaseServiceImpl implements Proce
throw new ServiceException(Status.TASK_INSTANCE_NOT_EXISTS, taskId);
}
if (!taskInstance.isDynamic()) {
if (!TaskTypeUtils.isDynamicTask(taskInstance.getTaskType())) {
putMsg(result, Status.TASK_INSTANCE_NOT_DYNAMIC_TASK, taskInstance.getName());
throw new ServiceException(Status.TASK_INSTANCE_NOT_EXISTS, taskId);
}
@ -548,7 +546,7 @@ public class ProcessInstanceServiceImpl extends BaseServiceImpl implements Proce
*/
private void addDependResultForTaskList(User loginUser, List<TaskInstance> taskInstanceList) throws IOException {
for (TaskInstance taskInstance : taskInstanceList) {
if (TASK_TYPE_DEPENDENT.equalsIgnoreCase(taskInstance.getTaskType())) {
if (TaskTypeUtils.isDependentTask(taskInstance.getTaskType())) {
log.info("DEPENDENT type task instance need to set dependent result, taskCode:{}, taskInstanceId:{}",
taskInstance.getTaskCode(), taskInstance.getId());
// TODO The result of dependent item should not be obtained from the log, waiting for optimization.
@ -628,9 +626,7 @@ public class ProcessInstanceServiceImpl extends BaseServiceImpl implements Proce
return result;
}
if (!taskInstance.isSubProcess()) {
log.warn("Task instance is not {} type instance, projectCode:{}, taskInstanceId:{}.",
TASK_TYPE_SUB_PROCESS, projectCode, taskId);
if (!TaskTypeUtils.isSubWorkflowTask(taskInstance.getTaskType())) {
putMsg(result, Status.TASK_INSTANCE_NOT_SUB_WORKFLOW_INSTANCE, taskInstance.getName());
return result;
}
@ -714,11 +710,7 @@ public class ProcessInstanceServiceImpl extends BaseServiceImpl implements Proce
return result;
}
for (TaskDefinitionLog taskDefinitionLog : taskDefinitionLogs) {
if (!TaskPluginManager.checkTaskParameters(ParametersNode.builder()
.taskType(taskDefinitionLog.getTaskType())
.taskParams(taskDefinitionLog.getTaskParams())
.dependence(taskDefinitionLog.getDependence())
.build())) {
if (!checkTaskParameters(taskDefinitionLog.getTaskType(), taskDefinitionLog.getTaskParams())) {
log.error("Task parameters are invalid, taskDefinitionName:{}.", taskDefinitionLog.getName());
putMsg(result, Status.PROCESS_NODE_S_PARAMETER_INVALID, taskDefinitionLog.getName());
return result;

10
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessTaskRelationServiceImpl.java
View File

@ -18,9 +18,6 @@
package org.apache.dolphinscheduler.api.service.impl;
import static java.util.stream.Collectors.toSet;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_CONDITIONS;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_DEPENDENT;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_SUB_PROCESS;
import org.apache.dolphinscheduler.api.dto.taskRelation.TaskRelationCreateRequest;
import org.apache.dolphinscheduler.api.dto.taskRelation.TaskRelationFilterRequest;
@ -46,6 +43,7 @@ import org.apache.dolphinscheduler.dao.mapper.ProcessTaskRelationMapper;
import org.apache.dolphinscheduler.dao.mapper.ProjectMapper;
import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionLogMapper;
import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionMapper;
import org.apache.dolphinscheduler.plugin.task.api.utils.TaskTypeUtils;
import org.apache.dolphinscheduler.service.process.ProcessService;
import org.apache.commons.collections4.CollectionUtils;
@ -354,9 +352,9 @@ public class ProcessTaskRelationServiceImpl extends BaseServiceImpl implements P
}
updateProcessDefiniteVersion(loginUser, result, processDefinition);
updateRelation(loginUser, result, processDefinition, processTaskRelationList);
if (TASK_TYPE_CONDITIONS.equals(taskDefinition.getTaskType())
|| TASK_TYPE_DEPENDENT.equals(taskDefinition.getTaskType())
|| TASK_TYPE_SUB_PROCESS.equals(taskDefinition.getTaskType())) {
if (TaskTypeUtils.isConditionTask(taskDefinition.getTaskType())
|| TaskTypeUtils.isSubWorkflowTask(taskDefinition.getTaskType())
|| TaskTypeUtils.isDependentTask(taskDefinition.getTaskType())) {
int deleteTaskDefinition = taskDefinitionMapper.deleteByCode(taskCode);
if (0 == deleteTaskDefinition) {
log.error("Delete task definition error, taskDefinitionCode:{}.", taskCode);

3
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectWorkerGroupRelationServiceImpl.java
View File

@ -32,6 +32,7 @@ import org.apache.dolphinscheduler.dao.mapper.ProjectWorkerGroupMapper;
import org.apache.dolphinscheduler.dao.mapper.ScheduleMapper;
import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionMapper;
import org.apache.dolphinscheduler.dao.mapper.WorkerGroupMapper;
import org.apache.dolphinscheduler.dao.utils.WorkerGroupUtils;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections4.SetUtils;
@ -118,7 +119,7 @@ public class ProjectWorkerGroupRelationServiceImpl extends BaseServiceImpl
workerGroupMapper.queryAllWorkerGroup().stream().map(WorkerGroup::getName).collect(
Collectors.toSet());
workerGroupNames.add(Constants.DEFAULT_WORKER_GROUP);
workerGroupNames.add(WorkerGroupUtils.getDefaultWorkerGroup());
Set<String> assignedWorkerGroupNames = new HashSet<>(workerGroups);

1522
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java
View File

File diff suppressed because it is too large Load Diff

30
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskDefinitionServiceImpl.java
View File

@ -24,6 +24,7 @@ import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationCon
import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.TASK_VERSION_VIEW;
import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.WORKFLOW_DEFINITION;
import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.WORKFLOW_SWITCH_TO_THIS_VERSION;
import static org.apache.dolphinscheduler.plugin.task.api.TaskPluginManager.checkTaskParameters;
import org.apache.dolphinscheduler.api.dto.task.TaskCreateRequest;
import org.apache.dolphinscheduler.api.dto.task.TaskFilterRequest;
@ -67,8 +68,6 @@ import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionLogMapper;
import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionMapper;
import org.apache.dolphinscheduler.dao.repository.ProcessTaskRelationLogDao;
import org.apache.dolphinscheduler.dao.repository.TaskDefinitionDao;
import org.apache.dolphinscheduler.plugin.task.api.TaskPluginManager;
import org.apache.dolphinscheduler.plugin.task.api.parameters.ParametersNode;
import org.apache.dolphinscheduler.service.process.ProcessService;
import org.apache.commons.collections4.CollectionUtils;
@ -167,11 +166,7 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe
return result;
}
for (TaskDefinitionLog taskDefinitionLog : taskDefinitionLogs) {
if (!TaskPluginManager.checkTaskParameters(ParametersNode.builder()
.taskType(taskDefinitionLog.getTaskType())
.taskParams(taskDefinitionLog.getTaskParams())
.dependence(taskDefinitionLog.getDependence())
.build())) {
if (!checkTaskParameters(taskDefinitionLog.getTaskType(), taskDefinitionLog.getTaskParams())) {
log.warn("Task definition {} parameters are invalid.", taskDefinitionLog.getName());
putMsg(result, Status.PROCESS_NODE_S_PARAMETER_INVALID, taskDefinitionLog.getName());
return result;
@ -208,11 +203,7 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe
Project project = projectMapper.queryByCode(taskDefinition.getProjectCode());
projectService.checkProjectAndAuthThrowException(user, project, permissions);
if (!TaskPluginManager.checkTaskParameters(ParametersNode.builder()
.taskType(taskDefinition.getTaskType())
.taskParams(taskDefinition.getTaskParams())
.dependence(taskDefinition.getDependence())
.build())) {
if (!checkTaskParameters(taskDefinition.getTaskType(), taskDefinition.getTaskParams())) {
throw new ServiceException(Status.PROCESS_NODE_S_PARAMETER_INVALID, taskDefinition.getName());
}
}
@ -321,12 +312,7 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe
putMsg(result, Status.DATA_IS_NOT_VALID, taskDefinitionJsonObj);
return result;
}
if (!TaskPluginManager.checkTaskParameters(ParametersNode.builder()
.taskType(taskDefinition.getTaskType())
.taskParams(taskDefinition.getTaskParams())
.dependence(taskDefinition.getDependence())
.build())) {
log.error("Task definition {} parameters are invalid", taskDefinition.getName());
if (!checkTaskParameters(taskDefinition.getTaskType(), taskDefinition.getTaskParams())) {
putMsg(result, Status.PROCESS_NODE_S_PARAMETER_INVALID, taskDefinition.getName());
return result;
}
@ -732,13 +718,7 @@ public class TaskDefinitionServiceImpl extends BaseServiceImpl implements TaskDe
putMsg(result, Status.DATA_IS_NOT_VALID, taskDefinitionJsonObj);
return null;
}
if (!TaskPluginManager.checkTaskParameters(ParametersNode.builder()
.taskType(taskDefinitionToUpdate.getTaskType())
.taskParams(taskDefinitionToUpdate.getTaskParams())
.dependence(taskDefinitionToUpdate.getDependence())
.build())) {
log.warn("Task definition parameters are invalid, taskDefinitionName:{}.",
taskDefinitionToUpdate.getName());
if (!checkTaskParameters(taskDefinitionToUpdate.getTaskType(), taskDefinitionToUpdate.getTaskParams())) {
putMsg(result, Status.PROCESS_NODE_S_PARAMETER_INVALID, taskDefinitionToUpdate.getName());
return null;
}

14
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TenantServiceImpl.java
View File

@ -39,7 +39,7 @@ import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper;
import org.apache.dolphinscheduler.dao.mapper.ScheduleMapper;
import org.apache.dolphinscheduler.dao.mapper.TenantMapper;
import org.apache.dolphinscheduler.dao.mapper.UserMapper;
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate;
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
@ -84,7 +84,7 @@ public class TenantServiceImpl extends BaseServiceImpl implements TenantService
private QueueService queueService;
@Autowired(required = false)
private StorageOperate storageOperate;
private StorageOperator storageOperator;
/**
* Check the tenant new object valid or not
@ -136,14 +136,13 @@ public class TenantServiceImpl extends BaseServiceImpl implements TenantService
* @param queueId queue id
* @param desc description
* @return create result code
* @throws Exception exception
*/
@Override
@Transactional(rollbackFor = Exception.class)
public Tenant createTenant(User loginUser,
String tenantCode,
int queueId,
String desc) throws Exception {
String desc) {
if (!canOperatorPermissions(loginUser, null, AuthorizationType.TENANT, TENANT_CREATE)) {
throw new ServiceException(Status.USER_NO_OPERATION_PERM);
}
@ -154,7 +153,6 @@ public class TenantServiceImpl extends BaseServiceImpl implements TenantService
createTenantValid(tenant);
tenantMapper.insert(tenant);
storageOperate.createTenantDirIfNotExists(tenantCode);
return tenant;
}
@ -209,11 +207,6 @@ public class TenantServiceImpl extends BaseServiceImpl implements TenantService
updateTenantValid(existsTenant, updateTenant);
updateTenant.setCreateTime(existsTenant.getCreateTime());
// updateProcessInstance tenant
// if the tenant code is modified, the original resource needs to be copied to the new tenant.
if (!Objects.equals(existsTenant.getTenantCode(), updateTenant.getTenantCode())) {
storageOperate.createTenantDirIfNotExists(tenantCode);
}
int update = tenantMapper.updateById(updateTenant);
if (update <= 0) {
throw new ServiceException(Status.UPDATE_TENANT_ERROR);
@ -262,7 +255,6 @@ public class TenantServiceImpl extends BaseServiceImpl implements TenantService
}
processInstanceMapper.updateProcessInstanceByTenantCode(tenant.getTenantCode(), Constants.DEFAULT);
storageOperate.deleteTenant(tenant.getTenantCode());
}
private List<ProcessInstance> getProcessInstancesByTenant(Tenant tenant) {

14
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UdfFuncServiceImpl.java
View File

@ -28,11 +28,10 @@ import org.apache.dolphinscheduler.dao.entity.UdfFunc;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.mapper.UDFUserMapper;
import org.apache.dolphinscheduler.dao.mapper.UdfFuncMapper;
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate;
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator;
import org.apache.commons.lang3.StringUtils;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
@ -62,7 +61,7 @@ public class UdfFuncServiceImpl extends BaseServiceImpl implements UdfFuncServic
private UDFUserMapper udfUserMapper;
@Autowired(required = false)
private StorageOperate storageOperate;
private StorageOperator storageOperator;
/**
* create udf function
@ -107,12 +106,7 @@ public class UdfFuncServiceImpl extends BaseServiceImpl implements UdfFuncServic
return result;
}
Boolean existResource = false;
try {
existResource = storageOperate.exists(fullName);
} catch (IOException e) {
log.error("Check resource error: {}", fullName, e);
}
boolean existResource = storageOperator.exists(fullName);
if (!existResource) {
log.error("resource full name {} is not exist", fullName);
@ -241,7 +235,7 @@ public class UdfFuncServiceImpl extends BaseServiceImpl implements UdfFuncServic
Boolean doesResExist = false;
try {
doesResExist = storageOperate.exists(fullName);
doesResExist = storageOperator.exists(fullName);
} catch (Exception e) {
log.error("udf resource :{} checking error", fullName, e);
result.setCode(Status.RESOURCE_NOT_EXIST.getCode());

56
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UsersServiceImpl.java
View File

@ -19,7 +19,6 @@ package org.apache.dolphinscheduler.api.service.impl;
import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.USER_MANAGER;
import org.apache.dolphinscheduler.api.dto.resources.ResourceComponent;
import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.exceptions.ServiceException;
import org.apache.dolphinscheduler.api.service.MetricsCleanUpService;
@ -50,7 +49,7 @@ import org.apache.dolphinscheduler.dao.mapper.ProjectUserMapper;
import org.apache.dolphinscheduler.dao.mapper.TenantMapper;
import org.apache.dolphinscheduler.dao.mapper.UDFUserMapper;
import org.apache.dolphinscheduler.dao.mapper.UserMapper;
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate;
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
@ -110,7 +109,7 @@ public class UsersServiceImpl extends BaseServiceImpl implements UsersService {
private ProjectMapper projectMapper;
@Autowired(required = false)
private StorageOperate storageOperate;
private StorageOperator storageOperator;
@Autowired
private K8sNamespaceUserMapper k8sNamespaceUserMapper;
@ -171,9 +170,6 @@ public class UsersServiceImpl extends BaseServiceImpl implements UsersService {
User user = createUser(userName, userPassword, email, tenantId, phone, queue, state);
Tenant tenant = tenantMapper.queryById(tenantId);
storageOperate.createTenantDirIfNotExists(tenant.getTenantCode());
log.info("User is created and id is {}.", user.getId());
result.put(Constants.DATA_LIST, user);
putMsg(result, Status.SUCCESS);
@ -1128,54 +1124,6 @@ public class UsersServiceImpl extends BaseServiceImpl implements UsersService {
return msg;
}
/**
* copy resource files
* xxx unchecked
*
* @param resourceComponent resource component
* @param srcBasePath src base path
* @param dstBasePath dst base path
* @throws IOException io exception
*/
private void copyResourceFiles(String oldTenantCode, String newTenantCode, ResourceComponent resourceComponent,
String srcBasePath, String dstBasePath) {
List<ResourceComponent> components = resourceComponent.getChildren();
try {
if (CollectionUtils.isNotEmpty(components)) {
for (ResourceComponent component : components) {
// verify whether exist
if (!storageOperate.exists(
String.format(Constants.FORMAT_S_S, srcBasePath, component.getFullName()))) {
log.error("Resource file: {} does not exist, copy error.", component.getFullName());
throw new ServiceException(Status.RESOURCE_NOT_EXIST);
}
if (!component.isDirctory()) {
// copy it to dst
storageOperate.copy(String.format(Constants.FORMAT_S_S, srcBasePath, component.getFullName()),
String.format(Constants.FORMAT_S_S, dstBasePath, component.getFullName()), false, true);
continue;
}
if (CollectionUtils.isEmpty(component.getChildren())) {
// if not exist,need create it
if (!storageOperate
.exists(String.format(Constants.FORMAT_S_S, dstBasePath, component.getFullName()))) {
storageOperate.mkdir(newTenantCode,
String.format(Constants.FORMAT_S_S, dstBasePath, component.getFullName()));
}
} else {
copyResourceFiles(oldTenantCode, newTenantCode, component, srcBasePath, dstBasePath);
}
}
}
} catch (IOException e) {
log.error("copy the resources failed,the error message is {}", e.getMessage());
}
}
/**
* registry user, default state is 0, default tenant_id is 1, no phone, no queue
*

17
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkFlowLineageServiceImpl.java
View File

@ -17,8 +17,6 @@
package org.apache.dolphinscheduler.api.service.impl;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_DEPENDENT;
import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.exceptions.ServiceException;
import org.apache.dolphinscheduler.api.service.WorkFlowLineageService;
@ -39,6 +37,7 @@ import org.apache.dolphinscheduler.dao.mapper.WorkFlowLineageMapper;
import org.apache.dolphinscheduler.plugin.task.api.model.DependentItem;
import org.apache.dolphinscheduler.plugin.task.api.model.DependentTaskModel;
import org.apache.dolphinscheduler.plugin.task.api.parameters.DependentParameters;
import org.apache.dolphinscheduler.plugin.task.api.utils.TaskTypeUtils;
import org.apache.commons.lang3.StringUtils;
@ -147,7 +146,7 @@ public class WorkFlowLineageServiceImpl extends BaseServiceImpl implements WorkF
List<Long> processDefinitionCodes) {
for (DependentProcessDefinition dependentProcessDefinition : dependentDefinitionList) {
for (DependentTaskModel dependentTaskModel : dependentProcessDefinition.getDependentParameters()
.getDependTaskList()) {
.getDependence().getDependTaskList()) {
for (DependentItem dependentItem : dependentTaskModel.getDependItemList()) {
if (!processDefinitionCodes.contains(dependentItem.getDefinitionCode())) {
processDefinitionCodes.add(dependentItem.getDefinitionCode());
@ -220,12 +219,12 @@ public class WorkFlowLineageServiceImpl extends BaseServiceImpl implements WorkF
List<TaskDefinitionLog> taskDefinitionLogs = taskDefinitionLogMapper.queryByTaskDefinitions(taskDefinitionList);
for (TaskDefinitionLog taskDefinitionLog : taskDefinitionLogs) {
if (taskDefinitionLog.getProjectCode() == projectCode) {
if (taskDefinitionLog.getTaskType().equals(TASK_TYPE_DEPENDENT)) {
if (TaskTypeUtils.isDependentTask(taskDefinitionLog.getTaskType())) {
DependentParameters dependentParameters =
JSONUtils.parseObject(taskDefinitionLog.getDependence(), DependentParameters.class);
if (dependentParameters != null) {
List<DependentTaskModel> dependTaskList =
dependentParameters.getDependTaskList();
dependentParameters.getDependence().getDependTaskList();
if (!CollectionUtils.isEmpty(dependTaskList)) {
for (DependentTaskModel taskModel : dependTaskList) {
List<DependentItem> dependItemList = taskModel.getDependItemList();
@ -247,9 +246,9 @@ public class WorkFlowLineageServiceImpl extends BaseServiceImpl implements WorkF
/**
* Query and return tasks dependence with string format, is a wrapper of queryTaskDepOnTask and task query method.
*
* @param projectCode Project code want to query tasks dependence
* @param projectCode Project code want to query tasks dependence
* @param processDefinitionCode Process definition code want to query tasks dependence
* @param taskCode Task code want to query tasks dependence
* @param taskCode Task code want to query tasks dependence
* @return Optional of formatter message
*/
@Override
@ -271,7 +270,7 @@ public class WorkFlowLineageServiceImpl extends BaseServiceImpl implements WorkF
/**
* Query tasks depend on process definition, include upstream or downstream
*
* @param projectCode Project code want to query tasks dependence
* @param projectCode Project code want to query tasks dependence
* @param processDefinitionCode Process definition code want to query tasks dependence
* @return Set of TaskMainInfo
*/
@ -291,7 +290,7 @@ public class WorkFlowLineageServiceImpl extends BaseServiceImpl implements WorkF
* Query downstream tasks depend on a process definition or a task
*
* @param processDefinitionCode Process definition code want to query tasks dependence
* @param taskCode Task code want to query tasks dependence
* @param taskCode Task code want to query tasks dependence
* @return downstream dependent tasks
*/
@Override

27
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkerGroupServiceImpl.java
View File

@ -32,7 +32,6 @@ import org.apache.dolphinscheduler.dao.entity.EnvironmentWorkerGroupRelation;
import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
import org.apache.dolphinscheduler.dao.entity.Schedule;
import org.apache.dolphinscheduler.dao.entity.TaskDefinition;
import org.apache.dolphinscheduler.dao.entity.TaskInstance;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.entity.WorkerGroup;
import org.apache.dolphinscheduler.dao.mapper.EnvironmentWorkerGroupRelationMapper;
@ -41,6 +40,7 @@ import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper;
import org.apache.dolphinscheduler.dao.mapper.ScheduleMapper;
import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionMapper;
import org.apache.dolphinscheduler.dao.mapper.WorkerGroupMapper;
import org.apache.dolphinscheduler.dao.utils.WorkerGroupUtils;
import org.apache.dolphinscheduler.registry.api.RegistryClient;
import org.apache.dolphinscheduler.registry.api.enums.RegistryNodeType;
import org.apache.dolphinscheduler.service.process.ProcessService;
@ -357,12 +357,12 @@ public class WorkerGroupServiceImpl extends BaseServiceImpl implements WorkerGro
workerGroups = workerGroupMapper.queryAllWorkerGroup();
}
boolean containDefaultWorkerGroups = workerGroups.stream()
.anyMatch(workerGroup -> Constants.DEFAULT_WORKER_GROUP.equals(workerGroup.getName()));
.anyMatch(workerGroup -> WorkerGroupUtils.isWorkerGroupEmpty(workerGroup.getName()));
if (!containDefaultWorkerGroups) {
// there doesn't exist a default WorkerGroup, we will add all worker to the default worker group.
Set<String> activeWorkerNodes = registryClient.getServerNodeSet(RegistryNodeType.WORKER);
WorkerGroup defaultWorkerGroup = new WorkerGroup();
defaultWorkerGroup.setName(Constants.DEFAULT_WORKER_GROUP);
defaultWorkerGroup.setName(WorkerGroupUtils.getDefaultWorkerGroup());
defaultWorkerGroup.setAddrList(String.join(Constants.COMMA, activeWorkerNodes));
defaultWorkerGroup.setCreateTime(new Date());
defaultWorkerGroup.setUpdateTime(new Date());
@ -431,27 +431,6 @@ public class WorkerGroupServiceImpl extends BaseServiceImpl implements WorkerGro
return result;
}
@Override
public String getTaskWorkerGroup(TaskInstance taskInstance) {
if (taskInstance == null) {
return null;
}
String workerGroup = taskInstance.getWorkerGroup();
if (StringUtils.isNotEmpty(workerGroup)) {
return workerGroup;
}
int processInstanceId = taskInstance.getProcessInstanceId();
ProcessInstance processInstance = processService.findProcessInstanceById(processInstanceId);
if (processInstance != null) {
return processInstance.getWorkerGroup();
}
log.info("task : {} will use default worker group", taskInstance.getId());
return Constants.DEFAULT_WORKER_GROUP;
}
@Override
public Map<Long, String> queryWorkerGroupByProcessDefinitionCodes(List<Long> processDefinitionCodeList) {
List<Schedule> processDefinitionScheduleList =

4
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/PageInfo.java
View File

@ -20,12 +20,16 @@ package org.apache.dolphinscheduler.api.utils;
import java.util.Collections;
import java.util.List;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.Setter;
import com.baomidou.mybatisplus.core.metadata.IPage;
@Data
@Builder
@AllArgsConstructor
public class PageInfo<T> {
/**

24
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/ITransformer.java Normal file
View File

@ -0,0 +1,24 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.validator;
public interface ITransformer<T, R> {
R transform(T t);
}

24
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/IValidator.java Normal file
View File

@ -0,0 +1,24 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.validator;
public interface IValidator<T> {
void validate(T t);
}

56
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/AbstractResourceTransformer.java Normal file
View File

@ -0,0 +1,56 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.validator.resource;
import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.exceptions.ServiceException;
import org.apache.dolphinscheduler.api.validator.ITransformer;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.repository.TenantDao;
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator;
import org.apache.dolphinscheduler.spi.enums.ResourceType;
import org.apache.commons.lang3.StringUtils;
import lombok.AllArgsConstructor;
@AllArgsConstructor
public abstract class AbstractResourceTransformer<T, R> implements ITransformer<T, R> {
protected TenantDao tenantDao;
protected StorageOperator storageOperator;
protected String getParentDirectoryAbsolutePath(User loginUser, String parentAbsoluteDirectory, ResourceType type) {
String tenantCode = tenantDao.queryOptionalById(loginUser.getTenantId())
.orElseThrow(() -> new ServiceException(Status.CURRENT_LOGIN_USER_TENANT_NOT_EXIST))
.getTenantCode();
String userResRootPath = storageOperator.getStorageBaseDirectory(tenantCode, type);
// If the parent directory is / then will transform to userResRootPath
// This only happens when the front-end go into the resource page first
// todo: we need to change the front-end logic to avoid this
if (parentAbsoluteDirectory.equals("/")) {
return userResRootPath;
}
if (!StringUtils.startsWith(parentAbsoluteDirectory, userResRootPath)) {
throw new ServiceException(Status.ILLEGAL_RESOURCE_PATH, parentAbsoluteDirectory);
}
return parentAbsoluteDirectory;
}
}

138
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/AbstractResourceValidator.java Normal file
View File

@ -0,0 +1,138 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.validator.resource;
import org.apache.dolphinscheduler.api.dto.resources.AbstractResourceDto;
import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.exceptions.ServiceException;
import org.apache.dolphinscheduler.api.validator.IValidator;
import org.apache.dolphinscheduler.common.enums.UserType;
import org.apache.dolphinscheduler.common.utils.FileUtils;
import org.apache.dolphinscheduler.dao.entity.Tenant;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.repository.TenantDao;
import org.apache.dolphinscheduler.plugin.storage.api.ResourceMetadata;
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator;
import org.apache.commons.lang3.StringUtils;
import java.util.Arrays;
import java.util.HashSet;
import java.util.Set;
import org.springframework.web.multipart.MultipartFile;
import com.google.common.io.Files;
public abstract class AbstractResourceValidator<T> implements IValidator<T> {
private static final Set<String> FILE_SUFFIXES_WHICH_CAN_FETCH_CONTENT = new HashSet<>(Arrays.asList(
StringUtils.defaultIfBlank(FileUtils.getResourceViewSuffixes(), "").split(",")));
protected final StorageOperator storageOperator;
private final TenantDao tenantDao;
public AbstractResourceValidator(StorageOperator storageOperator, TenantDao tenantDao) {
this.storageOperator = storageOperator;
this.tenantDao = tenantDao;
}
public void exceptionResourceAbsolutePathInvalidated(String resourceAbsolutePath) {
if (StringUtils.isBlank(resourceAbsolutePath)) {
throw new ServiceException("The resource path is null");
}
if (!resourceAbsolutePath.startsWith(storageOperator.getStorageBaseDirectory())) {
throw new ServiceException("Invalidated resource path: " + resourceAbsolutePath);
}
if (resourceAbsolutePath.contains("..")) {
throw new ServiceException("Invalidated resource path: " + resourceAbsolutePath);
}
}
public void exceptionFileInvalidated(MultipartFile file) {
if (file == null) {
throw new ServiceException("The file is null");
}
}
public void exceptionFileContentInvalidated(String fileContent) {
if (StringUtils.isEmpty(fileContent)) {
throw new ServiceException("The file content is null");
}
}
public void exceptionFileContentCannotFetch(String fileAbsolutePath) {
String fileExtension = Files.getFileExtension(fileAbsolutePath);
if (!FILE_SUFFIXES_WHICH_CAN_FETCH_CONTENT.contains(fileExtension)) {
throw new ServiceException("The file type: " + fileExtension + " cannot be fetched");
}
}
public void exceptionResourceNotExists(String resourceAbsolutePath) {
if (!storageOperator.exists(resourceAbsolutePath)) {
throw new ServiceException("Thr resource is not exists: " + resourceAbsolutePath);
}
}
public void exceptionResourceExists(String resourceAbsolutePath) {
if (storageOperator.exists(resourceAbsolutePath)) {
throw new ServiceException("The resource is already exist: " + resourceAbsolutePath);
}
}
public void exceptionResourceIsNotDirectory(String resourceAbsolutePath) {
if (StringUtils.isNotEmpty(Files.getFileExtension(resourceAbsolutePath))) {
throw new ServiceException("The path is not a directory: " + resourceAbsolutePath);
}
}
public void exceptionResourceIsNotFile(String fileAbsolutePath) {
if (StringUtils.isEmpty(Files.getFileExtension(fileAbsolutePath))) {
throw new ServiceException("The path is not a file: " + fileAbsolutePath);
}
}
public void exceptionUserNoResourcePermission(User user, AbstractResourceDto resourceDto) {
exceptionUserNoResourcePermission(user, resourceDto.getResourceAbsolutePath());
}
public void exceptionUserNoResourcePermission(User user, String resourceAbsolutePath) {
if (user.getUserType() == UserType.ADMIN_USER) {
return;
}
// check if the user have resource tenant permission
// Parse the resource path to get the tenant code
ResourceMetadata resourceMetaData = storageOperator.getResourceMetaData(resourceAbsolutePath);
if (!resourceAbsolutePath.startsWith(resourceMetaData.getResourceBaseDirectory())) {
throw new ServiceException("Invalidated resource path: " + resourceAbsolutePath);
}
// todo: inject the tenant when login
Tenant tenant = tenantDao.queryOptionalById(user.getTenantId())
.orElseThrow(() -> new ServiceException(Status.TENANT_NOT_EXIST, user.getTenantId()));
String userTenant = tenant.getTenantCode();
if (!userTenant.equals(resourceMetaData.getTenant())) {
throw new ServiceException(
"The user's tenant is " + userTenant + " have no permission to access the resource: "
+ resourceAbsolutePath);
}
}
}

50
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/CreateDirectoryDtoValidator.java Normal file
View File

@ -0,0 +1,50 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.validator.resource;
import org.apache.dolphinscheduler.api.dto.resources.CreateDirectoryDto;
import org.apache.dolphinscheduler.api.exceptions.ServiceException;
import org.apache.dolphinscheduler.dao.repository.TenantDao;
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator;
import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Component;
import com.google.common.io.Files;
@Component
public class CreateDirectoryDtoValidator extends AbstractResourceValidator<CreateDirectoryDto> {
public CreateDirectoryDtoValidator(StorageOperator storageOperator, TenantDao tenantDao) {
super(storageOperator, tenantDao);
}
@Override
public void validate(CreateDirectoryDto createDirectoryDto) {
String directoryAbsolutePath = createDirectoryDto.getDirectoryAbsolutePath();
exceptionResourceAbsolutePathInvalidated(directoryAbsolutePath);
exceptionResourceExists(directoryAbsolutePath);
exceptionUserNoResourcePermission(createDirectoryDto.getLoginUser(), directoryAbsolutePath);
exceptionResourceIsNotDirectory(directoryAbsolutePath);
if (StringUtils.isNotEmpty(Files.getFileExtension(directoryAbsolutePath))) {
throw new ServiceException("The path is not a directory: " + directoryAbsolutePath);
}
}
}

87
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/CreateDirectoryRequestTransformer.java Normal file
View File

@ -0,0 +1,87 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.validator.resource;
import static com.google.common.base.Preconditions.checkNotNull;
import org.apache.dolphinscheduler.api.dto.resources.CreateDirectoryDto;
import org.apache.dolphinscheduler.api.dto.resources.CreateDirectoryRequest;
import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.exceptions.ServiceException;
import org.apache.dolphinscheduler.api.validator.ITransformer;
import org.apache.dolphinscheduler.common.utils.FileUtils;
import org.apache.dolphinscheduler.dao.repository.TenantDao;
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
@Component
public class CreateDirectoryRequestTransformer implements ITransformer<CreateDirectoryRequest, CreateDirectoryDto> {
@Autowired
private TenantDao tenantDao;
@Autowired
private StorageOperator storageOperator;
@Override
public CreateDirectoryDto transform(CreateDirectoryRequest createDirectoryRequest) {
validateCreateDirectoryRequest(createDirectoryRequest);
return doTransform(createDirectoryRequest);
}
private CreateDirectoryDto doTransform(CreateDirectoryRequest createDirectoryRequest) {
String directoryAbsolutePath = getDirectoryAbsolutePath(createDirectoryRequest);
return CreateDirectoryDto.builder()
.loginUser(createDirectoryRequest.getLoginUser())
.directoryAbsolutePath(directoryAbsolutePath)
.build();
}
private void validateCreateDirectoryRequest(CreateDirectoryRequest createDirectoryRequest) {
checkNotNull(createDirectoryRequest.getLoginUser(), "loginUser is null");
checkNotNull(createDirectoryRequest.getType(), "resource type is null");
checkNotNull(createDirectoryRequest.getDirectoryName(), "directory name is null");
checkNotNull(createDirectoryRequest.getParentAbsoluteDirectory(), "parent directory is null");
}
private String getDirectoryAbsolutePath(CreateDirectoryRequest createDirectoryRequest) {
String tenantCode = tenantDao.queryOptionalById(createDirectoryRequest.getLoginUser().getTenantId())
.orElseThrow(() -> new ServiceException(Status.CURRENT_LOGIN_USER_TENANT_NOT_EXIST))
.getTenantCode();
String userResRootPath = storageOperator.getStorageBaseDirectory(tenantCode, createDirectoryRequest.getType());
String parentDirectoryName = createDirectoryRequest.getParentAbsoluteDirectory();
String directoryName = createDirectoryRequest.getDirectoryName();
// If the parent directory is / then will transform to userResRootPath
// This only happens when the front-end go into the resource page first
// todo: we need to change the front-end logic to avoid this
if (parentDirectoryName.equals("/")) {
return FileUtils.concatFilePath(userResRootPath, directoryName);
}
if (!StringUtils.startsWith(parentDirectoryName, userResRootPath)) {
throw new ServiceException(Status.ILLEGAL_RESOURCE_PATH, parentDirectoryName);
}
return FileUtils.concatFilePath(parentDirectoryName, directoryName);
}
}

47
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/CreateFileDtoValidator.java Normal file
View File

@ -0,0 +1,47 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.validator.resource;
import org.apache.dolphinscheduler.api.dto.resources.CreateFileDto;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.repository.TenantDao;
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator;
import org.springframework.stereotype.Component;
import org.springframework.web.multipart.MultipartFile;
@Component
public class CreateFileDtoValidator extends AbstractResourceValidator<CreateFileDto> {
public CreateFileDtoValidator(StorageOperator storageOperator, TenantDao tenantDao) {
super(storageOperator, tenantDao);
}
@Override
public void validate(CreateFileDto createFileDto) {
String fileAbsolutePath = createFileDto.getFileAbsolutePath();
User loginUser = createFileDto.getLoginUser();
MultipartFile file = createFileDto.getFile();
exceptionResourceAbsolutePathInvalidated(fileAbsolutePath);
exceptionResourceExists(fileAbsolutePath);
exceptionFileInvalidated(file);
exceptionUserNoResourcePermission(loginUser, fileAbsolutePath);
exceptionResourceIsNotFile(fileAbsolutePath);
}
}

46
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/CreateFileFromContentDtoValidator.java Normal file
View File

@ -0,0 +1,46 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.validator.resource;
import org.apache.dolphinscheduler.api.dto.resources.CreateFileFromContentDto;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.repository.TenantDao;
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator;
import org.springframework.stereotype.Component;
@Component
public class CreateFileFromContentDtoValidator extends AbstractResourceValidator<CreateFileFromContentDto> {
public CreateFileFromContentDtoValidator(StorageOperator storageOperator, TenantDao tenantDao) {
super(storageOperator, tenantDao);
}
@Override
public void validate(CreateFileFromContentDto createFileFromContentDto) {
String fileAbsolutePath = createFileFromContentDto.getFileAbsolutePath();
User loginUser = createFileFromContentDto.getLoginUser();
String fileContent = createFileFromContentDto.getFileContent();
exceptionResourceAbsolutePathInvalidated(fileAbsolutePath);
exceptionResourceIsNotFile(fileAbsolutePath);
exceptionResourceExists(fileAbsolutePath);
exceptionUserNoResourcePermission(loginUser, fileAbsolutePath);
exceptionFileContentInvalidated(fileContent);
}
}

43
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/DeleteResourceDtoValidator.java Normal file
View File

@ -0,0 +1,43 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.validator.resource;
import org.apache.dolphinscheduler.api.dto.resources.DeleteResourceDto;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.repository.TenantDao;
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator;
import org.springframework.stereotype.Component;
@Component
public class DeleteResourceDtoValidator extends AbstractResourceValidator<DeleteResourceDto> {
public DeleteResourceDtoValidator(StorageOperator storageOperator, TenantDao tenantDao) {
super(storageOperator, tenantDao);
}
@Override
public void validate(DeleteResourceDto deleteResourceDto) {
String resourceAbsolutePath = deleteResourceDto.getResourceAbsolutePath();
User loginUser = deleteResourceDto.getLoginUser();
exceptionResourceAbsolutePathInvalidated(resourceAbsolutePath);
exceptionResourceNotExists(resourceAbsolutePath);
exceptionUserNoResourcePermission(loginUser, resourceAbsolutePath);
}
}

44
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/DownloadFileDtoValidator.java Normal file
View File

@ -0,0 +1,44 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.validator.resource;
import org.apache.dolphinscheduler.api.dto.resources.DownloadFileDto;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.repository.TenantDao;
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator;
import org.springframework.stereotype.Component;
@Component
public class DownloadFileDtoValidator extends AbstractResourceValidator<DownloadFileDto> {
public DownloadFileDtoValidator(StorageOperator storageOperator, TenantDao tenantDao) {
super(storageOperator, tenantDao);
}
@Override
public void validate(DownloadFileDto downloadFileDto) {
String fileAbsolutePath = downloadFileDto.getFileAbsolutePath();
User loginUser = downloadFileDto.getLoginUser();
exceptionResourceNotExists(fileAbsolutePath);
exceptionResourceAbsolutePathInvalidated(fileAbsolutePath);
exceptionResourceIsNotFile(fileAbsolutePath);
exceptionUserNoResourcePermission(loginUser, fileAbsolutePath);
}
}

48
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/FetchFileContentDtoValidator.java Normal file
View File

@ -0,0 +1,48 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.validator.resource;
import org.apache.dolphinscheduler.api.dto.resources.FetchFileContentDto;
import org.apache.dolphinscheduler.api.exceptions.ServiceException;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.repository.TenantDao;
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator;
import org.springframework.stereotype.Component;
@Component
public class FetchFileContentDtoValidator extends AbstractResourceValidator<FetchFileContentDto> {
public FetchFileContentDtoValidator(StorageOperator storageOperator, TenantDao tenantDao) {
super(storageOperator, tenantDao);
}
@Override
public void validate(FetchFileContentDto fetchFileContentDto) {
if (fetchFileContentDto.getSkipLineNum() < 0) {
throw new ServiceException("skipLineNum must be greater than or equal to 0");
}
String resourceFileAbsolutePath = fetchFileContentDto.getResourceFileAbsolutePath();
User loginUser = fetchFileContentDto.getLoginUser();
exceptionResourceAbsolutePathInvalidated(resourceFileAbsolutePath);
exceptionResourceIsNotFile(resourceFileAbsolutePath);
exceptionUserNoResourcePermission(loginUser, resourceFileAbsolutePath);
exceptionFileContentCannotFetch(resourceFileAbsolutePath);
}
}

71
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/FileFromContentRequestTransformer.java Normal file
View File

@ -0,0 +1,71 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.validator.resource;
import static com.google.common.base.Preconditions.checkNotNull;
import org.apache.dolphinscheduler.api.dto.resources.CreateFileFromContentDto;
import org.apache.dolphinscheduler.api.dto.resources.CreateFileFromContentRequest;
import org.apache.dolphinscheduler.common.utils.FileUtils;
import org.apache.dolphinscheduler.dao.repository.TenantDao;
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator;
import org.springframework.stereotype.Component;
@Component
public class FileFromContentRequestTransformer
extends
AbstractResourceTransformer<CreateFileFromContentRequest, CreateFileFromContentDto> {
public FileFromContentRequestTransformer(TenantDao tenantDao, StorageOperator storageOperator) {
super(tenantDao, storageOperator);
}
@Override
public CreateFileFromContentDto transform(CreateFileFromContentRequest createFileFromContentRequest) {
validateCreateFileRequest(createFileFromContentRequest);
return doTransform(createFileFromContentRequest);
}
private void validateCreateFileRequest(CreateFileFromContentRequest createFileFromContentRequest) {
checkNotNull(createFileFromContentRequest.getLoginUser(), "loginUser is null");
checkNotNull(createFileFromContentRequest.getType(), "resource type is null");
checkNotNull(createFileFromContentRequest.getFileName(), "file name is null");
checkNotNull(createFileFromContentRequest.getParentAbsoluteDirectory(), "parent directory is null");
checkNotNull(createFileFromContentRequest.getFileContent(), "file content is null");
}
private CreateFileFromContentDto doTransform(CreateFileFromContentRequest createFileFromContentRequest) {
String fileAbsolutePath = getFileAbsolutePath(createFileFromContentRequest);
return CreateFileFromContentDto.builder()
.loginUser(createFileFromContentRequest.getLoginUser())
.fileAbsolutePath(fileAbsolutePath)
.fileContent(createFileFromContentRequest.getFileContent())
.build();
}
private String getFileAbsolutePath(CreateFileFromContentRequest createFileFromContentRequest) {
String parentDirectoryAbsolutePath = getParentDirectoryAbsolutePath(
createFileFromContentRequest.getLoginUser(),
createFileFromContentRequest.getParentAbsoluteDirectory(),
createFileFromContentRequest.getType());
return FileUtils.concatFilePath(parentDirectoryAbsolutePath, createFileFromContentRequest.getFileName());
}
}

68
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/FileRequestTransformer.java Normal file
View File

@ -0,0 +1,68 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.validator.resource;
import static com.google.common.base.Preconditions.checkNotNull;
import org.apache.dolphinscheduler.api.dto.resources.CreateFileDto;
import org.apache.dolphinscheduler.api.dto.resources.CreateFileRequest;
import org.apache.dolphinscheduler.common.utils.FileUtils;
import org.apache.dolphinscheduler.dao.repository.TenantDao;
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator;
import org.springframework.stereotype.Component;
@Component
public class FileRequestTransformer extends AbstractResourceTransformer<CreateFileRequest, CreateFileDto> {
public FileRequestTransformer(TenantDao tenantDao, StorageOperator storageOperator) {
super(tenantDao, storageOperator);
}
@Override
public CreateFileDto transform(CreateFileRequest createFileRequest) {
validateCreateFileRequest(createFileRequest);
return doTransform(createFileRequest);
}
private void validateCreateFileRequest(CreateFileRequest createFileRequest) {
checkNotNull(createFileRequest.getLoginUser(), "loginUser is null");
checkNotNull(createFileRequest.getType(), "resource type is null");
checkNotNull(createFileRequest.getFileName(), "file name is null");
checkNotNull(createFileRequest.getParentAbsoluteDirectory(), "parent directory is null");
checkNotNull(createFileRequest.getFile(), "file is null");
}
private CreateFileDto doTransform(CreateFileRequest createFileRequest) {
String fileAbsolutePath = getFileAbsolutePath(createFileRequest);
return CreateFileDto.builder()
.loginUser(createFileRequest.getLoginUser())
.file(createFileRequest.getFile())
.fileAbsolutePath(fileAbsolutePath)
.build();
}
private String getFileAbsolutePath(CreateFileRequest createFileRequest) {
String parentDirectoryAbsolutePath = getParentDirectoryAbsolutePath(
createFileRequest.getLoginUser(),
createFileRequest.getParentAbsoluteDirectory(),
createFileRequest.getType());
return FileUtils.concatFilePath(parentDirectoryAbsolutePath, createFileRequest.getFileName());
}
}

90
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/PagingResourceItemRequestTransformer.java Normal file
View File

@ -0,0 +1,90 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.validator.resource;
import static com.google.common.base.Preconditions.checkNotNull;
import org.apache.dolphinscheduler.api.dto.resources.PagingResourceItemRequest;
import org.apache.dolphinscheduler.api.dto.resources.QueryResourceDto;
import org.apache.dolphinscheduler.api.validator.ITransformer;
import org.apache.dolphinscheduler.common.enums.UserType;
import org.apache.dolphinscheduler.dao.entity.Tenant;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.repository.TenantDao;
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator;
import org.apache.dolphinscheduler.spi.enums.ResourceType;
import org.apache.commons.lang3.StringUtils;
import java.util.List;
import java.util.stream.Collectors;
import lombok.AllArgsConstructor;
import org.springframework.stereotype.Component;
import com.google.common.collect.Lists;
@Component
@AllArgsConstructor
public class PagingResourceItemRequestTransformer implements ITransformer<PagingResourceItemRequest, QueryResourceDto> {
private final StorageOperator storageOperator;
private final TenantDao tenantDao;
@Override
public QueryResourceDto transform(PagingResourceItemRequest pagingResourceItemRequest) {
validatePagingResourceItemRequest(pagingResourceItemRequest);
if (StringUtils.isNotEmpty(pagingResourceItemRequest.getResourceAbsolutePath())) {
// query from the given path
return QueryResourceDto.builder()
.resourceAbsolutePaths(Lists.newArrayList(pagingResourceItemRequest.getResourceAbsolutePath()))
.build();
}
ResourceType resourceType = pagingResourceItemRequest.getResourceType();
User loginUser = pagingResourceItemRequest.getLoginUser();
if (loginUser.getUserType() == UserType.ADMIN_USER) {
// If the current user is admin
// then will query all tenant resources
List<String> resourceAbsolutePaths = tenantDao.queryAll()
.stream()
.map(tenant -> storageOperator.getStorageBaseDirectory(tenant.getTenantCode(), resourceType))
.collect(Collectors.toList());
return QueryResourceDto.builder()
.resourceAbsolutePaths(resourceAbsolutePaths)
.build();
} else {
// todo: inject the tenantCode when login
Tenant tenant = tenantDao.queryById(loginUser.getTenantId());
String storageBaseDirectory = storageOperator.getStorageBaseDirectory(tenant.getTenantCode(), resourceType);
return QueryResourceDto.builder()
.resourceAbsolutePaths(Lists.newArrayList(storageBaseDirectory))
.build();
}
}
private void validatePagingResourceItemRequest(PagingResourceItemRequest pagingResourceItemRequest) {
checkNotNull(pagingResourceItemRequest.getLoginUser(), "loginUser is null");
checkNotNull(pagingResourceItemRequest.getResourceType(), "resourceType is null");
}
}

50
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/RenameDirectoryDtoValidator.java Normal file
View File

@ -0,0 +1,50 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.validator.resource;
import org.apache.dolphinscheduler.api.dto.resources.RenameDirectoryDto;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.repository.TenantDao;
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator;
import org.springframework.stereotype.Component;
@Component
public class RenameDirectoryDtoValidator extends AbstractResourceValidator<RenameDirectoryDto> {
public RenameDirectoryDtoValidator(StorageOperator storageOperator, TenantDao tenantDao) {
super(storageOperator, tenantDao);
}
@Override
public void validate(RenameDirectoryDto renameDirectoryDto) {
String originDirectoryAbsolutePath = renameDirectoryDto.getOriginDirectoryAbsolutePath();
User loginUser = renameDirectoryDto.getLoginUser();
String targetDirectoryAbsolutePath = renameDirectoryDto.getTargetDirectoryAbsolutePath();
exceptionResourceAbsolutePathInvalidated(originDirectoryAbsolutePath);
exceptionResourceIsNotDirectory(originDirectoryAbsolutePath);
exceptionResourceNotExists(originDirectoryAbsolutePath);
exceptionUserNoResourcePermission(loginUser, originDirectoryAbsolutePath);
exceptionResourceAbsolutePathInvalidated(targetDirectoryAbsolutePath);
exceptionResourceIsNotDirectory(targetDirectoryAbsolutePath);
exceptionResourceExists(targetDirectoryAbsolutePath);
exceptionUserNoResourcePermission(loginUser, targetDirectoryAbsolutePath);
}
}

56
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/RenameDirectoryRequestTransformer.java Normal file
View File

@ -0,0 +1,56 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.validator.resource;
import org.apache.dolphinscheduler.api.dto.resources.RenameDirectoryDto;
import org.apache.dolphinscheduler.api.dto.resources.RenameDirectoryRequest;
import org.apache.dolphinscheduler.api.validator.ITransformer;
import org.apache.commons.lang3.StringUtils;
import java.io.File;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
@Slf4j
@Component
public class RenameDirectoryRequestTransformer implements ITransformer<RenameDirectoryRequest, RenameDirectoryDto> {
@Override
public RenameDirectoryDto transform(RenameDirectoryRequest renameDirectoryRequest) {
String originDirectoryAbsolutePath = renameDirectoryRequest.getDirectoryAbsolutePath();
String targetDirectoryName = renameDirectoryRequest.getNewDirectoryName();
String targetDirectoryAbsolutePath =
getTargetDirectoryAbsolutePath(originDirectoryAbsolutePath, targetDirectoryName);
return RenameDirectoryDto.builder()
.loginUser(renameDirectoryRequest.getLoginUser())
.originDirectoryAbsolutePath(originDirectoryAbsolutePath)
.targetDirectoryAbsolutePath(targetDirectoryAbsolutePath)
.build();
}
private String getTargetDirectoryAbsolutePath(String originDirectoryAbsolutePath, String targetDirectoryName) {
String originDirectoryParentAbsolutePath = StringUtils.substringBeforeLast(
originDirectoryAbsolutePath, File.separator);
return originDirectoryParentAbsolutePath + File.separator + targetDirectoryName;
}
}

50
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/RenameFileDtoValidator.java Normal file
View File

@ -0,0 +1,50 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.validator.resource;
import org.apache.dolphinscheduler.api.dto.resources.RenameFileDto;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.repository.TenantDao;
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator;
import org.springframework.stereotype.Component;
@Component
public class RenameFileDtoValidator extends AbstractResourceValidator<RenameFileDto> {
public RenameFileDtoValidator(StorageOperator storageOperator, TenantDao tenantDao) {
super(storageOperator, tenantDao);
}
@Override
public void validate(RenameFileDto renameFileDto) {
String originFileAbsolutePath = renameFileDto.getOriginFileAbsolutePath();
User loginUser = renameFileDto.getLoginUser();
String targetFileAbsolutePath = renameFileDto.getTargetFileAbsolutePath();
exceptionResourceAbsolutePathInvalidated(originFileAbsolutePath);
exceptionResourceNotExists(originFileAbsolutePath);
exceptionResourceIsNotFile(originFileAbsolutePath);
exceptionUserNoResourcePermission(loginUser, originFileAbsolutePath);
exceptionResourceAbsolutePathInvalidated(targetFileAbsolutePath);
exceptionResourceExists(targetFileAbsolutePath);
exceptionResourceIsNotFile(targetFileAbsolutePath);
exceptionUserNoResourcePermission(loginUser, targetFileAbsolutePath);
}
}

47
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/RenameFileRequestTransformer.java Normal file
View File

@ -0,0 +1,47 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.validator.resource;
import org.apache.dolphinscheduler.api.dto.resources.RenameFileDto;
import org.apache.dolphinscheduler.api.dto.resources.RenameFileRequest;
import org.apache.dolphinscheduler.api.validator.ITransformer;
import org.apache.dolphinscheduler.common.utils.FileUtils;
import org.apache.dolphinscheduler.plugin.storage.api.ResourceMetadata;
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
@Component
public class RenameFileRequestTransformer implements ITransformer<RenameFileRequest, RenameFileDto> {
@Autowired
private StorageOperator storageOperator;
@Override
public RenameFileDto transform(RenameFileRequest renameFileRequest) {
ResourceMetadata resourceMetaData =
storageOperator.getResourceMetaData(renameFileRequest.getFileAbsolutePath());
return RenameFileDto.builder()
.loginUser(renameFileRequest.getLoginUser())
.originFileAbsolutePath(renameFileRequest.getFileAbsolutePath())
.targetFileAbsolutePath(FileUtils.concatFilePath(resourceMetaData.getResourceParentAbsolutePath(),
renameFileRequest.getNewFileName()))
.build();
}
}

57
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/UpdateFileDtoValidator.java Normal file
View File

@ -0,0 +1,57 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.validator.resource;
import org.apache.dolphinscheduler.api.dto.resources.UpdateFileDto;
import org.apache.dolphinscheduler.api.exceptions.ServiceException;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.repository.TenantDao;
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator;
import java.util.Objects;
import org.springframework.stereotype.Component;
import org.springframework.web.multipart.MultipartFile;
import com.google.common.io.Files;
@Component
public class UpdateFileDtoValidator extends AbstractResourceValidator<UpdateFileDto> {
public UpdateFileDtoValidator(StorageOperator storageOperator, TenantDao tenantDao) {
super(storageOperator, tenantDao);
}
@Override
public void validate(UpdateFileDto updateFileDto) {
String fileAbsolutePath = updateFileDto.getFileAbsolutePath();
User loginUser = updateFileDto.getLoginUser();
MultipartFile file = updateFileDto.getFile();
if (!Objects.equals(Files.getFileExtension(file.getName()),
Files.getFileExtension(updateFileDto.getFileAbsolutePath()))) {
throw new ServiceException("file extension cannot not change");
}
exceptionResourceAbsolutePathInvalidated(fileAbsolutePath);
exceptionResourceNotExists(fileAbsolutePath);
exceptionResourceIsNotFile(fileAbsolutePath);
exceptionUserNoResourcePermission(loginUser, fileAbsolutePath);
exceptionFileInvalidated(file);
}
}

47
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/UpdateFileFromContentDtoValidator.java Normal file
View File

@ -0,0 +1,47 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.validator.resource;
import org.apache.dolphinscheduler.api.dto.resources.UpdateFileFromContentDto;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.repository.TenantDao;
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator;
import org.springframework.stereotype.Component;
@Component
public class UpdateFileFromContentDtoValidator extends AbstractResourceValidator<UpdateFileFromContentDto> {
public UpdateFileFromContentDtoValidator(StorageOperator storageOperator, TenantDao tenantDao) {
super(storageOperator, tenantDao);
}
@Override
public void validate(UpdateFileFromContentDto updateFileFromContentDto) {
String fileAbsolutePath = updateFileFromContentDto.getFileAbsolutePath();
User loginUser = updateFileFromContentDto.getLoginUser();
String fileContent = updateFileFromContentDto.getFileContent();
exceptionResourceAbsolutePathInvalidated(fileAbsolutePath);
exceptionResourceNotExists(fileAbsolutePath);
exceptionResourceIsNotFile(fileAbsolutePath);
exceptionUserNoResourcePermission(loginUser, fileAbsolutePath);
exceptionFileContentCannotFetch(fileAbsolutePath);
exceptionFileContentInvalidated(fileContent);
}
}

39
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/UpdateFileFromContentRequestTransformer.java Normal file
View File

@ -0,0 +1,39 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.validator.resource;
import org.apache.dolphinscheduler.api.dto.resources.UpdateFileFromContentDto;
import org.apache.dolphinscheduler.api.dto.resources.UpdateFileFromContentRequest;
import org.apache.dolphinscheduler.api.validator.ITransformer;
import org.springframework.stereotype.Component;
@Component
public class UpdateFileFromContentRequestTransformer
implements
ITransformer<UpdateFileFromContentRequest, UpdateFileFromContentDto> {
@Override
public UpdateFileFromContentDto transform(UpdateFileFromContentRequest updateFileContentRequest) {
return UpdateFileFromContentDto.builder()
.loginUser(updateFileContentRequest.getLoginUser())
.fileAbsolutePath(updateFileContentRequest.getFileAbsolutePath())
.fileContent(updateFileContentRequest.getFileContent())
.build();
}
}

35
dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/common/UiChannelFactory.java → dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/UpdateFileRequestTransformer.java
View File

@ -15,29 +15,24 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.spi.common;
package org.apache.dolphinscheduler.api.validator.resource;
import org.apache.dolphinscheduler.spi.params.base.PluginParams;
import org.apache.dolphinscheduler.api.dto.resources.UpdateFileDto;
import org.apache.dolphinscheduler.api.dto.resources.UpdateFileRequest;
import org.apache.dolphinscheduler.api.validator.ITransformer;
import java.util.List;
import org.springframework.stereotype.Component;
public interface UiChannelFactory {
@Component
public class UpdateFileRequestTransformer implements ITransformer<UpdateFileRequest, UpdateFileDto> {
/**
* plugin name
* Must be UNIQUE .
* This alert plugin name eg: email , message ...
* Name can often be displayed on the page ui eg : email , message , MR , spark , hive ...
*
* @return this alert plugin name
*/
String getName();
/**
* Returns the configurable parameters that this plugin needs to display on the web ui
*
* @return this alert plugin params
*/
List<PluginParams> getParams();
@Override
public UpdateFileDto transform(UpdateFileRequest updateFileRequest) {
return UpdateFileDto.builder()
.loginUser(updateFileRequest.getLoginUser())
.fileAbsolutePath(updateFileRequest.getFileAbsolutePath())
.file(updateFileRequest.getFile())
.build();
}
}

Some files were not shown because too many files have changed in this diff Show More