natas_hw
/
dolphinscheduler
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'upstream/dev' into json_split 

# Conflicts:
#	dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessDefinitionController.java
#	dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionVersionServiceImpl.java
#	dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectServiceImpl.java
#	dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProcessDefinitionControllerTest.java
#	dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionVersionServiceTest.java
#	dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/TaskNode.java
#	dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterBaseTaskExecThread.java
#	dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/processdure/ProcedureTask.java
#	dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java
This commit is contained in:
baoliang 2021-04-26 14:14:05 +08:00
parent c3558965c1 6b8461e901
commit 746d9eca1d
188 changed files with 5332 additions and 3730 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.github/actions/comment-on-issue vendored Submodule

@ -0,0 +1 @@
Subproject commit 966bbbd2113de1f8008322fd1cc7674981a6b1ab

1
.github/actions/lable-on-issue vendored Submodule

@ -0,0 +1 @@
Subproject commit 978f846c4ca6299fd136f465b42c5e87aca28cac

1
.github/actions/translation-on-issue vendored Submodule

@ -0,0 +1 @@
Subproject commit 4b5062133773227695d089319f0374ff0868c9fb

33
.github/workflows/issue_robot.yml vendored
View File

@ -22,24 +22,29 @@ on:
types: [opened]
jobs:
comment:
issueRobot:
runs-on: ubuntu-latest
steps:
- uses: ben-z/actions-comment-on-issue@1.0.2
with:
message: "Hi:\n* Thank you for your feedback, we have received your issue, Please wait patiently for a reply.\n* In order to let us know your request as soon as possible, please provide detailed information、version or pictures.\n* If you haven't received a reply for a long time, you can subscribe to the developer's email，Mail subscription steps reference https://dolphinscheduler.apache.org/zh-cn/community/development/subscribe.html ,then send questions to dev@dolphinscheduler.apache.org."
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: "Checkout ${{ github.ref }}"
uses: actions/checkout@v2
with:
persist-credentials: false
submodules: true
autoTranslate:
runs-on: ubuntu-latest
steps:
- uses: actions-cool/translation-helper@v1.1.1
- name: "Translation into English in issue"
uses: ./.github/actions/translation-on-issue
with:
translate-title: true
translate-body: false
automate-issues-labels:
runs-on: ubuntu-latest
steps:
- name: initial labeling
uses: andymckay/labeler@master
- name: "Comment in issue"
uses: ./.github/actions/comment-on-issue
with:
message: "Hi:\n* Thank you for your feedback, we have received your issue, Please wait patiently for a reply.\n* In order for us to understand your request as soon as possible, please provide detailed information、version or pictures.\n* If you haven't received a reply for a long time, you can subscribe to the developer's email，Mail subscription steps reference https://dolphinscheduler.apache.org/zh-cn/community/development/subscribe.html ,Then write the issue URL in the email content and send question to dev@dolphinscheduler.apache.org."
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: "Add lable in issue"
uses: ./.github/actions/lable-on-issue
with:
add-labels: "Waiting for reply"
ignore-if-labeled: true

26
.gitmodules vendored Normal file
View File

@ -0,0 +1,26 @@
# Licensed to Apache Software Foundation (ASF) under one or more contributor
# license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright
# ownership. Apache Software Foundation (ASF) licenses this file to you under
# the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
[submodule ".github/actions/comment-on-issue"]
path = .github/actions/comment-on-issue
url = https://github.com/xingchun-chen/actions-comment-on-issue
[submodule ".github/actions/lable-on-issue"]
path = .github/actions/lable-on-issue
url = https://github.com/xingchun-chen/labeler
[submodule ".github/actions/translation-on-issue"]
path = .github/actions/translation-on-issue
url = https://github.com/xingchun-chen/translation-helper

6
CONTRIBUTING.md
View File

@ -14,12 +14,12 @@ There are three branches in the remote repository currently:
So, you should fork the `dev` branch.
After forking the [dolphinscheduler upstream source repository](https://github.com/apache/incubator-dolphinscheduler/fork) to your personal repository, you can set your personal development environment.
After forking the [dolphinscheduler upstream source repository](https://github.com/apache/dolphinscheduler/fork) to your personal repository, you can set your personal development environment.
```sh
$ cd <your work direcotry>
$ git clone < your personal forked dolphinscheduler repo>
$ cd incubator-dolphinscheduler
$ cd dolphinscheduler
```
## Set git remote as ``upstream``
@ -27,7 +27,7 @@ $ cd incubator-dolphinscheduler
Add remote repository address, named upstream
```sh
git remote add upstream https://github.com/apache/incubator-dolphinscheduler.git
git remote add upstream https://github.com/apache/dolphinscheduler.git
```
View repository:

5
DISCLAIMER
View File

@ -1,5 +0,0 @@
Apache DolphinScheduler is an effort undergoing incubation at The Apache Software Foundation (ASF), sponsored by the Apache Incubator.
Incubation is required of all newly accepted projects until a further review indicates that the infrastructure,
communications, and decision making process have stabilized in a manner consistent with other successful ASF projects.
While incubation status is not necessarily a reflection of the completeness or stability of the code,
it does indicate that the project has yet to be fully endorsed by the ASF.

4
LICENSE
View File

@ -200,9 +200,9 @@
See the License for the specific language governing permissions and
limitations under the License.
=======================================================================
Apache DolphinScheduler (incubating) Subcomponents:
Apache DolphinScheduler Subcomponents:
The Apache DolphinScheduler (incubating) project contains subcomponents with separate copyright
The Apache DolphinScheduler project contains subcomponents with separate copyright
notices and license terms. Your use of the source code for the these
subcomponents is subject to the terms and conditions of the following
licenses.

2
NOTICE
View File

@ -1,4 +1,4 @@
Apache DolphinScheduler (incubating)
Apache DolphinScheduler
Copyright 2019-2021 The Apache Software Foundation
This product includes software developed at

14
README.md
View File

@ -3,15 +3,15 @@ Dolphin Scheduler Official Website
============
[![License](https://img.shields.io/badge/license-Apache%202-4EB1BA.svg)](https://www.apache.org/licenses/LICENSE-2.0.html)
[![Total Lines](https://tokei.rs/b1/github/apache/Incubator-DolphinScheduler?category=lines)](https://github.com/apache/Incubator-DolphinScheduler)
[![codecov](https://codecov.io/gh/apache/incubator-dolphinscheduler/branch/dev/graph/badge.svg)](https://codecov.io/gh/apache/incubator-dolphinscheduler/branch/dev)
[![Total Lines](https://tokei.rs/b1/github/apache/dolphinscheduler?category=lines)](https://github.com/apache/dolphinscheduler)
[![codecov](https://codecov.io/gh/apache/dolphinscheduler/branch/dev/graph/badge.svg)](https://codecov.io/gh/apache/dolphinscheduler/branch/dev)
[![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=apache-dolphinscheduler&metric=alert_status)](https://sonarcloud.io/dashboard?id=apache-dolphinscheduler)
[![Twitter Follow](https://img.shields.io/twitter/follow/dolphinschedule.svg?style=social&label=Follow)](https://twitter.com/dolphinschedule)
[![Slack Status](https://img.shields.io/badge/slack-join_chat-white.svg?logo=slack&style=social)](https://join.slack.com/t/asf-dolphinscheduler/shared_invite/zt-omtdhuio-_JISsxYhiVsltmC5h38yfw)
[![Stargazers over time](https://starchart.cc/apache/incubator-dolphinscheduler.svg)](https://starchart.cc/apache/incubator-dolphinscheduler)
[![Stargazers over time](https://starchart.cc/apache/dolphinscheduler.svg)](https://starchart.cc/apache/dolphinscheduler)
[![EN doc](https://img.shields.io/badge/document-English-blue.svg)](README.md)
[![CN doc](https://img.shields.io/badge/文档-中文版-blue.svg)](README_zh_CN.md)
@ -73,8 +73,8 @@ Please referer the official website document: [QuickStart in Kubernetes](https:/
Artifact:
```
dolphinscheduler-dist/target/apache-dolphinscheduler-incubating-${latest.release.version}-dolphinscheduler-bin.tar.gz: Binary package of DolphinScheduler
dolphinscheduler-dist/target/apache-dolphinscheduler-incubating-${latest.release.version}-src.zip: Source code package of DolphinScheduler
dolphinscheduler-dist/target/apache-dolphinscheduler-${latest.release.version}-bin.tar.gz: Binary package of DolphinScheduler
dolphinscheduler-dist/target/apache-dolphinscheduler-${latest.release.version}-src.tar.gz: Source code package of DolphinScheduler
```
## Thanks
@ -84,7 +84,7 @@ We would like to express our deep gratitude to all the open-source projects used
## Get Help
1. Submit an [issue](https://github.com/apache/incubator-dolphinscheduler/issues/new/choose)
1. Submit an [issue](https://github.com/apache/dolphinscheduler/issues/new/choose)
1. Subscribe to this mailing list: https://dolphinscheduler.apache.org/en-us/community/development/subscribe.html, then email dev@dolphinscheduler.apache.org
## Community
@ -99,4 +99,4 @@ The community welcomes everyone to contribute, please refer to this page to find
## License
Please refer to the [LICENSE](https://github.com/apache/incubator-dolphinscheduler/blob/dev/LICENSE) file.
Please refer to the [LICENSE](https://github.com/apache/dolphinscheduler/blob/dev/LICENSE) file.

14
README_zh_CN.md
View File

@ -3,12 +3,12 @@ Dolphin Scheduler Official Website
============
[![License](https://img.shields.io/badge/license-Apache%202-4EB1BA.svg)](https://www.apache.org/licenses/LICENSE-2.0.html)
[![Total Lines](https://tokei.rs/b1/github/apache/Incubator-DolphinScheduler?category=lines)](https://github.com/apache/Incubator-DolphinScheduler)
[![codecov](https://codecov.io/gh/apache/incubator-dolphinscheduler/branch/dev/graph/badge.svg)](https://codecov.io/gh/apache/incubator-dolphinscheduler/branch/dev)
[![Total Lines](https://tokei.rs/b1/github/apache/dolphinscheduler?category=lines)](https://github.com/apache/dolphinscheduler)
[![codecov](https://codecov.io/gh/apache/dolphinscheduler/branch/dev/graph/badge.svg)](https://codecov.io/gh/apache/dolphinscheduler/branch/dev)
[![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=apache-dolphinscheduler&metric=alert_status)](https://sonarcloud.io/dashboard?id=apache-dolphinscheduler)
[![Stargazers over time](https://starchart.cc/apache/incubator-dolphinscheduler.svg)](https://starchart.cc/apache/incubator-dolphinscheduler)
[![Stargazers over time](https://starchart.cc/apache/dolphinscheduler.svg)](https://starchart.cc/apache/dolphinscheduler)
[![CN doc](https://img.shields.io/badge/文档-中文版-blue.svg)](README_zh_CN.md)
[![EN doc](https://img.shields.io/badge/document-English-blue.svg)](README.md)
@ -52,7 +52,7 @@ Dolphin Scheduler Official Website
## 近期研发计划
DolphinScheduler的工作计划：<a href="https://github.com/apache/incubator-dolphinscheduler/projects/1" target="_blank">研发计划</a> ，其中 In Develop卡片下是正在研发的功能，TODO卡片是待做事项(包括 feature ideas)
DolphinScheduler的工作计划：<a href="https://github.com/apache/dolphinscheduler/projects/1" target="_blank">研发计划</a> ，其中 In Develop卡片下是正在研发的功能，TODO卡片是待做事项(包括 feature ideas)
## 参与贡献
@ -76,8 +76,8 @@ DolphinScheduler的工作计划：<a href="https://github.com/apache/incubator-d
制品:
```
dolphinscheduler-dist/target/apache-dolphinscheduler-incubating-${latest.release.version}-dolphinscheduler-bin.tar.gz: DolphinScheduler 二进制包
dolphinscheduler-dist/target/apache-dolphinscheduler-incubating-${latest.release.version}-src.zip: DolphinScheduler 源代码包
dolphinscheduler-dist/target/apache-dolphinscheduler-${latest.release.version}-bin.tar.gz: DolphinScheduler 二进制包
dolphinscheduler-dist/target/apache-dolphinscheduler-${latest.release.version}-src.tar.gz: DolphinScheduler 源代码包
```
## 感谢
@ -97,4 +97,4 @@ Dolphin Scheduler使用了很多优秀的开源项目，比如google的guava、g
## 版权
请参考 [LICENSE](https://github.com/apache/incubator-dolphinscheduler/blob/dev/LICENSE) 文件.
请参考 [LICENSE](https://github.com/apache/dolphinscheduler/blob/dev/LICENSE) 文件.

38
ambari_plugin/README.md
View File

@ -10,7 +10,7 @@
1. Prepare the RPM packages
- It is generated by executing the command ```mvn -U clean install -Prpmbuild -Dmaven.test.skip=true -X``` in the project root directory (In the directory: dolphinscheduler-dist/target/rpm/apache-dolphinscheduler-incubating/RPMS/noarch )
- It is generated by executing the command ```mvn -U clean install -Prpmbuild -Dmaven.test.skip=true -X``` in the project root directory (In the directory: dolphinscheduler-dist/target/rpm/apache-dolphinscheduler/RPMS/noarch )
2. Create an installation for DS，who have read and write access to the installation directory (/opt/soft)
@ -18,7 +18,7 @@
- Manual installation (recommended）：
- Copy the prepared RPM packages to each node of the cluster.
- Execute with DS installation user: ```rpm -ivh apache-dolphinscheduler-incubating-xxx.noarch.rpm```
- Execute with DS installation user: ```rpm -ivh apache-dolphinscheduler-xxx.noarch.rpm```
- Mysql-connector-java packaged using the default POM file will not be included.
- The RPM package was packaged in the project with the installation path of /opt/soft.
If you use mysql as the database, you need add it manually.
@ -54,41 +54,41 @@
1. Install Dolphin Scheduler on ambari web interface
![](https://github.com/apache/incubator-dolphinscheduler-website/blob/master/img/ambari-plugin/DS2_AMBARI_001.png)
![](https://github.com/apache/dolphinscheduler-website/blob/master/img/ambari-plugin/DS2_AMBARI_001.png)
2. Select the nodes for the Dolphin Scheduler's Master installation
![](https://github.com/apache/incubator-dolphinscheduler-website/blob/master/img/ambari-plugin/DS2_AMBARI_002.png)
![](https://github.com/apache/dolphinscheduler-website/blob/master/img/ambari-plugin/DS2_AMBARI_002.png)
3. Configure the Dolphin Scheduler's nodes for Worker, Api, Logger, Alert installation
![](https://github.com/apache/incubator-dolphinscheduler-website/blob/master/img/ambari-plugin/DS2_AMBARI_003.png)
![](https://github.com/apache/dolphinscheduler-website/blob/master/img/ambari-plugin/DS2_AMBARI_003.png)
4. Set the installation users of the Dolphin Scheduler service (created in step 1) and the user groups they belong to
![](https://github.com/apache/incubator-dolphinscheduler-website/blob/master/img/ambari-plugin/DS2_AMBARI_004.png)
![](https://github.com/apache/dolphinscheduler-website/blob/master/img/ambari-plugin/DS2_AMBARI_004.png)
5. System Env Optimization will export some system environment config. Modify according to actual situation
![](https://github.com/apache/incubator-dolphinscheduler-website/blob/master/img/ambari-plugin/DS2_AMBARI_020.png)
![](https://github.com/apache/dolphinscheduler-website/blob/master/img/ambari-plugin/DS2_AMBARI_020.png)
6. Configure the database information (same as in the initialization database in step 1)
![](https://github.com/apache/incubator-dolphinscheduler-website/blob/master/img/ambari-plugin/DS2_AMBARI_005.png)
![](https://github.com/apache/dolphinscheduler-website/blob/master/img/ambari-plugin/DS2_AMBARI_005.png)
7. Configure additional information if needed
![](https://github.com/apache/incubator-dolphinscheduler-website/blob/master/img/ambari-plugin/DS2_AMBARI_006.png)
![](https://github.com/apache/dolphinscheduler-website/blob/master/img/ambari-plugin/DS2_AMBARI_006.png)
![](https://github.com/apache/incubator-dolphinscheduler-website/blob/master/img/ambari-plugin/DS2_AMBARI_007.png)
![](https://github.com/apache/dolphinscheduler-website/blob/master/img/ambari-plugin/DS2_AMBARI_007.png)
8. Perform the next steps as normal
![](https://github.com/apache/incubator-dolphinscheduler-website/blob/master/img/ambari-plugin/DS2_AMBARI_008.png)
![](https://github.com/apache/dolphinscheduler-website/blob/master/img/ambari-plugin/DS2_AMBARI_008.png)
9. The interface after successful installation
![](https://github.com/apache/incubator-dolphinscheduler-website/blob/master/img/ambari-plugin/DS2_AMBARI_009.png)
![](https://github.com/apache/dolphinscheduler-website/blob/master/img/ambari-plugin/DS2_AMBARI_009.png)
@ -102,31 +102,31 @@
1. Locate the component node to add -- for example, node ark3
![DS2_AMBARI_011](https://github.com/apache/incubator-dolphinscheduler-website/blob/master/img/ambari-plugin/DS2_AMBARI_011.png)
![DS2_AMBARI_011](https://github.com/apache/dolphinscheduler-website/blob/master/img/ambari-plugin/DS2_AMBARI_011.png)
2. Add components -- the drop-down list is all addable
![DS2_AMBARI_012](https://github.com/apache/incubator-dolphinscheduler-website/blob/master/img/ambari-plugin/DS2_AMBARI_012.png)
![DS2_AMBARI_012](https://github.com/apache/dolphinscheduler-website/blob/master/img/ambari-plugin/DS2_AMBARI_012.png)
3. Confirm component addition
![DS2_AMBARI_013](https://github.com/apache/incubator-dolphinscheduler-website/blob/master/img/ambari-plugin/DS2_AMBARI_013.png)
![DS2_AMBARI_013](https://github.com/apache/dolphinscheduler-website/blob/master/img/ambari-plugin/DS2_AMBARI_013.png)
4. After adding DS Worker and DS Logger components
![DS2_AMBARI_015](https://github.com/apache/incubator-dolphinscheduler-website/blob/master/img/ambari-plugin/DS2_AMBARI_015.png)
![DS2_AMBARI_015](https://github.com/apache/dolphinscheduler-website/blob/master/img/ambari-plugin/DS2_AMBARI_015.png)
5. Start the component
![DS2_AMBARI_016](https://github.com/apache/incubator-dolphinscheduler-website/blob/master/img/ambari-plugin/DS2_AMBARI_016.png)
![DS2_AMBARI_016](https://github.com/apache/dolphinscheduler-website/blob/master/img/ambari-plugin/DS2_AMBARI_016.png)
#### Remove the component from the node with Ambari
1. Stop the component in the corresponding node
![DS2_AMBARI_018](https://github.com/apache/incubator-dolphinscheduler-website/blob/master/img/ambari-plugin/DS2_AMBARI_018.png)
![DS2_AMBARI_018](https://github.com/apache/dolphinscheduler-website/blob/master/img/ambari-plugin/DS2_AMBARI_018.png)
2. Remove components
![DS2_AMBARI_019](https://github.com/apache/incubator-dolphinscheduler-website/blob/master/img/ambari-plugin/DS2_AMBARI_019.png)
![DS2_AMBARI_019](https://github.com/apache/dolphinscheduler-website/blob/master/img/ambari-plugin/DS2_AMBARI_019.png)

2
ambari_plugin/common-services/DOLPHIN/1.3.0/metainfo.xml
View File

@ -103,7 +103,7 @@
<osFamily>any</osFamily>
<packages>
<package>
<name>apache-dolphinscheduler-incubating*</name>
<name>apache-dolphinscheduler*</name>
</package>
</packages>
</osSpecific>

2
ambari_plugin/common-services/DOLPHIN/1.3.3/metainfo.xml
View File

@ -103,7 +103,7 @@
<osFamily>any</osFamily>
<packages>
<package>
<name>apache-dolphinscheduler-incubating*</name>
<name>apache-dolphinscheduler*</name>
</package>
</packages>
</osSpecific>

4
docker/build/Dockerfile
View File

@ -34,8 +34,8 @@ RUN apk update && \
rm -rf /var/cache/apk/*
# 2. add dolphinscheduler
ADD ./apache-dolphinscheduler-incubating-${VERSION}-dolphinscheduler-bin.tar.gz /opt/
RUN ln -s /opt/apache-dolphinscheduler-incubating-${VERSION}-dolphinscheduler-bin /opt/dolphinscheduler
ADD ./apache-dolphinscheduler-${VERSION}-bin.tar.gz /opt/
RUN ln -s /opt/apache-dolphinscheduler-${VERSION}-bin /opt/dolphinscheduler
ENV DOLPHINSCHEDULER_HOME /opt/dolphinscheduler
# 3. add configuration and modify permissions and set soft links

8
docker/build/README.md
View File

@ -2,7 +2,7 @@
DolphinScheduler is a distributed and easy-to-expand visual DAG workflow scheduling system, dedicated to solving the complex dependencies in data processing, making the scheduling system out of the box for data processing.
GitHub URL: https://github.com/apache/incubator-dolphinscheduler
GitHub URL: https://github.com/apache/dolphinscheduler
Official Website: https://dolphinscheduler.apache.org
@ -111,14 +111,14 @@ You can build a docker image in A Unix-like operating system, You can also build
In Unix-Like, Example:
```bash
$ cd path/incubator-dolphinscheduler
$ cd path/dolphinscheduler
$ sh ./docker/build/hooks/build
```
In Windows, Example:
```bat
C:\incubator-dolphinscheduler>.\docker\build\hooks\build.bat
C:\dolphinscheduler>.\docker\build\hooks\build.bat
```
Please read `./docker/build/hooks/build` `./docker/build/hooks/build.bat` script files if you don't understand
@ -485,4 +485,4 @@ docker build -t apache/dolphinscheduler:oracle-driver .
6. Add a Oracle datasource in `Datasource manage`
For more information please refer to the [incubator-dolphinscheduler](https://github.com/apache/incubator-dolphinscheduler.git) documentation.
For more information please refer to the [dolphinscheduler](https://github.com/apache/dolphinscheduler.git) documentation.

8
docker/build/README_zh_CN.md
View File

@ -2,7 +2,7 @@
一个分布式易扩展的可视化DAG工作流任务调度系统。致力于解决数据处理流程中错综复杂的依赖关系，使调度系统在数据处理流程中`开箱即用`。
GitHub URL: https://github.com/apache/incubator-dolphinscheduler
GitHub URL: https://github.com/apache/dolphinscheduler
Official Website: https://dolphinscheduler.apache.org
@ -111,14 +111,14 @@ apache/dolphinscheduler:latest alert-server
类Unix系统, 如下:
```bash
$ cd path/incubator-dolphinscheduler
$ cd path/dolphinscheduler
$ sh ./docker/build/hooks/build
```
Windows系统, 如下:
```bat
C:\incubator-dolphinscheduler>.\docker\build\hooks\build.bat
C:\dolphinscheduler>.\docker\build\hooks\build.bat
```
如果你不理解这些脚本 `./docker/build/hooks/build` `./docker/build/hooks/build.bat`，请阅读里面的内容。
@ -485,4 +485,4 @@ docker build -t apache/dolphinscheduler:oracle-driver .
6. 在数据源中心添加一个 Oracle 数据源
更多信息请查看 [incubator-dolphinscheduler](https://github.com/apache/incubator-dolphinscheduler.git) 文档.
更多信息请查看 [dolphinscheduler](https://github.com/apache/dolphinscheduler.git) 文档.

4
docker/build/conf/dolphinscheduler/application-api.properties.tpl
View File

@ -35,10 +35,10 @@ spring.servlet.multipart.max-request-size=1024MB
server.compression.enabled=true
server.compression.mime-types=text/html,text/xml,text/plain,text/css,text/javascript,application/javascript,application/json,application/xml
# post content
# max http post size
server.jetty.max-http-form-post-size=5000000
# i18n
# messages encoding
spring.messages.encoding=UTF-8
# i18n classpath folder , file prefix messages， if have many files, use "," seperator

4
docker/build/hooks/build
View File

@ -43,8 +43,8 @@ echo -e "mvn -B clean compile package -Prelease -Dmaven.test.skip=true"
mvn -B clean compile package -Prelease -Dmaven.test.skip=true
# mv dolphinscheduler-bin.tar.gz file to docker/build directory
echo -e "mv $(pwd)/dolphinscheduler-dist/target/apache-dolphinscheduler-incubating-${VERSION}-dolphinscheduler-bin.tar.gz $(pwd)/docker/build/\n"
mv $(pwd)/dolphinscheduler-dist/target/apache-dolphinscheduler-incubating-${VERSION}-dolphinscheduler-bin.tar.gz $(pwd)/docker/build/
echo -e "mv $(pwd)/dolphinscheduler-dist/target/apache-dolphinscheduler-${VERSION}-bin.tar.gz $(pwd)/docker/build/\n"
mv $(pwd)/dolphinscheduler-dist/target/apache-dolphinscheduler-${VERSION}-bin.tar.gz $(pwd)/docker/build/
# docker build
BUILD_COMMAND="docker build --build-arg VERSION=${VERSION} -t $DOCKER_REPO:${VERSION} $(pwd)/docker/build/"

4
docker/build/hooks/build.bat
View File

@ -44,8 +44,8 @@ call mvn clean compile package -Prelease -DskipTests=true
if "%errorlevel%"=="1" goto :mvnFailed
:: move dolphinscheduler-bin.tar.gz file to docker/build directory
echo "move %cd%\dolphinscheduler-dist\target\apache-dolphinscheduler-incubating-%VERSION%-SNAPSHOT-dolphinscheduler-bin.tar.gz %cd%\docker\build\"
move %cd%\dolphinscheduler-dist\target\apache-dolphinscheduler-incubating-%VERSION%-dolphinscheduler-bin.tar.gz %cd%\docker\build\
echo "move %cd%\dolphinscheduler-dist\target\apache-dolphinscheduler-%VERSION%-bin.tar.gz %cd%\docker\build\"
move %cd%\dolphinscheduler-dist\target\apache-dolphinscheduler-%VERSION%-bin.tar.gz %cd%\docker\build\
:: docker build
echo "docker build --build-arg VERSION=%VERSION% -t %DOCKER_REPO%:%VERSION% %cd%\docker\build\"

6
docker/kubernetes/dolphinscheduler/README.md
View File

@ -16,8 +16,8 @@ This chart bootstraps a [DolphinScheduler](https://dolphinscheduler.apache.org)
To install the chart with the release name `dolphinscheduler`:
```bash
$ git clone https://github.com/apache/incubator-dolphinscheduler.git
$ cd incubator-dolphinscheduler/docker/kubernetes/dolphinscheduler
$ git clone https://github.com/apache/dolphinscheduler.git
$ cd dolphinscheduler/docker/kubernetes/dolphinscheduler
$ helm repo add bitnami https://charts.bitnami.com/bitnami
$ helm dependency update .
$ helm install dolphinscheduler .
@ -361,4 +361,4 @@ docker build -t apache/dolphinscheduler:oracle-driver .
7. Add a Oracle datasource in `Datasource manage`
For more information please refer to the [incubator-dolphinscheduler](https://github.com/apache/incubator-dolphinscheduler.git) documentation.
For more information please refer to the [dolphinscheduler](https://github.com/apache/dolphinscheduler.git) documentation.

14
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/ApiApplicationServer.java
View File

@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api;
import org.springframework.boot.SpringApplication;
@ -25,15 +26,12 @@ import org.springframework.context.annotation.FilterType;
@SpringBootApplication
@ServletComponentScan
@ComponentScan(basePackages = {"org.apache.dolphinscheduler"},
excludeFilters = @ComponentScan.Filter(type = FilterType.REGEX,
pattern = "org.apache.dolphinscheduler.server.*"))
@ComponentScan(value = "org.apache.dolphinscheduler",
excludeFilters = @ComponentScan.Filter(type = FilterType.REGEX, pattern = "org.apache.dolphinscheduler.server.*"))
public class ApiApplicationServer extends SpringBootServletInitializer {
public static void main(String[] args) {
SpringApplication.run(ApiApplicationServer.class, args);
}
public static void main(String[] args) {
SpringApplication.run(ApiApplicationServer.class, args);
}
}

23
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/configuration/AppConfiguration.java
View File

@ -19,18 +19,24 @@ package org.apache.dolphinscheduler.api.configuration;
import org.apache.dolphinscheduler.api.interceptor.LocaleChangeInterceptor;
import org.apache.dolphinscheduler.api.interceptor.LoginHandlerInterceptor;
import org.apache.dolphinscheduler.api.interceptor.RateLimitInterceptor;
import java.util.Locale;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.web.cors.CorsConfiguration;
import org.springframework.web.cors.UrlBasedCorsConfigurationSource;
import org.springframework.web.filter.CorsFilter;
import org.springframework.web.servlet.LocaleResolver;
import org.springframework.web.servlet.config.annotation.*;
import org.springframework.web.servlet.config.annotation.ContentNegotiationConfigurer;
import org.springframework.web.servlet.config.annotation.InterceptorRegistry;
import org.springframework.web.servlet.config.annotation.ResourceHandlerRegistry;
import org.springframework.web.servlet.config.annotation.ViewControllerRegistry;
import org.springframework.web.servlet.config.annotation.WebMvcConfigurer;
import org.springframework.web.servlet.i18n.CookieLocaleResolver;
import java.util.Locale;
/**
* application configuration
*/
@ -43,6 +49,9 @@ public class AppConfiguration implements WebMvcConfigurer {
public static final String PATH_PATTERN = "/**";
public static final String LOCALE_LANGUAGE_COOKIE = "language";
@Autowired
private TrafficConfiguration trafficConfiguration;
@Bean
public CorsFilter corsFilter() {
CorsConfiguration config = new CorsConfiguration();
@ -79,10 +88,18 @@ public class AppConfiguration implements WebMvcConfigurer {
return new LocaleChangeInterceptor();
}
@Bean
public RateLimitInterceptor createRateLimitInterceptor() {
return new RateLimitInterceptor(trafficConfiguration);
}
@Override
public void addInterceptors(InterceptorRegistry registry) {
// i18n
registry.addInterceptor(localeChangeInterceptor());
if (trafficConfiguration.isTrafficGlobalControlSwitch() || trafficConfiguration.isTrafficTenantControlSwitch()) {
registry.addInterceptor(createRateLimitInterceptor());
}
registry.addInterceptor(loginInterceptor())
.addPathPatterns(LOGIN_INTERCEPTOR_PATH_PATTERN)
.excludePathPatterns(LOGIN_PATH_PATTERN, REGISTER_PATH_PATTERN,

78
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/configuration/TrafficConfiguration.java Normal file
View File

@ -0,0 +1,78 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.configuration;
import java.util.Map;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Configuration;
@Configuration
public class TrafficConfiguration {
@Value("${traffic.control.global.switch:false}")
private boolean trafficGlobalControlSwitch;
@Value("${traffic.control.max.global.qps.rate:300}")
private Integer maxGlobalQpsRate;
@Value("${traffic.control.tenant.switch:false}")
private boolean trafficTenantControlSwitch;
@Value("${traffic.control.default.tenant.qps.rate:10}")
private Integer defaultTenantQpsRate;
@Value("#{'${traffic.control.customize.tenant.qps.rate:}'.empty?null:'${traffic.control.customize.tenant.qps.rate:}'}")
private Map<String, Integer> customizeTenantQpsRate;
public boolean isTrafficGlobalControlSwitch() {
return trafficGlobalControlSwitch;
}
public void setTrafficGlobalControlSwitch(boolean trafficGlobalControlSwitch) {
this.trafficGlobalControlSwitch = trafficGlobalControlSwitch;
}
public Integer getMaxGlobalQpsRate() {
return maxGlobalQpsRate;
}
public void setMaxGlobalQpsRate(Integer maxGlobalQpsRate) {
this.maxGlobalQpsRate = maxGlobalQpsRate;
}
public boolean isTrafficTenantControlSwitch() {
return trafficTenantControlSwitch;
}
public void setTrafficTenantControlSwitch(boolean trafficTenantControlSwitch) {
this.trafficTenantControlSwitch = trafficTenantControlSwitch;
}
public Integer getDefaultTenantQpsRate() {
return defaultTenantQpsRate;
}
public void setDefaultTenantQpsRate(Integer defaultTenantQpsRate) {
this.defaultTenantQpsRate = defaultTenantQpsRate;
}
public Map<String, Integer> getCustomizeTenantQpsRate() {
return customizeTenantQpsRate;
}
public void setCustomizeTenantQpsRate(Map<String, Integer> customizeTenantQpsRate) {
this.customizeTenantQpsRate = customizeTenantQpsRate;
}
}

157
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataSourceController.java
View File

@ -31,9 +31,12 @@ import static org.apache.dolphinscheduler.api.enums.Status.VERIFY_DATASOURCE_NAM
import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.exceptions.ApiException;
import org.apache.dolphinscheduler.api.service.DataSourceService;
import org.apache.dolphinscheduler.api.utils.RegexUtils;
import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.DbConnectType;
import org.apache.dolphinscheduler.common.datasource.BaseDataSourceParamDTO;
import org.apache.dolphinscheduler.common.datasource.ConnectionParam;
import org.apache.dolphinscheduler.common.datasource.DatasourceUtil;
import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.common.utils.CommonUtils;
import org.apache.dolphinscheduler.common.utils.ParameterUtils;
@ -48,6 +51,7 @@ import org.springframework.http.HttpStatus;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestAttribute;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.ResponseStatus;
@ -57,6 +61,7 @@ import io.swagger.annotations.Api;
import io.swagger.annotations.ApiImplicitParam;
import io.swagger.annotations.ApiImplicitParams;
import io.swagger.annotations.ApiOperation;
import io.swagger.annotations.ApiParam;
import springfox.documentation.annotations.ApiIgnore;
/**
@ -76,58 +81,19 @@ public class DataSourceController extends BaseController {
* create data source
*
* @param loginUser login user
* @param name data source name
* @param note data source description
* @param type data source type
* @param host host
* @param port port
* @param database data base
* @param principal principal
* @param userName user name
* @param password password
* @param other other arguments
* @param dataSourceParam datasource param
* @return create result code
*/
@ApiOperation(value = "createDataSource", notes = "CREATE_DATA_SOURCE_NOTES")
@ApiImplicitParams({
@ApiImplicitParam(name = "name", value = "DATA_SOURCE_NAME", required = true, dataType = "String"),
@ApiImplicitParam(name = "note", value = "DATA_SOURCE_NOTE", dataType = "String"),
@ApiImplicitParam(name = "type", value = "DB_TYPE", required = true, dataType = "DbType"),
@ApiImplicitParam(name = "host", value = "DATA_SOURCE_HOST", required = true, dataType = "String"),
@ApiImplicitParam(name = "port", value = "DATA_SOURCE_PORT", required = true, dataType = "String"),
@ApiImplicitParam(name = "database", value = "DATABASE_NAME", required = true, dataType = "String"),
@ApiImplicitParam(name = "principal", value = "DATA_SOURCE_PRINCIPAL", dataType = "String"),
@ApiImplicitParam(name = "userName", value = "USER_NAME", required = true, dataType = "String"),
@ApiImplicitParam(name = "password", value = "PASSWORD", dataType = "String"),
@ApiImplicitParam(name = "connectType", value = "CONNECT_TYPE", dataType = "DbConnectType"),
@ApiImplicitParam(name = "other", value = "DATA_SOURCE_OTHER", dataType = "String"),
@ApiImplicitParam(name = "javaSecurityKrb5Conf", value = "DATA_SOURCE_KERBEROS_KRB5_CONF", dataType = "String"),
@ApiImplicitParam(name = "loginUserKeytabUsername", value = "DATA_SOURCE_KERBEROS_KEYTAB_USERNAME", dataType = "String"),
@ApiImplicitParam(name = "loginUserKeytabPath", value = "DATA_SOURCE_KERBEROS_KEYTAB_PATH", dataType = "String")
})
@PostMapping(value = "/create")
@ResponseStatus(HttpStatus.CREATED)
@ApiException(CREATE_DATASOURCE_ERROR)
public Result createDataSource(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
@RequestParam("name") String name,
@RequestParam(value = "note", required = false) String note,
@RequestParam(value = "type") DbType type,
@RequestParam(value = "host") String host,
@RequestParam(value = "port") String port,
@RequestParam(value = "database") String database,
@RequestParam(value = "principal") String principal,
@RequestParam(value = "userName") String userName,
@RequestParam(value = "password") String password,
@RequestParam(value = "connectType") DbConnectType connectType,
@RequestParam(value = "other") String other,
@RequestParam(value = "javaSecurityKrb5Conf", required = false) String javaSecurityKrb5Conf,
@RequestParam(value = "loginUserKeytabUsername", required = false) String loginUserKeytabUsername,
@RequestParam(value = "loginUserKeytabPath", required = false) String loginUserKeytabPath) {
logger.info("login user {} create datasource name: {}, note: {}, type: {}, host: {}, port: {}, database : {}, principal: {}, userName : {}, connectType: {}, other: {}",
loginUser.getUserName(), name, note, type, host, port, database, principal, userName, connectType, other);
String parameter = dataSourceService.buildParameter(type, host, port, database, principal, userName, password, connectType, other,
javaSecurityKrb5Conf, loginUserKeytabUsername, loginUserKeytabPath);
return dataSourceService.createDataSource(loginUser, name, note, type, parameter);
@ApiParam(name = "DATA_SOURCE_PARAM", required = true)
@RequestBody BaseDataSourceParamDTO dataSourceParam) {
String userName = RegexUtils.escapeNRT(loginUser.getUserName());
logger.info("login user {} create datasource : {}", userName, dataSourceParam);
return dataSourceService.createDataSource(loginUser, dataSourceParam);
}
@ -135,61 +101,21 @@ public class DataSourceController extends BaseController {
* updateProcessInstance data source
*
* @param loginUser login user
* @param name data source name
* @param note description
* @param type data source type
* @param other other arguments
* @param id data source di
* @param host host
* @param port port
* @param database database
* @param principal principal
* @param userName user name
* @param password password
* @param dataSourceParam datasource param
* @return update result code
*/
@ApiOperation(value = "updateDataSource", notes = "UPDATE_DATA_SOURCE_NOTES")
@ApiImplicitParams({
@ApiImplicitParam(name = "id", value = "DATA_SOURCE_ID", required = true, dataType = "Int", example = "100"),
@ApiImplicitParam(name = "name", value = "DATA_SOURCE_NAME", required = true, dataType = "String"),
@ApiImplicitParam(name = "note", value = "DATA_SOURCE_NOTE", dataType = "String"),
@ApiImplicitParam(name = "type", value = "DB_TYPE", required = true, dataType = "DbType"),
@ApiImplicitParam(name = "host", value = "DATA_SOURCE_HOST", required = true, dataType = "String"),
@ApiImplicitParam(name = "port", value = "DATA_SOURCE_PORT", required = true, dataType = "String"),
@ApiImplicitParam(name = "database", value = "DATABASE_NAME", required = true, dataType = "String"),
@ApiImplicitParam(name = "principal", value = "DATA_SOURCE_PRINCIPAL", dataType = "String"),
@ApiImplicitParam(name = "userName", value = "USER_NAME", required = true, dataType = "String"),
@ApiImplicitParam(name = "password", value = "PASSWORD", dataType = "String"),
@ApiImplicitParam(name = "connectType", value = "CONNECT_TYPE", dataType = "DbConnectType"),
@ApiImplicitParam(name = "other", value = "DATA_SOURCE_OTHER", dataType = "String"),
@ApiImplicitParam(name = "javaSecurityKrb5Conf", value = "DATA_SOURCE_KERBEROS_KRB5_CONF", dataType = "String"),
@ApiImplicitParam(name = "loginUserKeytabUsername", value = "DATA_SOURCE_KERBEROS_KEYTAB_USERNAME", dataType = "String"),
@ApiImplicitParam(name = "loginUserKeytabPath", value = "DATA_SOURCE_KERBEROS_KEYTAB_PATH", dataType = "String")
})
@PostMapping(value = "/update")
@ResponseStatus(HttpStatus.OK)
@ApiException(UPDATE_DATASOURCE_ERROR)
public Result updateDataSource(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
@RequestParam("id") int id,
@RequestParam("name") String name,
@RequestParam(value = "note", required = false) String note,
@RequestParam(value = "type") DbType type,
@RequestParam(value = "host") String host,
@RequestParam(value = "port") String port,
@RequestParam(value = "database") String database,
@RequestParam(value = "principal") String principal,
@RequestParam(value = "userName") String userName,
@RequestParam(value = "password") String password,
@RequestParam(value = "connectType") DbConnectType connectType,
@RequestParam(value = "other") String other,
@RequestParam(value = "javaSecurityKrb5Conf", required = false) String javaSecurityKrb5Conf,
@RequestParam(value = "loginUserKeytabUsername", required = false) String loginUserKeytabUsername,
@RequestParam(value = "loginUserKeytabPath", required = false) String loginUserKeytabPath) {
logger.info("login user {} updateProcessInstance datasource name: {}, note: {}, type: {}, connectType: {}, other: {}",
loginUser.getUserName(), name, note, type, connectType, other);
String parameter = dataSourceService.buildParameter(type, host, port, database, principal, userName, password, connectType, other,
javaSecurityKrb5Conf, loginUserKeytabUsername, loginUserKeytabPath);
return dataSourceService.updateDataSource(id, loginUser, name, note, type, parameter);
@RequestBody BaseDataSourceParamDTO dataSourceParam) {
String userName = RegexUtils.escapeNRT(loginUser.getUserName());
logger.info("login user {} updateProcessInstance datasource : {}", userName, dataSourceParam);
return dataSourceService.updateDataSource(dataSourceParam.getId(), loginUser, dataSourceParam);
}
/**
@ -270,58 +196,23 @@ public class DataSourceController extends BaseController {
* connect datasource
*
* @param loginUser login user
* @param name data source name
* @param note data soruce description
* @param type data source type
* @param other other parameters
* @param host host
* @param port port
* @param database data base
* @param principal principal
* @param userName user name
* @param password password
* @param dataSourceParam datasource param
* @return connect result code
*/
@ApiOperation(value = "connectDataSource", notes = "CONNECT_DATA_SOURCE_NOTES")
@ApiImplicitParams({
@ApiImplicitParam(name = "name", value = "DATA_SOURCE_NAME", required = true, dataType = "String"),
@ApiImplicitParam(name = "note", value = "DATA_SOURCE_NOTE", dataType = "String"),
@ApiImplicitParam(name = "type", value = "DB_TYPE", required = true, dataType = "DbType"),
@ApiImplicitParam(name = "host", value = "DATA_SOURCE_HOST", required = true, dataType = "String"),
@ApiImplicitParam(name = "port", value = "DATA_SOURCE_PORT", required = true, dataType = "String"),
@ApiImplicitParam(name = "database", value = "DATABASE_NAME", required = true, dataType = "String"),
@ApiImplicitParam(name = "principal", value = "DATA_SOURCE_PRINCIPAL", dataType = "String"),
@ApiImplicitParam(name = "userName", value = "USER_NAME", required = true, dataType = "String"),
@ApiImplicitParam(name = "password", value = "PASSWORD", dataType = "String"),
@ApiImplicitParam(name = "connectType", value = "CONNECT_TYPE", dataType = "DbConnectType"),
@ApiImplicitParam(name = "other", value = "DATA_SOURCE_OTHER", dataType = "String"),
@ApiImplicitParam(name = "javaSecurityKrb5Conf", value = "DATA_SOURCE_KERBEROS_KRB5_CONF", dataType = "String"),
@ApiImplicitParam(name = "loginUserKeytabUsername", value = "DATA_SOURCE_KERBEROS_KEYTAB_USERNAME", dataType = "String"),
@ApiImplicitParam(name = "loginUserKeytabPath", value = "DATA_SOURCE_KERBEROS_KEYTAB_PATH", dataType = "String")
})
@PostMapping(value = "/connect")
@ResponseStatus(HttpStatus.OK)
@ApiException(CONNECT_DATASOURCE_FAILURE)
public Result connectDataSource(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
@RequestParam("name") String name,
@RequestParam(value = "note", required = false) String note,
@RequestParam(value = "type") DbType type,
@RequestParam(value = "host") String host,
@RequestParam(value = "port") String port,
@RequestParam(value = "database") String database,
@RequestParam(value = "principal") String principal,
@RequestParam(value = "userName") String userName,
@RequestParam(value = "password") String password,
@RequestParam(value = "connectType") DbConnectType connectType,
@RequestParam(value = "other") String other,
@RequestParam(value = "javaSecurityKrb5Conf", required = false) String javaSecurityKrb5Conf,
@RequestParam(value = "loginUserKeytabUsername", required = false) String loginUserKeytabUsername,
@RequestParam(value = "loginUserKeytabPath", required = false) String loginUserKeytabPath) {
logger.info("login user {}, connect datasource: {}, note: {}, type: {}, connectType: {}, other: {}",
loginUser.getUserName(), name, note, type, connectType, other);
String parameter = dataSourceService.buildParameter(type, host, port, database, principal, userName, password, connectType, other,
javaSecurityKrb5Conf, loginUserKeytabUsername, loginUserKeytabPath);
return dataSourceService.checkConnection(type, parameter);
@RequestBody BaseDataSourceParamDTO dataSourceParam) {
String userName = RegexUtils.escapeNRT(loginUser.getUserName());
logger.info("login user {}, connect datasource: {}", userName, dataSourceParam);
DatasourceUtil.checkDatasourceParam(dataSourceParam);
ConnectionParam connectionParams = DatasourceUtil.buildConnectionParams(dataSourceParam);
return dataSourceService.checkConnection(dataSourceParam.getType(), connectionParams);
}
/**

132
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessDefinitionController.java
View File

@ -122,7 +122,13 @@ public class ProcessDefinitionController extends BaseController {
logger.info("login user {}, create process definition, project name: {}, process definition name: {}, "
+ "process_definition_json: {}, desc: {} locations:{}, connects:{}",
loginUser.getUserName(), projectName, name, json, description, locations, connects);
RegexUtils.escapeNRT(loginUser.getUserName()),
RegexUtils.escapeNRT(projectName),
RegexUtils.escapeNRT(name),
RegexUtils.escapeNRT(json),
RegexUtils.escapeNRT(description),
RegexUtils.escapeNRT(locations),
RegexUtils.escapeNRT(connects));
Map<String, Object> result = processDefinitionService.createProcessDefinition(loginUser, projectName, name, json,
description, locations, connects);
return returnDataList(result);
@ -209,7 +215,9 @@ public class ProcessDefinitionController extends BaseController {
@ApiParam(name = "projectName", value = "PROJECT_NAME", required = true) @PathVariable String projectName,
@RequestParam(value = "name", required = true) String name) {
logger.info("verify process definition name unique, user:{}, project name:{}, process definition name:{}",
loginUser.getUserName(), projectName, name);
RegexUtils.escapeNRT(loginUser.getUserName()),
RegexUtils.escapeNRT(projectName),
RegexUtils.escapeNRT(name));
Map<String, Object> result = processDefinitionService.verifyProcessDefinitionName(loginUser, projectName, name);
return returnDataList(result);
}
@ -251,9 +259,16 @@ public class ProcessDefinitionController extends BaseController {
@RequestParam(value = "description", required = false) String description,
@RequestParam(value = "releaseState", required = false, defaultValue = "OFFLINE") ReleaseState releaseState) {
logger.info("login user {}, update process define, project name: {}, process define name: {}, "
logger.info("login user {}, update process define, project name: {}, process define name: {},"
+ "process_definition_json: {}, desc: {}, locations:{}, connects:{}",
loginUser.getUserName(), projectName, name, processDefinitionJson, description, locations, connects);
RegexUtils.escapeNRT(loginUser.getUserName()),
RegexUtils.escapeNRT(projectName),
RegexUtils.escapeNRT(name),
RegexUtils.escapeNRT(processDefinitionJson),
RegexUtils.escapeNRT(description),
RegexUtils.escapeNRT(locations),
RegexUtils.escapeNRT(connects));
Map<String, Object> result = processDefinitionService.updateProcessDefinition(loginUser, projectName, id, name,
processDefinitionJson, description, locations, connects);
// If the update fails, the result will be returned directly
@ -294,9 +309,21 @@ public class ProcessDefinitionController extends BaseController {
@RequestParam(value = "processDefinitionCode") long processDefinitionCode) {
logger.info("login user {}, query process versions, project name: {}, pageNo: {}, pageSize: {}, processDefinitionCode: {}",
loginUser.getUserName(), projectName, pageNo, pageSize, processDefinitionCode);
Map<String, Object> result = processDefinitionService.queryProcessDefinitionVersions(loginUser
, projectName, pageNo, pageSize, processDefinitionCode);
logger.info("query process definition versions, login user {}, project name: {}, process define id: {}, list paging, pageNo: {}, pageSize: {}",
RegexUtils.escapeNRT(loginUser.getUserName()),
RegexUtils.escapeNRT(projectName),
processDefinitionCode,
pageNo,
pageSize);
Map<String, Object> result = checkPageParams(pageNo, pageSize);
if (result.get(Constants.STATUS) != Status.SUCCESS) {
return returnDataListPaging(result);
}
result = processDefinitionService.queryProcessDefinitionVersions(loginUser
, projectName, pageNo, pageSize, processDefinitionCode);
return returnDataList(result);
}
@ -321,8 +348,11 @@ public class ProcessDefinitionController extends BaseController {
@ApiParam(name = "projectName", value = "PROJECT_NAME", required = true) @PathVariable String projectName,
@RequestParam(value = "processDefinitionId") int processDefinitionId,
@RequestParam(value = "version") long version) {
logger.info("login user {}, switch process version, project name: {}, processDefinitionId: {}, version: {}",
loginUser.getUserName(), projectName, processDefinitionId, version);
logger.info("switch certain process definition version, login user {}, project name:{}, process definition id:{}, version:{}",
RegexUtils.escapeNRT(loginUser.getUserName()),
RegexUtils.escapeNRT(projectName),
processDefinitionId,
version);
Map<String, Object> result = processDefinitionService.switchProcessDefinitionVersion(loginUser, projectName
, processDefinitionId, version);
return returnDataList(result);
@ -349,8 +379,11 @@ public class ProcessDefinitionController extends BaseController {
@ApiParam(name = "projectName", value = "PROJECT_NAME", required = true) @PathVariable String projectName,
@RequestParam(value = "processDefinitionId") int processDefinitionId,
@RequestParam(value = "version") long version) {
logger.info("login user {}, delete process definition, project name: {}, processDefinitionId: {}, version: {}",
loginUser.getUserName(), projectName, processDefinitionId, version);
logger.info("delete the certain process definition version by version and process definition id, login user {}, project name:{}, process definition id:{}, version:{}",
RegexUtils.escapeNRT(loginUser.getUserName()),
RegexUtils.escapeNRT(projectName),
processDefinitionId,
version);
Map<String, Object> result = processDefinitionService.deleteByProcessDefinitionIdAndVersion(loginUser, projectName, processDefinitionId, version);
return returnDataList(result);
}
@ -379,13 +412,15 @@ public class ProcessDefinitionController extends BaseController {
@RequestParam(value = "releaseState", required = true) ReleaseState releaseState) {
logger.info("login user {}, release process definition, project name: {}, release state: {}",
loginUser.getUserName(), projectName, releaseState);
RegexUtils.escapeNRT(loginUser.getUserName()),
RegexUtils.escapeNRT(projectName),
releaseState);
Map<String, Object> result = processDefinitionService.releaseProcessDefinition(loginUser, projectName, processId, releaseState);
return returnDataList(result);
}
/**
* query datail of process definition by id
* query detail of process definition by id
*
* @param loginUser login user
* @param projectName project name
@ -401,16 +436,18 @@ public class ProcessDefinitionController extends BaseController {
@ApiException(QUERY_DATAIL_OF_PROCESS_DEFINITION_ERROR)
public Result queryProcessDefinitionById(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
@ApiParam(name = "projectName", value = "PROJECT_NAME", required = true) @PathVariable String projectName,
@RequestParam("processId") Integer processId
) {
@RequestParam("processId") Integer processId) {
logger.info("query detail of process definition, login user:{}, project name:{}, process definition id:{}",
loginUser.getUserName(), projectName, processId);
RegexUtils.escapeNRT(loginUser.getUserName()),
RegexUtils.escapeNRT(projectName),
processId);
Map<String, Object> result = processDefinitionService.queryProcessDefinitionById(loginUser, projectName, processId);
return returnDataList(result);
}
/**
* query datail of process definition by name
* query detail of process definition by name
*
* @param loginUser login user
* @param projectName project name
@ -426,8 +463,12 @@ public class ProcessDefinitionController extends BaseController {
@ApiException(QUERY_DATAIL_OF_PROCESS_DEFINITION_ERROR)
public Result<ProcessDefinition> queryProcessDefinitionByName(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
@ApiParam(name = "projectName", value = "PROJECT_NAME", required = true) @PathVariable String projectName,
@RequestParam("processDefinitionName") String processDefinitionName
) {
@RequestParam("processDefinitionName") String processDefinitionName) {
logger.info("query detail of process definition by name, login user:{}, project name:{}, process definition id:{}",
RegexUtils.escapeNRT(loginUser.getUserName()),
RegexUtils.escapeNRT(projectName),
RegexUtils.escapeNRT(processDefinitionName));
Map<String, Object> result = processDefinitionService.queryProcessDefinitionByName(loginUser, projectName, processDefinitionName);
return returnDataList(result);
}
@ -444,10 +485,11 @@ public class ProcessDefinitionController extends BaseController {
@ResponseStatus(HttpStatus.OK)
@ApiException(QUERY_PROCESS_DEFINITION_LIST)
public Result queryProcessDefinitionList(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
@ApiParam(name = "projectName", value = "PROJECT_NAME", required = true) @PathVariable String projectName
) {
@ApiParam(name = "projectName", value = "PROJECT_NAME", required = true) @PathVariable String projectName) {
logger.info("query process definition list, login user:{}, project name:{}",
loginUser.getUserName(), projectName);
RegexUtils.escapeNRT(loginUser.getUserName()),
RegexUtils.escapeNRT(projectName));
Map<String, Object> result = processDefinitionService.queryProcessDefinitionList(loginUser, projectName);
return returnDataList(result);
}
@ -479,7 +521,13 @@ public class ProcessDefinitionController extends BaseController {
@RequestParam(value = "searchVal", required = false) String searchVal,
@RequestParam(value = "userId", required = false, defaultValue = "0") Integer userId,
@RequestParam("pageSize") Integer pageSize) {
logger.info("query process definition list paging, login user:{}, project name:{}", loginUser.getUserName(), projectName);
logger.info("query process definition list paging, login user:{}, project name:{}, searchVal:{}, userId:{}, pageSize:{}",
RegexUtils.escapeNRT(loginUser.getUserName()),
RegexUtils.escapeNRT(projectName),
RegexUtils.escapeNRT(searchVal),
userId,
pageSize);
Map<String, Object> result = checkPageParams(pageNo, pageSize);
if (result.get(Constants.STATUS) != Status.SUCCESS) {
return returnDataListPaging(result);
@ -490,7 +538,7 @@ public class ProcessDefinitionController extends BaseController {
}
/**
* encapsulation treeview structure
* encapsulation tree view structure
*
* @param loginUser login user
* @param projectName project name
@ -510,6 +558,12 @@ public class ProcessDefinitionController extends BaseController {
@ApiParam(name = "projectName", value = "PROJECT_NAME", required = true) @PathVariable String projectName,
@RequestParam("processId") Integer id,
@RequestParam("limit") Integer limit) throws Exception {
logger.info("encapsulation tree view structure, login user:{}, project name:{}, processId:{}, limit:{}",
RegexUtils.escapeNRT(loginUser.getUserName()),
RegexUtils.escapeNRT(projectName),
id,
limit);
Map<String, Object> result = processDefinitionService.viewTree(id, limit);
return returnDataList(result);
}
@ -533,8 +587,10 @@ public class ProcessDefinitionController extends BaseController {
@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
@ApiParam(name = "projectName", value = "PROJECT_NAME", required = true) @PathVariable String projectName,
@RequestParam("processDefinitionCode") Long processDefinitionCode) throws Exception {
logger.info("query task node name list by definitionCode, login user:{}, project name:{}, code : {}",
loginUser.getUserName(), projectName, processDefinitionCode);
logger.info("query task node name list by definitionId, login user:{}, project name:{}, code : {}",
RegexUtils.escapeNRT(loginUser.getUserName()),
RegexUtils.escapeNRT(projectName),
processDefinitionCode);
Map<String, Object> result = processDefinitionService.getTaskNodeListByDefinitionCode(processDefinitionCode);
return returnDataList(result);
}
@ -559,8 +615,10 @@ public class ProcessDefinitionController extends BaseController {
@ApiParam(name = "projectName", value = "PROJECT_NAME", required = true) @PathVariable String projectName,
@RequestParam("processDefinitionCodeList") String processDefinitionCodeList) {
logger.info("query task node name list by definitionId list, login user:{}, project name:{}, code list: {}",
loginUser.getUserName(), projectName, processDefinitionCodeList);
logger.info("query task node name list by definitionId list, login user:{}, project name:{}, id list: {}",
RegexUtils.escapeNRT(loginUser.getUserName()),
RegexUtils.escapeNRT(projectName),
RegexUtils.escapeNRT(processDefinitionCodeList));
Map<String, Object> result = processDefinitionService.getTaskNodeListByDefinitionCodeList(processDefinitionCodeList);
return returnDataList(result);
}
@ -584,8 +642,11 @@ public class ProcessDefinitionController extends BaseController {
@ApiParam(name = "projectName", value = "PROJECT_NAME", required = true) @PathVariable String projectName,
@RequestParam("processDefinitionId") Integer processDefinitionId
) {
logger.info("delete process definition by id, login user:{}, project name:{}, process definition id:{}",
loginUser.getUserName(), projectName, processDefinitionId);
RegexUtils.escapeNRT(loginUser.getUserName()),
RegexUtils.escapeNRT(projectName),
processDefinitionId);
Map<String, Object> result = processDefinitionService.deleteProcessDefinitionById(loginUser, projectName, processDefinitionId);
return returnDataList(result);
}
@ -609,9 +670,11 @@ public class ProcessDefinitionController extends BaseController {
@ApiParam(name = "projectName", value = "PROJECT_NAME", required = true) @PathVariable String projectName,
@RequestParam("processDefinitionIds") String processDefinitionIds
) {
logger.info("delete process definition by ids, login user:{}, project name:{}, process definition ids:{}",
loginUser.getUserName(), projectName, processDefinitionIds);
logger.info("delete process definition by ids, login user:{}, project name:{}, process definition ids:{}",
RegexUtils.escapeNRT(loginUser.getUserName()),
RegexUtils.escapeNRT(projectName),
RegexUtils.escapeNRT(processDefinitionIds));
Map<String, Object> result = new HashMap<>();
List<String> deleteFailedIdList = new ArrayList<>();
if (StringUtils.isNotEmpty(processDefinitionIds)) {
@ -660,8 +723,11 @@ public class ProcessDefinitionController extends BaseController {
@RequestParam("processDefinitionIds") String processDefinitionIds,
HttpServletResponse response) {
try {
logger.info("batch export process definition by ids, login user:{}, project name:{}, process definition ids:{}",
loginUser.getUserName(), projectName, processDefinitionIds);
RegexUtils.escapeNRT(loginUser.getUserName()),
RegexUtils.escapeNRT(projectName),
RegexUtils.escapeNRT(processDefinitionIds));
processDefinitionService.batchExportProcessDefinitionByIds(loginUser, projectName, processDefinitionIds, response);
} catch (Exception e) {
logger.error(Status.BATCH_EXPORT_PROCESS_DEFINE_BY_IDS_ERROR.getMsg(), e);
@ -681,8 +747,10 @@ public class ProcessDefinitionController extends BaseController {
@ApiException(QUERY_PROCESS_DEFINITION_LIST)
public Result queryProcessDefinitionAllByProjectId(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
@RequestParam("projectId") Integer projectId) {
logger.info("query process definition list, login user:{}, project id:{}",
loginUser.getUserName(), projectId);
RegexUtils.escapeNRT(loginUser.getUserName()),
projectId);
Map<String, Object> result = processDefinitionService.queryProcessDefinitionAllByProjectId(projectId);
return returnDataList(result);
}

6
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessInstanceController.java
View File

@ -126,8 +126,12 @@ public class ProcessInstanceController extends BaseController {
"search value:{},executor name:{},state type:{},host:{},start time:{}, end time:{},page number:{}, page size:{}",
loginUser.getUserName(), projectName, processDefinitionId, searchVal, executorName, stateType, host,
startTime, endTime, pageNo, pageSize);
Map<String, Object> result = checkPageParams(pageNo, pageSize);
if (result.get(Constants.STATUS) != Status.SUCCESS) {
return returnDataListPaging(result);
}
searchVal = ParameterUtils.handleEscapes(searchVal);
Map<String, Object> result = processInstanceService.queryProcessInstanceList(
result = processInstanceService.queryProcessInstanceList(
loginUser, projectName, processDefinitionId, startTime, endTime, searchVal, executorName, stateType, host, pageNo, pageSize);
return returnDataListPaging(result);
}

7
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProjectController.java
View File

@ -27,6 +27,7 @@ import static org.apache.dolphinscheduler.api.enums.Status.QUERY_PROJECT_DETAILS
import static org.apache.dolphinscheduler.api.enums.Status.QUERY_UNAUTHORIZED_PROJECT_ERROR;
import static org.apache.dolphinscheduler.api.enums.Status.UPDATE_PROJECT_ERROR;
import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.exceptions.ApiException;
import org.apache.dolphinscheduler.api.service.ProcessDefinitionService;
import org.apache.dolphinscheduler.api.service.ProjectService;
@ -172,8 +173,12 @@ public class ProjectController extends BaseController {
) {
logger.info("login user {}, query project list paging", loginUser.getUserName());
Map<String, Object> result = checkPageParams(pageNo, pageSize);
if (result.get(Constants.STATUS) != Status.SUCCESS) {
return returnDataListPaging(result);
}
searchVal = ParameterUtils.handleEscapes(searchVal);
Map<String, Object> result = projectService.queryProjectListPaging(loginUser, pageSize, pageNo, searchVal);
result = projectService.queryProjectListPaging(loginUser, pageSize, pageNo, searchVal);
return returnDataListPaging(result);
}

8
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/SchedulerController.java
View File

@ -27,10 +27,12 @@ import static org.apache.dolphinscheduler.api.enums.Status.QUERY_SCHEDULE_LIST_P
import static org.apache.dolphinscheduler.api.enums.Status.UPDATE_SCHEDULE_ERROR;
import static org.apache.dolphinscheduler.common.Constants.SESSION_USER;
import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.exceptions.ApiException;
import org.apache.dolphinscheduler.api.service.SchedulerService;
import org.apache.dolphinscheduler.api.utils.RegexUtils;
import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.FailureStrategy;
import org.apache.dolphinscheduler.common.enums.Priority;
import org.apache.dolphinscheduler.common.enums.ReleaseState;
@ -245,8 +247,12 @@ public class SchedulerController extends BaseController {
@RequestParam("pageSize") Integer pageSize) {
logger.info("login user {}, query schedule, project name: {}, process definition id: {}",
loginUser.getUserName(), projectName, processDefinitionId);
Map<String, Object> result = checkPageParams(pageNo, pageSize);
if (result.get(Constants.STATUS) != Status.SUCCESS) {
return returnDataListPaging(result);
}
searchVal = ParameterUtils.handleEscapes(searchVal);
Map<String, Object> result = schedulerService.querySchedule(loginUser, projectName, processDefinitionId, searchVal, pageNo, pageSize);
result = schedulerService.querySchedule(loginUser, projectName, processDefinitionId, searchVal, pageNo, pageSize);
return returnDataListPaging(result);
}

8
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TaskInstanceController.java
View File

@ -20,6 +20,7 @@ package org.apache.dolphinscheduler.api.controller;
import static org.apache.dolphinscheduler.api.enums.Status.FORCE_TASK_SUCCESS_ERROR;
import static org.apache.dolphinscheduler.api.enums.Status.QUERY_TASK_LIST_PAGING_ERROR;
import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.exceptions.ApiException;
import org.apache.dolphinscheduler.api.service.TaskInstanceService;
import org.apache.dolphinscheduler.api.utils.RegexUtils;
@ -122,8 +123,13 @@ public class TaskInstanceController extends BaseController {
RegexUtils.escapeNRT(host),
RegexUtils.escapeNRT(startTime),
RegexUtils.escapeNRT(endTime));
Map<String, Object> result = checkPageParams(pageNo, pageSize);
if (result.get(Constants.STATUS) != Status.SUCCESS) {
return returnDataListPaging(result);
}
searchVal = ParameterUtils.handleEscapes(searchVal);
Map<String, Object> result = taskInstanceService.queryTaskListPaging(
result = taskInstanceService.queryTaskListPaging(
loginUser, projectName, processInstanceId, processInstanceName, taskName, executorName, startTime, endTime, searchVal, stateType, host, pageNo, pageSize);
return returnDataListPaging(result);
}

24
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/WorkerGroupController.java
View File

@ -18,9 +18,11 @@
package org.apache.dolphinscheduler.api.controller;
import static org.apache.dolphinscheduler.api.enums.Status.DELETE_WORKER_GROUP_FAIL;
import static org.apache.dolphinscheduler.api.enums.Status.QUERY_WORKER_ADDRESS_LIST_FAIL;
import static org.apache.dolphinscheduler.api.enums.Status.QUERY_WORKER_GROUP_FAIL;
import static org.apache.dolphinscheduler.api.enums.Status.SAVE_ERROR;
import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.exceptions.ApiException;
import org.apache.dolphinscheduler.api.service.WorkerGroupService;
import org.apache.dolphinscheduler.api.utils.RegexUtils;
@ -116,8 +118,12 @@ public class WorkerGroupController extends BaseController {
) {
logger.info("query all worker group paging: login user {}, pageNo:{}, pageSize:{}, searchVal:{}",
RegexUtils.escapeNRT(loginUser.getUserName()), pageNo, pageSize, searchVal);
Map<String, Object> result = checkPageParams(pageNo, pageSize);
if (result.get(Constants.STATUS) != Status.SUCCESS) {
return returnDataListPaging(result);
}
searchVal = ParameterUtils.handleEscapes(searchVal);
Map<String, Object> result = workerGroupService.queryAllGroupPaging(loginUser, pageNo, pageSize, searchVal);
result = workerGroupService.queryAllGroupPaging(loginUser, pageNo, pageSize, searchVal);
return returnDataListPaging(result);
}
@ -159,4 +165,20 @@ public class WorkerGroupController extends BaseController {
return returnDataList(result);
}
/**
* query worker address list
*
* @param loginUser login user
* @return all worker address list
*/
@ApiOperation(value = "queryWorkerAddressList", notes = "QUERY_WORKER_ADDRESS_LIST_NOTES")
@GetMapping(value = "/worker-address-list")
@ResponseStatus(HttpStatus.OK)
@ApiException(QUERY_WORKER_ADDRESS_LIST_FAIL)
public Result queryWorkerAddressList(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser) {
logger.info("query worker address list: login user {}", RegexUtils.escapeNRT(loginUser.getUserName()));
Map<String, Object> result = workerGroupService.getWorkerAddressList();
return returnDataList(result);
}
}

1
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java
View File

@ -209,6 +209,7 @@ public enum Status {
CREATE_WORKER_GROUP_FORBIDDEN_IN_DOCKER(10175, "create worker group forbidden in docker ", "创建worker分组在docker中禁止"),
DELETE_WORKER_GROUP_FORBIDDEN_IN_DOCKER(10176, "delete worker group forbidden in docker ", "删除worker分组在docker中禁止"),
WORKER_ADDRESS_INVALID(10177, "worker address {0} invalid", "worker地址[{0}]无效"),
QUERY_WORKER_ADDRESS_LIST_FAIL(10178, "query worker address list fail ", "查询worker地址列表失败"),
UDF_FUNCTION_NOT_EXIST(20001, "UDF function not found", "UDF函数不存在"),
UDF_FUNCTION_EXISTS(20002, "UDF function already exists", "UDF函数已存在"),

104
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/interceptor/RateLimitInterceptor.java Normal file
View File

@ -0,0 +1,104 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.interceptor;
import org.apache.dolphinscheduler.api.configuration.TrafficConfiguration;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.commons.collections.MapUtils;
import java.util.Map;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.http.HttpStatus;
import org.springframework.web.servlet.HandlerInterceptor;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import com.google.common.util.concurrent.RateLimiter;
/**
* This interceptor is used to control the traffic, consists with global traffic control and tenant-leve traffic control.
* If the current coming tenant reaches his tenant-level request quota, his request will be reject fast.
* If the current system request number reaches the global request quota, all coming request will be reject fast.
*/
public class RateLimitInterceptor implements HandlerInterceptor {
private static final Logger logger = LoggerFactory.getLogger(RateLimitInterceptor.class);
private TrafficConfiguration trafficConfiguration;
private RateLimiter globalRateLimiter;
private LoadingCache<String, RateLimiter> tenantRateLimiterCache = CacheBuilder.newBuilder()
.maximumSize(100)
.expireAfterAccess(10, TimeUnit.MINUTES)
.build(new CacheLoader<String, RateLimiter>() {
@Override
public RateLimiter load(String token) {
// use tenant customize rate limit
Map<String, Integer> customizeTenantQpsRate = trafficConfiguration.getCustomizeTenantQpsRate();
int tenantQuota = trafficConfiguration.getDefaultTenantQpsRate();
if (MapUtils.isNotEmpty(customizeTenantQpsRate)) {
tenantQuota = customizeTenantQpsRate.getOrDefault(token, trafficConfiguration.getDefaultTenantQpsRate());
}
// use tenant default rate limit
return RateLimiter.create(tenantQuota, 1, TimeUnit.SECONDS);
}
});
@Override
public boolean preHandle(HttpServletRequest request, HttpServletResponse response, Object handler) throws ExecutionException {
// tenant-level rate limit
if (trafficConfiguration.isTrafficTenantControlSwitch()) {
String token = request.getHeader("token");
if (StringUtils.isNotEmpty(token)) {
RateLimiter tenantRateLimiter = tenantRateLimiterCache.get(token);
if (!tenantRateLimiter.tryAcquire()) {
response.setStatus(HttpStatus.TOO_MANY_REQUESTS.value());
logger.warn("Too many request, reach tenant rate limit, current tenant:{} qps is {}", token, tenantRateLimiter.getRate());
return false;
}
}
}
// global rate limit
if (trafficConfiguration.isTrafficGlobalControlSwitch()) {
if (!globalRateLimiter.tryAcquire()) {
response.setStatus(HttpStatus.TOO_MANY_REQUESTS.value());
logger.warn("Too many request, reach global rate limit, current qps is {}", globalRateLimiter.getRate());
return false;
}
}
return true;
}
public RateLimitInterceptor(TrafficConfiguration trafficConfiguration) {
this.trafficConfiguration = trafficConfiguration;
if (trafficConfiguration.isTrafficGlobalControlSwitch()) {
this.globalRateLimiter = RateLimiter.create(trafficConfiguration.getMaxGlobalQpsRate(), 1, TimeUnit.SECONDS);
}
}
}

39
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java
View File

@ -18,7 +18,8 @@
package org.apache.dolphinscheduler.api.service;
import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.enums.DbConnectType;
import org.apache.dolphinscheduler.common.datasource.BaseDataSourceParamDTO;
import org.apache.dolphinscheduler.common.datasource.ConnectionParam;
import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.dao.entity.User;
@ -33,26 +34,20 @@ public interface DataSourceService {
* create data source
*
* @param loginUser login user
* @param name data source name
* @param desc data source description
* @param type data source type
* @param parameter datasource parameters
* @param datasourceParam datasource parameter
* @return create result code
*/
Result<Object> createDataSource(User loginUser, String name, String desc, DbType type, String parameter);
Result<Object> createDataSource(User loginUser, BaseDataSourceParamDTO datasourceParam);
/**
* updateProcessInstance datasource
*
* @param loginUser login user
* @param name data source name
* @param desc data source description
* @param type data source type
* @param parameter datasource parameters
* @param id data source id
* @param id data source id
* @param dataSourceParam data source params
* @return update result code
*/
Result<Object> updateDataSource(int id, User loginUser, String name, String desc, DbType type, String parameter);
Result<Object> updateDataSource(int id, User loginUser, BaseDataSourceParamDTO dataSourceParam);
/**
* updateProcessInstance datasource
@ -97,7 +92,7 @@ public interface DataSourceService {
* @param parameter data source parameters
* @return true if connect successfully, otherwise false
*/
Result<Object> checkConnection(DbType type, String parameter);
Result<Object> checkConnection(DbType type, ConnectionParam parameter);
/**
* test connection
@ -107,24 +102,6 @@ public interface DataSourceService {
*/
Result<Object> connectionTest(int id);
/**
* build paramters
*
* @param type data source type
* @param host data source host
* @param port data source port
* @param database data source database name
* @param userName user name
* @param password password
* @param other other parameters
* @param principal principal
* @return datasource parameter
*/
String buildParameter(DbType type, String host,
String port, String database, String principal, String userName,
String password, DbConnectType connectType, String other,
String javaSecurityKrb5Conf, String loginUserKeytabUsername, String loginUserKeytabPath);
/**
* delete datasource
*

7
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/WorkerGroupService.java
View File

@ -62,4 +62,11 @@ public interface WorkerGroupService {
*/
Map<String, Object> deleteWorkerGroupById(User loginUser, Integer id);
/**
* query all worker address list
*
* @return all worker address list
*/
Map<String, Object> getWorkerAddressList();
}

358
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataSourceServiceImpl.java
View File

@ -18,37 +18,30 @@
package org.apache.dolphinscheduler.api.service.impl;
import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.exceptions.ServiceException;
import org.apache.dolphinscheduler.api.service.DataSourceService;
import org.apache.dolphinscheduler.api.utils.PageInfo;
import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.DbConnectType;
import org.apache.dolphinscheduler.common.datasource.BaseConnectionParam;
import org.apache.dolphinscheduler.common.datasource.BaseDataSourceParamDTO;
import org.apache.dolphinscheduler.common.datasource.ConnectionParam;
import org.apache.dolphinscheduler.common.datasource.DatasourceUtil;
import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.common.utils.CommonUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.dao.datasource.BaseDataSource;
import org.apache.dolphinscheduler.dao.datasource.DataSourceFactory;
import org.apache.dolphinscheduler.dao.datasource.MySQLDataSource;
import org.apache.dolphinscheduler.dao.datasource.OracleDataSource;
import org.apache.dolphinscheduler.dao.entity.DataSource;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.mapper.DataSourceMapper;
import org.apache.dolphinscheduler.dao.mapper.DataSourceUserMapper;
import org.apache.commons.collections4.MapUtils;
import java.sql.Connection;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.regex.Pattern;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -68,25 +61,6 @@ public class DataSourceServiceImpl extends BaseServiceImpl implements DataSource
private static final Logger logger = LoggerFactory.getLogger(DataSourceServiceImpl.class);
public static final String NAME = "name";
public static final String NOTE = "note";
public static final String TYPE = "type";
public static final String HOST = "host";
public static final String PORT = "port";
public static final String PRINCIPAL = "principal";
public static final String DATABASE = "database";
public static final String USER_NAME = "userName";
public static final String OTHER = "other";
private static final Pattern IPV4_PATTERN = Pattern.compile("^[a-zA-Z0-9\\_\\-\\.]+$");
private static final Pattern IPV6_PATTERN = Pattern.compile("^[a-zA-Z0-9\\_\\-\\.\\:\\[\\]]+$");
private static final Pattern DATABASE_PATTER = Pattern.compile("^[a-zA-Z0-9\\_\\-\\.]+$");
private static final Pattern PARAMS_PATTER = Pattern.compile("^[a-zA-Z0-9]+$");
@Autowired
private DataSourceMapper dataSourceMapper;
@ -97,23 +71,23 @@ public class DataSourceServiceImpl extends BaseServiceImpl implements DataSource
* create data source
*
* @param loginUser login user
* @param name data source name
* @param desc data source description
* @param type data source type
* @param parameter datasource parameters
* @param datasourceParam datasource parameters
* @return create result code
*/
@Override
public Result<Object> createDataSource(User loginUser, String name, String desc, DbType type, String parameter) {
public Result<Object> createDataSource(User loginUser, BaseDataSourceParamDTO datasourceParam) {
DatasourceUtil.checkDatasourceParam(datasourceParam);
Result<Object> result = new Result<>();
// check name can use or not
if (checkName(name)) {
if (checkName(datasourceParam.getName())) {
putMsg(result, Status.DATASOURCE_EXIST);
return result;
}
Result<Object> isConnection = checkConnection(type, parameter);
// check connect
ConnectionParam connectionParam = DatasourceUtil.buildConnectionParams(datasourceParam);
Result<Object> isConnection = checkConnection(datasourceParam.getType(), connectionParam);
if (Status.SUCCESS.getCode() != isConnection.getCode()) {
putMsg(result, Status.DATASOURCE_CONNECT_FAILED);
return result;
}
@ -121,12 +95,12 @@ public class DataSourceServiceImpl extends BaseServiceImpl implements DataSource
DataSource dataSource = new DataSource();
Date now = new Date();
dataSource.setName(name.trim());
dataSource.setNote(desc);
dataSource.setName(datasourceParam.getName().trim());
dataSource.setNote(datasourceParam.getNote());
dataSource.setUserId(loginUser.getId());
dataSource.setUserName(loginUser.getUserName());
dataSource.setType(type);
dataSource.setConnectionParams(parameter);
dataSource.setType(datasourceParam.getType());
dataSource.setConnectionParams(JSONUtils.toJsonString(connectionParam));
dataSource.setCreateTime(now);
dataSource.setUpdateTime(now);
dataSourceMapper.insert(dataSource);
@ -148,8 +122,8 @@ public class DataSourceServiceImpl extends BaseServiceImpl implements DataSource
* @return update result code
*/
@Override
public Result<Object> updateDataSource(int id, User loginUser, String name, String desc, DbType type, String parameter) {
public Result<Object> updateDataSource(int id, User loginUser, BaseDataSourceParamDTO dataSourceParam) {
DatasourceUtil.checkDatasourceParam(dataSourceParam);
Result<Object> result = new Result<>();
// determine whether the data source exists
DataSource dataSource = dataSourceMapper.selectById(id);
@ -164,33 +138,31 @@ public class DataSourceServiceImpl extends BaseServiceImpl implements DataSource
}
//check name can use or not
if (!name.trim().equals(dataSource.getName()) && checkName(name)) {
if (!dataSource.getName().trim().equals(dataSource.getName()) && checkName(dataSource.getName())) {
putMsg(result, Status.DATASOURCE_EXIST);
return result;
}
//check password，if the password is not updated, set to the old password.
ObjectNode paramObject = JSONUtils.parseObject(parameter);
String password = paramObject.path(Constants.PASSWORD).asText();
BaseConnectionParam connectionParam = (BaseConnectionParam) DatasourceUtil.buildConnectionParams(dataSourceParam);
String password = connectionParam.getPassword();
if (StringUtils.isBlank(password)) {
String oldConnectionParams = dataSource.getConnectionParams();
ObjectNode oldParams = JSONUtils.parseObject(oldConnectionParams);
paramObject.put(Constants.PASSWORD, oldParams.path(Constants.PASSWORD).asText());
connectionParam.setPassword(oldParams.path(Constants.PASSWORD).asText());
}
// connectionParams json
String connectionParams = paramObject.toString();
Result<Object> isConnection = checkConnection(type, parameter);
Result<Object> isConnection = checkConnection(dataSource.getType(), connectionParam);
if (Status.SUCCESS.getCode() != isConnection.getCode()) {
return result;
}
Date now = new Date();
dataSource.setName(name.trim());
dataSource.setNote(desc);
dataSource.setName(dataSource.getName().trim());
dataSource.setNote(dataSource.getNote());
dataSource.setUserName(loginUser.getUserName());
dataSource.setType(type);
dataSource.setConnectionParams(connectionParams);
dataSource.setType(dataSource.getType());
dataSource.setConnectionParams(JSONUtils.toJsonString(connectionParam));
dataSource.setUpdateTime(now);
dataSourceMapper.updateById(dataSource);
putMsg(result, Status.SUCCESS);
@ -218,79 +190,13 @@ public class DataSourceServiceImpl extends BaseServiceImpl implements DataSource
return result;
}
// type
String dataSourceType = dataSource.getType().toString();
// name
String dataSourceName = dataSource.getName();
// desc
String desc = dataSource.getNote();
// parameter
String parameter = dataSource.getConnectionParams();
BaseDataSourceParamDTO baseDataSourceParamDTO = DatasourceUtil.buildDatasourceParamDTO(
dataSource.getType(), dataSource.getConnectionParams());
baseDataSourceParamDTO.setId(dataSource.getId());
baseDataSourceParamDTO.setName(dataSource.getName());
baseDataSourceParamDTO.setNote(dataSource.getNote());
BaseDataSource datasourceForm = DataSourceFactory.getDatasource(dataSource.getType(), parameter);
DbConnectType connectType = null;
String hostSeperator = Constants.DOUBLE_SLASH;
if (DbType.ORACLE.equals(dataSource.getType())) {
connectType = ((OracleDataSource) datasourceForm).getConnectType();
if (DbConnectType.ORACLE_SID.equals(connectType)) {
hostSeperator = Constants.AT_SIGN;
}
}
String database = datasourceForm.getDatabase();
// jdbc connection params
String other = datasourceForm.getOther();
String address = datasourceForm.getAddress();
String[] hostsPorts = getHostsAndPort(address, hostSeperator);
// ip host
String host = hostsPorts[0];
// prot
String port = hostsPorts[1];
String separator = "";
switch (dataSource.getType()) {
case HIVE:
case SQLSERVER:
separator = ";";
break;
case MYSQL:
case POSTGRESQL:
case CLICKHOUSE:
case ORACLE:
case PRESTO:
separator = "&";
break;
default:
separator = "&";
break;
}
Map<String, String> otherMap = new LinkedHashMap<>();
if (other != null) {
String[] configs = other.split(separator);
for (String config : configs) {
otherMap.put(config.split("=")[0], config.split("=")[1]);
}
}
Map<String, Object> map = new HashMap<>();
map.put(NAME, dataSourceName);
map.put(NOTE, desc);
map.put(TYPE, dataSourceType);
if (connectType != null) {
map.put(Constants.ORACLE_DB_CONNECT_TYPE, connectType);
}
map.put(HOST, host);
map.put(PORT, port);
map.put(PRINCIPAL, datasourceForm.getPrincipal());
map.put(Constants.KERBEROS_KRB5_CONF_PATH, datasourceForm.getJavaSecurityKrb5Conf());
map.put(Constants.KERBEROS_KEY_TAB_USERNAME, datasourceForm.getLoginUserKeytabUsername());
map.put(Constants.KERBEROS_KEY_TAB_PATH, datasourceForm.getLoginUserKeytabPath());
map.put(DATABASE, database);
map.put(USER_NAME, datasourceForm.getUser());
map.put(OTHER, otherMap);
result.put(Constants.DATA_LIST, map);
result.put(Constants.DATA_LIST, baseDataSourceParamDTO);
putMsg(result, Status.SUCCESS);
return result;
}
@ -400,14 +306,9 @@ public class DataSourceServiceImpl extends BaseServiceImpl implements DataSource
* @return true if connect successfully, otherwise false
*/
@Override
public Result<Object> checkConnection(DbType type, String parameter) {
public Result<Object> checkConnection(DbType type, ConnectionParam connectionParam) {
Result<Object> result = new Result<>();
BaseDataSource datasource = DataSourceFactory.getDatasource(type, parameter);
if (datasource == null) {
putMsg(result, Status.DATASOURCE_TYPE_NOT_EXIST, type);
return result;
}
try (Connection connection = datasource.getConnection()) {
try (Connection connection = DatasourceUtil.getConnection(type, connectionParam)) {
if (connection == null) {
putMsg(result, Status.CONNECTION_TEST_FAILURE);
return result;
@ -415,7 +316,7 @@ public class DataSourceServiceImpl extends BaseServiceImpl implements DataSource
putMsg(result, Status.SUCCESS);
return result;
} catch (Exception e) {
logger.error("datasource test connection error, dbType:{}, jdbcUrl:{}, message:{}.", type, datasource.getJdbcUrl(), e.getMessage());
logger.error("datasource test connection error, dbType:{}, connectionParam:{}, message:{}.", type, connectionParam, e.getMessage());
return new Result<>(Status.CONNECTION_TEST_FAILURE.getCode(), e.getMessage());
}
}
@ -434,135 +335,7 @@ public class DataSourceServiceImpl extends BaseServiceImpl implements DataSource
putMsg(result, Status.RESOURCE_NOT_EXIST);
return result;
}
return checkConnection(dataSource.getType(), dataSource.getConnectionParams());
}
/**
* build paramters
*
* @param type data source type
* @param host data source host
* @param port data source port
* @param database data source database name
* @param userName user name
* @param password password
* @param other other parameters
* @param principal principal
* @return datasource parameter
*/
@Override
public String buildParameter(DbType type, String host,
String port, String database, String principal, String userName,
String password, DbConnectType connectType, String other,
String javaSecurityKrb5Conf, String loginUserKeytabUsername, String loginUserKeytabPath) {
checkParams(type, port, host, database, other);
String address = buildAddress(type, host, port, connectType);
Map<String, Object> parameterMap = new LinkedHashMap<>();
String jdbcUrl;
if (DbType.SQLSERVER == type) {
jdbcUrl = address + ";databaseName=" + database;
} else {
jdbcUrl = address + "/" + database;
}
if (Constants.ORACLE.equals(type.name())) {
parameterMap.put(Constants.ORACLE_DB_CONNECT_TYPE, connectType);
}
if (CommonUtils.getKerberosStartupState()
&& (type == DbType.HIVE || type == DbType.SPARK)) {
jdbcUrl += ";principal=" + principal;
}
String separator = "";
if (Constants.MYSQL.equals(type.name())
|| Constants.POSTGRESQL.equals(type.name())
|| Constants.CLICKHOUSE.equals(type.name())
|| Constants.ORACLE.equals(type.name())
|| Constants.PRESTO.equals(type.name())) {
separator = "&";
} else if (Constants.HIVE.equals(type.name())
|| Constants.SPARK.equals(type.name())
|| Constants.DB2.equals(type.name())
|| Constants.SQLSERVER.equals(type.name())) {
separator = ";";
}
parameterMap.put(Constants.ADDRESS, address);
parameterMap.put(Constants.DATABASE, database);
parameterMap.put(Constants.JDBC_URL, jdbcUrl);
parameterMap.put(Constants.USER, userName);
parameterMap.put(Constants.PASSWORD, CommonUtils.encodePassword(password));
if (CommonUtils.getKerberosStartupState()
&& (type == DbType.HIVE || type == DbType.SPARK)) {
parameterMap.put(Constants.PRINCIPAL, principal);
parameterMap.put(Constants.KERBEROS_KRB5_CONF_PATH, javaSecurityKrb5Conf);
parameterMap.put(Constants.KERBEROS_KEY_TAB_USERNAME, loginUserKeytabUsername);
parameterMap.put(Constants.KERBEROS_KEY_TAB_PATH, loginUserKeytabPath);
}
Map<String, String> map = JSONUtils.toMap(other);
if (type == DbType.MYSQL) {
map = MySQLDataSource.buildOtherParams(other);
}
if (MapUtils.isNotEmpty(map)) {
StringBuilder otherSb = new StringBuilder();
for (Map.Entry<String, String> entry : map.entrySet()) {
otherSb.append(String.format("%s=%s%s", entry.getKey(), entry.getValue(), separator));
}
if (!Constants.DB2.equals(type.name())) {
otherSb.deleteCharAt(otherSb.length() - 1);
}
parameterMap.put(Constants.OTHER, otherSb);
}
if (logger.isDebugEnabled()) {
logger.info("parameters map:{}", JSONUtils.toJsonString(parameterMap));
}
return JSONUtils.toJsonString(parameterMap);
}
private String buildAddress(DbType type, String host, String port, DbConnectType connectType) {
StringBuilder sb = new StringBuilder();
if (Constants.MYSQL.equals(type.name())) {
sb.append(Constants.JDBC_MYSQL);
sb.append(host).append(":").append(port);
} else if (Constants.POSTGRESQL.equals(type.name())) {
sb.append(Constants.JDBC_POSTGRESQL);
sb.append(host).append(":").append(port);
} else if (Constants.HIVE.equals(type.name()) || Constants.SPARK.equals(type.name())) {
sb.append(Constants.JDBC_HIVE_2);
String[] hostArray = host.split(",");
if (hostArray.length > 0) {
for (String zkHost : hostArray) {
sb.append(String.format("%s:%s,", zkHost, port));
}
sb.deleteCharAt(sb.length() - 1);
}
} else if (Constants.CLICKHOUSE.equals(type.name())) {
sb.append(Constants.JDBC_CLICKHOUSE);
sb.append(host).append(":").append(port);
} else if (Constants.ORACLE.equals(type.name())) {
if (connectType == DbConnectType.ORACLE_SID) {
sb.append(Constants.JDBC_ORACLE_SID);
} else {
sb.append(Constants.JDBC_ORACLE_SERVICE_NAME);
}
sb.append(host).append(":").append(port);
} else if (Constants.SQLSERVER.equals(type.name())) {
sb.append(Constants.JDBC_SQLSERVER);
sb.append(host).append(":").append(port);
} else if (Constants.DB2.equals(type.name())) {
sb.append(Constants.JDBC_DB2);
sb.append(host).append(":").append(port);
} else if (Constants.PRESTO.equals(type.name())) {
sb.append(Constants.JDBC_PRESTO);
sb.append(host).append(":").append(port);
}
return sb.toString();
return checkConnection(dataSource.getType(), DatasourceUtil.buildConnectionParams(dataSource.getType(), dataSource.getConnectionParams()));
}
/**
@ -661,61 +434,4 @@ public class DataSourceServiceImpl extends BaseServiceImpl implements DataSource
return result;
}
/**
* get host and port by address
*
* @param address address
* @param separator separator
* @return sting array: [host,port]
*/
private String[] getHostsAndPort(String address, String separator) {
String[] result = new String[2];
String[] tmpArray = address.split(separator);
String hostsAndPorts = tmpArray[tmpArray.length - 1];
StringBuilder hosts = new StringBuilder();
String[] hostPortArray = hostsAndPorts.split(Constants.COMMA);
String port = hostPortArray[0].split(Constants.COLON)[1];
for (String hostPort : hostPortArray) {
hosts.append(hostPort.split(Constants.COLON)[0]).append(Constants.COMMA);
}
hosts.deleteCharAt(hosts.length() - 1);
result[0] = hosts.toString();
result[1] = port;
return result;
}
private void checkParams(DbType type, String port, String host, String database, String other) {
if (null == DbType.of(type.getCode())) {
throw new ServiceException(Status.DATASOURCE_DB_TYPE_ILLEGAL);
}
if (!isNumeric(port)) {
throw new ServiceException(Status.DATASOURCE_PORT_ILLEGAL);
}
if (!IPV4_PATTERN.matcher(host).matches() || !IPV6_PATTERN.matcher(host).matches()) {
throw new ServiceException(Status.DATASOURCE_HOST_ILLEGAL);
}
if (!DATABASE_PATTER.matcher(database).matches()) {
throw new ServiceException(Status.DATASOURCE_NAME_ILLEGAL);
}
if (StringUtils.isBlank(other)) {
return;
}
Map<String, String> map = JSONUtils.toMap(other);
if (MapUtils.isEmpty(map)) {
return;
}
boolean paramsCheck = map.entrySet().stream().allMatch(p -> PARAMS_PATTER.matcher(p.getValue()).matches());
if (!paramsCheck) {
throw new ServiceException(Status.DATASOURCE_OTHER_PARAMS_ILLEGAL);
}
}
private static boolean isNumeric(String str) {
for (int i = str.length(); --i >= 0; ) {
if (!Character.isDigit(str.charAt(i))) {
return false;
}
}
return true;
}
}

22
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkerGroupServiceImpl.java
View File

@ -81,10 +81,6 @@ public class WorkerGroupServiceImpl extends BaseServiceImpl implements WorkerGro
if (isNotAdmin(loginUser, result)) {
return result;
}
if (Constants.DOCKER_MODE && !Constants.KUBERNETES_MODE) {
putMsg(result, Status.CREATE_WORKER_GROUP_FORBIDDEN_IN_DOCKER);
return result;
}
if (StringUtils.isEmpty(name)) {
putMsg(result, Status.NAME_NULL);
return result;
@ -303,10 +299,6 @@ public class WorkerGroupServiceImpl extends BaseServiceImpl implements WorkerGro
if (isNotAdmin(loginUser, result)) {
return result;
}
if (Constants.DOCKER_MODE && !Constants.KUBERNETES_MODE) {
putMsg(result, Status.DELETE_WORKER_GROUP_FORBIDDEN_IN_DOCKER);
return result;
}
WorkerGroup workerGroup = workerGroupMapper.selectById(id);
if (workerGroup == null) {
putMsg(result, Status.DELETE_WORKER_GROUP_NOT_EXIST);
@ -323,4 +315,18 @@ public class WorkerGroupServiceImpl extends BaseServiceImpl implements WorkerGro
return result;
}
/**
* query all worker address list
*
* @return all worker address list
*/
@Override
public Map<String, Object> getWorkerAddressList() {
Map<String, Object> result = new HashMap<>();
List<String> serverNodeList = zookeeperMonitor.getServerNodeList(ZKNodeType.WORKER, true);
result.put(Constants.DATA_LIST, serverNodeList);
putMsg(result, Status.SUCCESS);
return result;
}
}

13
dolphinscheduler-api/src/main/resources/application-api.properties
View File

@ -35,10 +35,10 @@ spring.servlet.multipart.max-request-size=1024MB
server.compression.enabled=true
server.compression.mime-types=text/html,text/xml,text/plain,text/css,text/javascript,application/javascript,application/json,application/xml
# post content
# max http post size
server.jetty.max-http-form-post-size=5000000
# i18n
# messages encoding
spring.messages.encoding=UTF-8
# i18n classpath folder , file prefix messages， if have many files, use "," seperator
@ -47,6 +47,15 @@ spring.messages.basename=i18n/messages
# Authentication types (supported types: PASSWORD)
security.authentication.type=PASSWORD
# Traffic control, if you turn on this config, the maximum number of request/s will be limited.
# global max request number per second
# default tenant-level max request number
#traffic.control.global.switch=true
#traffic.control.max.global.qps.rate=500
#traffic.control.tenant.switch=true
#traffic.control.default.tenant.qps.rate=10
#traffic.control.customize.tenant.qps.rate={'tenant1':11,'tenant2':20}
#============================================================================
# LDAP Config
# mock ldap server from https://www.forumsys.com/tutorials/integration-how-to/ldap/online-ldap-test-server/

2
dolphinscheduler-api/src/main/resources/i18n/messages_zh_CN.properties
View File

@ -52,7 +52,7 @@ COUNT_PROCESS_INSTANCE_NOTES=统计流程实例状态
COUNT_PROCESS_DEFINITION_BY_USER_NOTES=统计用户创建的流程定义
COUNT_COMMAND_STATE_NOTES=统计命令状态
COUNT_QUEUE_STATE_NOTES=统计队列里任务状态
ACCESS_TOKEN_TAG=access token相关操作，需要先登录
ACCESS_TOKEN_TAG=访问token相关操作
MONITOR_TAG=监控相关操作
MASTER_LIST_NOTES=master服务列表
WORKER_LIST_NOTES=worker服务列表

60
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/configuration/TrafficConfigurationTest.java Normal file
View File

@ -0,0 +1,60 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.configuration;
import org.apache.commons.collections.MapUtils;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.junit4.SpringRunner;
@RunWith(SpringRunner.class)
@SpringBootTest
public class TrafficConfigurationTest {
@Autowired
private TrafficConfiguration trafficConfiguration;
@Test
public void isTrafficGlobalControlSwitch() {
Assert.assertFalse(trafficConfiguration.isTrafficGlobalControlSwitch());
}
@Test
public void getMaxGlobalQpsLimit() {
Assert.assertEquals(300, (int) trafficConfiguration.getMaxGlobalQpsRate());
}
@Test
public void isTrafficTenantControlSwitch() {
Assert.assertFalse(trafficConfiguration.isTrafficTenantControlSwitch());
}
@Test
public void getDefaultTenantQpsLimit() {
Assert.assertEquals(10, (int) trafficConfiguration.getDefaultTenantQpsRate());
}
@Test
public void getCustomizeTenantQpsRate() {
Assert.assertTrue(MapUtils.isEmpty(trafficConfiguration.getCustomizeTenantQpsRate()));
}
}

3
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/AbstractControllerTest.java
View File

@ -38,7 +38,6 @@ import org.springframework.web.context.WebApplicationContext;
/**
* abstract controller test
*/
@Ignore
@RunWith(SpringRunner.class)
@SpringBootTest(classes = ApiApplicationServer.class)
public class AbstractControllerTest {
@ -64,7 +63,7 @@ public class AbstractControllerTest {
}
@After
public void after(){
public void after() throws Exception {
sessionService.signOut("127.0.0.1", user);
}

28
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/DataSourceControllerTest.java
View File

@ -24,8 +24,11 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.
import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.datasource.mysql.MysqlDatasourceParamDTO;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import java.util.HashMap;
import org.junit.Assert;
import org.junit.Ignore;
import org.junit.Test;
@ -46,20 +49,19 @@ public class DataSourceControllerTest extends AbstractControllerTest{
@Ignore
@Test
public void testCreateDataSource() throws Exception {
MultiValueMap<String, String> paramsMap = new LinkedMultiValueMap<>();
paramsMap.add("name","mysql");
paramsMap.add("node","mysql data source test");
paramsMap.add("type","MYSQL");
paramsMap.add("host","192.168.xxxx.xx");
paramsMap.add("port","3306");
paramsMap.add("principal","");
paramsMap.add("database","dolphinscheduler");
paramsMap.add("userName","root");
paramsMap.add("password","root@123");
paramsMap.add("other","");
MysqlDatasourceParamDTO mysqlDatasourceParam = new MysqlDatasourceParamDTO();
mysqlDatasourceParam.setName("mysql");
mysqlDatasourceParam.setNote("mysql data source test");
mysqlDatasourceParam.setHost("192.168.xxxx.xx");
mysqlDatasourceParam.setPort(3306);
mysqlDatasourceParam.setDatabase("dolphinscheduler");
mysqlDatasourceParam.setUserName("root");
mysqlDatasourceParam.setPassword("root@123");
mysqlDatasourceParam.setOther(new HashMap<>());
MvcResult mvcResult = mockMvc.perform(post("/datasources/create")
.header("sessionId", sessionId)
.params(paramsMap))
.contentType(MediaType.APPLICATION_JSON_UTF8)
.content(JSONUtils.toJsonString(mysqlDatasourceParam)))
.andExpect(status().isCreated())
.andExpect(content().contentType(MediaType.APPLICATION_JSON_UTF8))
.andReturn();
@ -91,7 +93,7 @@ public class DataSourceControllerTest extends AbstractControllerTest{
.andExpect(content().contentType(MediaType.APPLICATION_JSON_UTF8))
.andReturn();
Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue());
Assert.assertEquals(Status.SUCCESS.getCode(), result.getCode().intValue());
logger.info(mvcResult.getResponse().getContentAsString());
}

23
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProcessDefinitionControllerTest.java
View File

@ -383,6 +383,13 @@ public class ProcessDefinitionControllerTest {
@Test
public void testQueryProcessDefinitionVersions() {
String projectName = "test";
Result result = processDefinitionController.queryProcessDefinitionVersions(user, projectName, 1, -10, 1);
Assert.assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR.getCode(), result.getCode().intValue());
result = processDefinitionController.queryProcessDefinitionVersions(user, projectName, -1, 10, 1);
Assert.assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR.getCode(), result.getCode().intValue());
Map<String, Object> resultMap = new HashMap<>();
putMsg(resultMap, Status.SUCCESS);
resultMap.put(Constants.DATA_LIST, new PageInfo<ProcessDefinitionLog>(1, 10));
@ -393,7 +400,7 @@ public class ProcessDefinitionControllerTest {
, 10
, 1))
.thenReturn(resultMap);
Result result = processDefinitionController.queryProcessDefinitionVersions(
result = processDefinitionController.queryProcessDefinitionVersions(
user
, projectName
, 1
@ -408,17 +415,8 @@ public class ProcessDefinitionControllerTest {
String projectName = "test";
Map<String, Object> resultMap = new HashMap<>();
putMsg(resultMap, Status.SUCCESS);
Mockito.when(processDefinitionService.switchProcessDefinitionVersion(
user
, projectName
, 1
, 10))
.thenReturn(resultMap);
Result result = processDefinitionController.switchProcessDefinitionVersion(
user
, projectName
, 1
, 10);
Mockito.when(processDefinitionService.switchProcessDefinitionVersion(user, projectName, 1, 10)).thenReturn(resultMap);
Result result = processDefinitionController.switchProcessDefinitionVersion(user, projectName, 1, 10);
Assert.assertEquals(Status.SUCCESS.getCode(), (int) result.getCode());
}
@ -439,7 +437,6 @@ public class ProcessDefinitionControllerTest {
, projectName
, 1
, 10);
Assert.assertEquals(Status.SUCCESS.getCode(), (int) result.getCode());
}

70
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProjectControllerTest.java
View File

@ -26,7 +26,9 @@ import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test;
import org.slf4j.Logger;
@ -36,19 +38,32 @@ import org.springframework.test.web.servlet.MvcResult;
import org.springframework.util.LinkedMultiValueMap;
import org.springframework.util.MultiValueMap;
import com.fasterxml.jackson.core.type.TypeReference;
/**
* project controller test
*/
public class ProjectControllerTest extends AbstractControllerTest{
public class ProjectControllerTest extends AbstractControllerTest {
private static Logger logger = LoggerFactory.getLogger(ProjectControllerTest.class);
@Test
public void testCreateProject() throws Exception {
private String projectId;
@Before
public void before() throws Exception {
projectId = testCreateProject("project_test1", "the test project");
}
@After
public void after() throws Exception {
testDeleteProject(projectId);
}
private String testCreateProject(String projectName, String description) throws Exception {
MultiValueMap<String, String> paramsMap = new LinkedMultiValueMap<>();
paramsMap.add("projectName","project_test1");
paramsMap.add("description","the test project");
paramsMap.add("projectName",projectName);
paramsMap.add("description",description);
MvcResult mvcResult = mockMvc.perform(post("/projects/create")
.header(SESSION_ID, sessionId)
@ -57,17 +72,19 @@ public class ProjectControllerTest extends AbstractControllerTest{
.andExpect(content().contentType(MediaType.APPLICATION_JSON_UTF8))
.andReturn();
Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), new TypeReference<Result<String>>() {});
Assert.assertEquals(Status.SUCCESS.getCode(), result.getCode().intValue());
Assert.assertNotNull(result.getData());
logger.info(mvcResult.getResponse().getContentAsString());
logger.info("create project return result:{}", mvcResult.getResponse().getContentAsString());
return (String)result.getData();
}
@Test
public void testUpdateProject() throws Exception {
MultiValueMap<String, String> paramsMap = new LinkedMultiValueMap<>();
paramsMap.add("projectId","18");
paramsMap.add("projectId", projectId);
paramsMap.add("projectName","project_test_update");
paramsMap.add("desc","the test project update");
@ -80,16 +97,15 @@ public class ProjectControllerTest extends AbstractControllerTest{
Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue());
logger.info(mvcResult.getResponse().getContentAsString());
logger.info("update project return result:{}", mvcResult.getResponse().getContentAsString());
}
@Test
public void testQueryProjectById() throws Exception {
MultiValueMap<String, String> paramsMap = new LinkedMultiValueMap<>();
paramsMap.add("projectId","18");
paramsMap.add("projectId", projectId);
MvcResult mvcResult = mockMvc.perform(get("/projects/query-by-id")
.header(SESSION_ID, sessionId)
@ -99,8 +115,10 @@ public class ProjectControllerTest extends AbstractControllerTest{
.andReturn();
Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue());
Assert.assertEquals(Status.SUCCESS.getCode(), result.getCode().intValue());
logger.info(mvcResult.getResponse().getContentAsString());
logger.info("query project by id :{}, return result:{}", projectId, mvcResult.getResponse().getContentAsString());
}
@Test
@ -111,7 +129,6 @@ public class ProjectControllerTest extends AbstractControllerTest{
paramsMap.add("pageSize","2");
paramsMap.add("pageNo","2");
MvcResult mvcResult = mockMvc.perform(get("/projects/list-paging")
.header(SESSION_ID, sessionId)
.params(paramsMap))
@ -121,10 +138,9 @@ public class ProjectControllerTest extends AbstractControllerTest{
Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue());
logger.info(mvcResult.getResponse().getContentAsString());
logger.info("query list-paging project return result:{}", mvcResult.getResponse().getContentAsString());
}
@Test
public void testQueryUnauthorizedProject() throws Exception {
@ -140,16 +156,14 @@ public class ProjectControllerTest extends AbstractControllerTest{
Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue());
logger.info(mvcResult.getResponse().getContentAsString());
logger.info("query unauth project return result:{}", mvcResult.getResponse().getContentAsString());
}
@Test
public void testQueryAuthorizedProject() throws Exception {
MultiValueMap<String, String> paramsMap = new LinkedMultiValueMap<>();
paramsMap.add("userId","2");
paramsMap.add("userId",String.valueOf(user.getId()));
MvcResult mvcResult = mockMvc.perform(get("/projects/authed-project")
.header(SESSION_ID, sessionId)
@ -160,9 +174,9 @@ public class ProjectControllerTest extends AbstractControllerTest{
Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue());
logger.info(mvcResult.getResponse().getContentAsString());
}
logger.info("query authed project return result:{}", mvcResult.getResponse().getContentAsString());
}
@Test
public void testQueryAllProjectList() throws Exception {
@ -178,7 +192,8 @@ public class ProjectControllerTest extends AbstractControllerTest{
Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue());
logger.info(mvcResult.getResponse().getContentAsString());
logger.info("query all project return result:{}", mvcResult.getResponse().getContentAsString());
}
@Ignore
@ -200,12 +215,10 @@ public class ProjectControllerTest extends AbstractControllerTest{
logger.info(mvcResult.getResponse().getContentAsString());
}
@Test
public void testDeleteProject() throws Exception {
private void testDeleteProject(String projectId) throws Exception {
MultiValueMap<String, String> paramsMap = new LinkedMultiValueMap<>();
paramsMap.add("projectId","18");
paramsMap.add("projectId", projectId);
MvcResult mvcResult = mockMvc.perform(get("/projects/delete")
.header(SESSION_ID, sessionId)
@ -216,6 +229,7 @@ public class ProjectControllerTest extends AbstractControllerTest{
Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue());
logger.info(mvcResult.getResponse().getContentAsString());
logger.info("delete project return result:{}", mvcResult.getResponse().getContentAsString());
}
}

53
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/QueueControllerTest.java
View File

@ -38,10 +38,12 @@ import org.springframework.util.MultiValueMap;
/**
* queue controller test
*/
public class QueueControllerTest extends AbstractControllerTest{
public class QueueControllerTest extends AbstractControllerTest {
private static Logger logger = LoggerFactory.getLogger(QueueControllerTest.class);
private static final String QUEUE_CREATE_STRING = "queue1";
@Test
public void testQueryList() throws Exception {
@ -53,14 +55,13 @@ public class QueueControllerTest extends AbstractControllerTest{
Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue());
logger.info(mvcResult.getResponse().getContentAsString());
logger.info("query list queue return result:{}", mvcResult.getResponse().getContentAsString());
}
@Test
public void testQueryQueueListPaging() throws Exception {
MultiValueMap<String, String> paramsMap = new LinkedMultiValueMap<>();
//paramsMap.add("processInstanceId","1380");
paramsMap.add("searchVal","");
paramsMap.add("pageNo","1");
paramsMap.add("pageSize","20");
@ -73,18 +74,16 @@ public class QueueControllerTest extends AbstractControllerTest{
.andReturn();
Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue());
logger.info(mvcResult.getResponse().getContentAsString());
logger.info("query list-paging queue return result:{}", mvcResult.getResponse().getContentAsString());
}
@Test
public void testCreateQueue() throws Exception {
MultiValueMap<String, String> paramsMap = new LinkedMultiValueMap<>();
paramsMap.add("queue","ait");
paramsMap.add("queueName","ait");
paramsMap.add("queue", QUEUE_CREATE_STRING);
paramsMap.add("queueName","root.queue1");
MvcResult mvcResult = mockMvc.perform(post("/queue/create")
.header(SESSION_ID, sessionId)
@ -93,17 +92,17 @@ public class QueueControllerTest extends AbstractControllerTest{
.andExpect(content().contentType(MediaType.APPLICATION_JSON_UTF8))
.andReturn();
Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
// Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue());
logger.info(mvcResult.getResponse().getContentAsString());
Assert.assertEquals(Status.SUCCESS.getCode(), result.getCode().intValue());
logger.info("create queue return result:{}", mvcResult.getResponse().getContentAsString());
}
@Test
public void testUpdateQueue() throws Exception {
MultiValueMap<String, String> paramsMap = new LinkedMultiValueMap<>();
paramsMap.add("id","2");
paramsMap.add("queue","ait12");
paramsMap.add("queueName","aitName");
paramsMap.add("id","1");
paramsMap.add("queue","queue2");
paramsMap.add("queueName","root.queue2");
MvcResult mvcResult = mockMvc.perform(post("/queue/update")
.header(SESSION_ID, sessionId)
@ -112,16 +111,17 @@ public class QueueControllerTest extends AbstractControllerTest{
.andExpect(content().contentType(MediaType.APPLICATION_JSON_UTF8))
.andReturn();
Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
//Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue());
logger.info(mvcResult.getResponse().getContentAsString());
Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue());
logger.info("update queue return result:{}", mvcResult.getResponse().getContentAsString());
}
@Test
public void testVerifyQueue() throws Exception {
// queue value exist
MultiValueMap<String, String> paramsMap = new LinkedMultiValueMap<>();
paramsMap.add("queue","ait123");
paramsMap.add("queueName","aitName");
paramsMap.add("queue",QUEUE_CREATE_STRING);
paramsMap.add("queueName","queue.name");
MvcResult mvcResult = mockMvc.perform(post("/queue/verify-queue")
.header(SESSION_ID, sessionId)
@ -130,7 +130,22 @@ public class QueueControllerTest extends AbstractControllerTest{
.andExpect(content().contentType(MediaType.APPLICATION_JSON_UTF8))
.andReturn();
Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
//Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue());
Assert.assertEquals(Status.QUEUE_VALUE_EXIST.getCode(),result.getCode().intValue());
// success
paramsMap.clear();
paramsMap.add("queue","ait123");
paramsMap.add("queueName","aitName");
mvcResult = mockMvc.perform(post("/queue/verify-queue")
.header(SESSION_ID, sessionId)
.params(paramsMap))
.andExpect(status().isOk())
.andExpect(content().contentType(MediaType.APPLICATION_JSON_UTF8))
.andReturn();
result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue());
logger.info(mvcResult.getResponse().getContentAsString());
logger.info("verify queue return result:{}", mvcResult.getResponse().getContentAsString());
}
}

89
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/interceptor/RateLimitInterceptorTest.java Normal file
View File

@ -0,0 +1,89 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.interceptor;
import org.apache.dolphinscheduler.api.configuration.TrafficConfiguration;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.ExecutionException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mockito;
import org.powermock.api.mockito.PowerMockito;
import org.powermock.modules.junit4.PowerMockRunner;
@RunWith(PowerMockRunner.class)
public class RateLimitInterceptorTest {
@Test
public void testPreHandleWithoutControl() throws ExecutionException {
HttpServletRequest request = Mockito.mock(HttpServletRequest.class);
HttpServletResponse response = Mockito.mock(HttpServletResponse.class);
RateLimitInterceptor rateLimitInterceptor = new RateLimitInterceptor(new TrafficConfiguration());
Assert.assertTrue(rateLimitInterceptor.preHandle(request, response, null));
Assert.assertTrue(rateLimitInterceptor.preHandle(request, response, null));
}
@Test
public void testPreHandleWithTenantLevenControl() throws ExecutionException {
TrafficConfiguration trafficConfiguration = new TrafficConfiguration();
trafficConfiguration.setTrafficTenantControlSwitch(true);
Map<String, Integer> map = new HashMap<>();
map.put("tenant1", 2);
map.put("tenant2", 2);
trafficConfiguration.setCustomizeTenantQpsRate(map);
trafficConfiguration.setDefaultTenantQpsRate(4);
RateLimitInterceptor rateLimitInterceptor = new RateLimitInterceptor(trafficConfiguration);
HttpServletRequest tenant1Request = Mockito.mock(HttpServletRequest.class);
HttpServletRequest tenant2Request = Mockito.mock(HttpServletRequest.class);
PowerMockito.when(tenant1Request.getHeader(Mockito.any())).thenReturn("tenant1");
PowerMockito.when(tenant2Request.getHeader(Mockito.any())).thenReturn("tenant2");
HttpServletResponse response = Mockito.mock(HttpServletResponse.class);
for (int i = 0; i < 2; i++) {
rateLimitInterceptor.preHandle(tenant1Request, response, null);
}
Assert.assertFalse(rateLimitInterceptor.preHandle(tenant1Request, response, null));
Assert.assertTrue(rateLimitInterceptor.preHandle(tenant2Request, response, null));
}
@Test
public void testPreHandleWithGlobalControl() throws ExecutionException {
TrafficConfiguration trafficConfiguration = new TrafficConfiguration();
trafficConfiguration.setTrafficTenantControlSwitch(true);
trafficConfiguration.setTrafficGlobalControlSwitch(true);
trafficConfiguration.setMaxGlobalQpsRate(3);
RateLimitInterceptor rateLimitInterceptor = new RateLimitInterceptor(trafficConfiguration);
HttpServletRequest request = Mockito.mock(HttpServletRequest.class);
HttpServletResponse response = Mockito.mock(HttpServletResponse.class);
for (int i = 0; i < 2; i++) {
rateLimitInterceptor.preHandle(request, response, null);
}
Assert.assertFalse(rateLimitInterceptor.preHandle(request, response, null));
}
}

169
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataSourceServiceTest.java
View File

@ -21,15 +21,18 @@ import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.service.impl.DataSourceServiceImpl;
import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.datasource.ConnectionParam;
import org.apache.dolphinscheduler.common.datasource.DatasourceUtil;
import org.apache.dolphinscheduler.common.datasource.hive.HiveDataSourceParamDTO;
import org.apache.dolphinscheduler.common.datasource.mysql.MysqlDatasourceParamDTO;
import org.apache.dolphinscheduler.common.datasource.oracle.OracleDatasourceParamDTO;
import org.apache.dolphinscheduler.common.datasource.postgresql.PostgreSqlDatasourceParamDTO;
import org.apache.dolphinscheduler.common.enums.DbConnectType;
import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.common.enums.UserType;
import org.apache.dolphinscheduler.common.utils.CommonUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.PropertyUtils;
import org.apache.dolphinscheduler.dao.datasource.BaseDataSource;
import org.apache.dolphinscheduler.dao.datasource.DataSourceFactory;
import org.apache.dolphinscheduler.dao.datasource.MySQLDataSource;
import org.apache.dolphinscheduler.dao.entity.DataSource;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.mapper.DataSourceMapper;
@ -37,6 +40,7 @@ import org.apache.dolphinscheduler.dao.mapper.DataSourceUserMapper;
import java.sql.Connection;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@ -56,7 +60,7 @@ import org.powermock.modules.junit4.PowerMockRunner;
*/
@RunWith(PowerMockRunner.class)
@PowerMockIgnore({"sun.security.*", "javax.net.*"})
@PrepareForTest({DataSourceFactory.class, CommonUtils.class})
@PrepareForTest({DatasourceUtil.class, CommonUtils.class})
public class DataSourceServiceTest {
@InjectMocks
@ -70,11 +74,17 @@ public class DataSourceServiceTest {
public void createDataSourceTest() {
User loginUser = getAdminUser();
String dataSourceName = "dataSource01";
String dataSourceDesc = "test dataSource";
DbType dataSourceType = DbType.POSTGRESQL;
String parameter = dataSourceService.buildParameter(dataSourceType, "172.16.133.200", "5432", "dolphinscheduler", null, "postgres", "", null, null, null, null, null);
PostgreSqlDatasourceParamDTO postgreSqlDatasourceParam = new PostgreSqlDatasourceParamDTO();
postgreSqlDatasourceParam.setDatabase(dataSourceName);
postgreSqlDatasourceParam.setNote(dataSourceDesc);
postgreSqlDatasourceParam.setHost("172.16.133.200");
postgreSqlDatasourceParam.setPort(5432);
postgreSqlDatasourceParam.setDatabase("dolphinscheduler");
postgreSqlDatasourceParam.setUserName("postgres");
postgreSqlDatasourceParam.setPassword("");
// data source exits
List<DataSource> dataSourceList = new ArrayList<>();
@ -82,30 +92,30 @@ public class DataSourceServiceTest {
dataSource.setName(dataSourceName);
dataSourceList.add(dataSource);
PowerMockito.when(dataSourceMapper.queryDataSourceByName(dataSourceName.trim())).thenReturn(dataSourceList);
Result dataSourceExitsResult = dataSourceService.createDataSource(loginUser, dataSourceName, dataSourceDesc, dataSourceType, parameter);
Result dataSourceExitsResult = dataSourceService.createDataSource(loginUser, postgreSqlDatasourceParam);
Assert.assertEquals(Status.DATASOURCE_EXIST.getCode(), dataSourceExitsResult.getCode().intValue());
ConnectionParam connectionParam = DatasourceUtil.buildConnectionParams(postgreSqlDatasourceParam);
DbType dataSourceType = postgreSqlDatasourceParam.getType();
// data source exits
PowerMockito.when(dataSourceMapper.queryDataSourceByName(dataSourceName.trim())).thenReturn(null);
Result connectionResult = new Result(Status.DATASOURCE_CONNECT_FAILED.getCode(), Status.DATASOURCE_CONNECT_FAILED.getMsg());
//PowerMockito.when(dataSourceService.checkConnection(dataSourceType, parameter)).thenReturn(connectionResult);
PowerMockito.doReturn(connectionResult).when(dataSourceService).checkConnection(dataSourceType, parameter);
Result connectFailedResult = dataSourceService.createDataSource(loginUser, dataSourceName, dataSourceDesc, dataSourceType, parameter);
PowerMockito.doReturn(connectionResult).when(dataSourceService).checkConnection(dataSourceType, connectionParam);
Result connectFailedResult = dataSourceService.createDataSource(loginUser, postgreSqlDatasourceParam);
Assert.assertEquals(Status.DATASOURCE_CONNECT_FAILED.getCode(), connectFailedResult.getCode().intValue());
// data source exits
PowerMockito.when(dataSourceMapper.queryDataSourceByName(dataSourceName.trim())).thenReturn(null);
connectionResult = new Result(Status.SUCCESS.getCode(), Status.SUCCESS.getMsg());
PowerMockito.when(dataSourceService.checkConnection(dataSourceType, parameter)).thenReturn(connectionResult);
PowerMockito.when(DataSourceFactory.getDatasource(dataSourceType, parameter)).thenReturn(null);
Result notValidError = dataSourceService.createDataSource(loginUser, dataSourceName, dataSourceDesc, dataSourceType, parameter);
PowerMockito.when(dataSourceService.checkConnection(dataSourceType, connectionParam)).thenReturn(connectionResult);
Result notValidError = dataSourceService.createDataSource(loginUser, postgreSqlDatasourceParam);
Assert.assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR.getCode(), notValidError.getCode().intValue());
// success
PowerMockito.when(dataSourceMapper.queryDataSourceByName(dataSourceName.trim())).thenReturn(null);
PowerMockito.when(dataSourceService.checkConnection(dataSourceType, parameter)).thenReturn(connectionResult);
PowerMockito.when(DataSourceFactory.getDatasource(dataSourceType, parameter)).thenReturn(JSONUtils.parseObject(parameter, MySQLDataSource.class));
Result success = dataSourceService.createDataSource(loginUser, dataSourceName, dataSourceDesc, dataSourceType, parameter);
PowerMockito.when(dataSourceService.checkConnection(dataSourceType, connectionParam)).thenReturn(connectionResult);
Result success = dataSourceService.createDataSource(loginUser, postgreSqlDatasourceParam);
Assert.assertEquals(Status.SUCCESS.getCode(), success.getCode().intValue());
}
@ -115,18 +125,25 @@ public class DataSourceServiceTest {
int dataSourceId = 12;
String dataSourceName = "dataSource01";
String dataSourceDesc = "test dataSource";
DbType dataSourceType = DbType.POSTGRESQL;
String parameter = dataSourceService.buildParameter(dataSourceType, "172.16.133.200", "5432", "dolphinscheduler", null, "postgres", "", null, null, null, null, null);
PostgreSqlDatasourceParamDTO postgreSqlDatasourceParam = new PostgreSqlDatasourceParamDTO();
postgreSqlDatasourceParam.setDatabase(dataSourceName);
postgreSqlDatasourceParam.setNote(dataSourceDesc);
postgreSqlDatasourceParam.setHost("172.16.133.200");
postgreSqlDatasourceParam.setPort(5432);
postgreSqlDatasourceParam.setDatabase("dolphinscheduler");
postgreSqlDatasourceParam.setUserName("postgres");
postgreSqlDatasourceParam.setPassword("");
// data source not exits
PowerMockito.when(dataSourceMapper.selectById(dataSourceId)).thenReturn(null);
Result resourceNotExits = dataSourceService.updateDataSource(dataSourceId, loginUser, dataSourceName, dataSourceDesc, dataSourceType, parameter);
Result resourceNotExits = dataSourceService.updateDataSource(dataSourceId, loginUser, postgreSqlDatasourceParam);
Assert.assertEquals(Status.RESOURCE_NOT_EXIST.getCode(), resourceNotExits.getCode().intValue());
// user no operation perm
DataSource dataSource = new DataSource();
dataSource.setUserId(0);
PowerMockito.when(dataSourceMapper.selectById(dataSourceId)).thenReturn(dataSource);
Result userNoOperationPerm = dataSourceService.updateDataSource(dataSourceId, loginUser, dataSourceName, dataSourceDesc, dataSourceType, parameter);
Result userNoOperationPerm = dataSourceService.updateDataSource(dataSourceId, loginUser, postgreSqlDatasourceParam);
Assert.assertEquals(Status.USER_NO_OPERATION_PERM.getCode(), userNoOperationPerm.getCode().intValue());
// data source name exits
@ -135,23 +152,25 @@ public class DataSourceServiceTest {
dataSourceList.add(dataSource);
PowerMockito.when(dataSourceMapper.selectById(dataSourceId)).thenReturn(dataSource);
PowerMockito.when(dataSourceMapper.queryDataSourceByName(dataSourceName)).thenReturn(dataSourceList);
Result dataSourceNameExist = dataSourceService.updateDataSource(dataSourceId, loginUser, dataSourceName, dataSourceDesc, dataSourceType, parameter);
Result dataSourceNameExist = dataSourceService.updateDataSource(dataSourceId, loginUser, postgreSqlDatasourceParam);
Assert.assertEquals(Status.DATASOURCE_EXIST.getCode(), dataSourceNameExist.getCode().intValue());
// data source connect failed
DbType dataSourceType = postgreSqlDatasourceParam.getType();
ConnectionParam connectionParam = DatasourceUtil.buildConnectionParams(postgreSqlDatasourceParam);
PowerMockito.when(dataSourceMapper.selectById(dataSourceId)).thenReturn(dataSource);
PowerMockito.when(dataSourceMapper.queryDataSourceByName(dataSourceName)).thenReturn(null);
Result connectionResult = new Result(Status.SUCCESS.getCode(), Status.SUCCESS.getMsg());
PowerMockito.when(dataSourceService.checkConnection(dataSourceType, parameter)).thenReturn(connectionResult);
Result connectFailed = dataSourceService.updateDataSource(dataSourceId, loginUser, dataSourceName, dataSourceDesc, dataSourceType, parameter);
PowerMockito.when(dataSourceService.checkConnection(dataSourceType, connectionParam)).thenReturn(connectionResult);
Result connectFailed = dataSourceService.updateDataSource(dataSourceId, loginUser, postgreSqlDatasourceParam);
Assert.assertEquals(Status.DATASOURCE_CONNECT_FAILED.getCode(), connectFailed.getCode().intValue());
//success
PowerMockito.when(dataSourceMapper.selectById(dataSourceId)).thenReturn(dataSource);
PowerMockito.when(dataSourceMapper.queryDataSourceByName(dataSourceName)).thenReturn(null);
connectionResult = new Result(Status.DATASOURCE_CONNECT_FAILED.getCode(), Status.DATASOURCE_CONNECT_FAILED.getMsg());
PowerMockito.when(dataSourceService.checkConnection(dataSourceType, parameter)).thenReturn(connectionResult);
Result success = dataSourceService.updateDataSource(dataSourceId, loginUser, dataSourceName, dataSourceDesc, dataSourceType, parameter);
PowerMockito.when(dataSourceService.checkConnection(dataSourceType, connectionParam)).thenReturn(connectionResult);
Result success = dataSourceService.updateDataSource(dataSourceId, loginUser, postgreSqlDatasourceParam);
Assert.assertEquals(Status.SUCCESS.getCode(), success.getCode().intValue());
}
@ -279,40 +298,69 @@ public class DataSourceServiceTest {
@Test
public void buildParameter() {
String param = dataSourceService.buildParameter(DbType.ORACLE, "192.168.9.1", "1521", "im"
, "", "test", "test", DbConnectType.ORACLE_SERVICE_NAME, "", "", "", "");
String expected = "{\"connectType\":\"ORACLE_SERVICE_NAME\",\"address\":\"jdbc:oracle:thin:@//192.168.9.1:1521\",\"database\":\"im\","
+ "\"jdbcUrl\":\"jdbc:oracle:thin:@//192.168.9.1:1521/im\",\"user\":\"test\",\"password\":\"test\"}";
Assert.assertEquals(expected, param);
OracleDatasourceParamDTO oracleDatasourceParamDTO = new OracleDatasourceParamDTO();
oracleDatasourceParamDTO.setHost("192.168.9.1");
oracleDatasourceParamDTO.setPort(1521);
oracleDatasourceParamDTO.setDatabase("im");
oracleDatasourceParamDTO.setUserName("test");
oracleDatasourceParamDTO.setPassword("test");
oracleDatasourceParamDTO.setConnectType(DbConnectType.ORACLE_SERVICE_NAME);
ConnectionParam connectionParam = DatasourceUtil.buildConnectionParams(oracleDatasourceParamDTO);
String expected = "{\"user\":\"test\",\"password\":\"test\",\"address\":\"jdbc:oracle:thin:@//192.168.9.1:1521\","
+ "\"database\":\"im\",\"jdbcUrl\":\"jdbc:oracle:thin:@//192.168.9.1:1521/im\",\"connectType\":\"ORACLE_SERVICE_NAME\"}";
Assert.assertEquals(expected, JSONUtils.toJsonString(connectionParam));
PowerMockito.mockStatic(CommonUtils.class);
PowerMockito.when(CommonUtils.getKerberosStartupState()).thenReturn(true);
PowerMockito.when(CommonUtils.encodePassword(Mockito.anyString())).thenReturn("test");
param = dataSourceService.buildParameter(DbType.HIVE, "192.168.9.1", "10000", "im"
, "hive/hdfs-mycluster@ESZ.COM", "test", "test", null, "", "/opt/krb5.conf", "test2/hdfs-mycluster@ESZ.COM", "/opt/hdfs.headless.keytab");
expected = "{\"address\":\"jdbc:hive2://192.168.9.1:10000\",\"database\":\"im\",\"jdbcUrl\":\"jdbc:hive2://192.168.9.1:10000/im;principal=hive/hdfs-mycluster@ESZ.COM\","
+ "\"user\":\"test\",\"password\":\"test\",\"principal\":\"hive/hdfs-mycluster@ESZ.COM\",\"javaSecurityKrb5Conf\":\"/opt/krb5.conf\","
+ "\"loginUserKeytabUsername\":\"test2/hdfs-mycluster@ESZ.COM\",\"loginUserKeytabPath\":\"/opt/hdfs.headless.keytab\"}";
Assert.assertEquals(expected, param);
HiveDataSourceParamDTO hiveDataSourceParamDTO = new HiveDataSourceParamDTO();
hiveDataSourceParamDTO.setHost("192.168.9.1");
hiveDataSourceParamDTO.setPort(10000);
hiveDataSourceParamDTO.setDatabase("im");
hiveDataSourceParamDTO.setPrincipal("hive/hdfs-mycluster@ESZ.COM");
hiveDataSourceParamDTO.setUserName("test");
hiveDataSourceParamDTO.setPassword("test");
hiveDataSourceParamDTO.setJavaSecurityKrb5Conf("/opt/krb5.conf");
hiveDataSourceParamDTO.setLoginUserKeytabPath("/opt/hdfs.headless.keytab");
hiveDataSourceParamDTO.setLoginUserKeytabUsername("test2/hdfs-mycluster@ESZ.COM");
connectionParam = DatasourceUtil.buildConnectionParams(hiveDataSourceParamDTO);
expected = "{\"user\":\"test\",\"password\":\"test\",\"address\":\"jdbc:hive2://192.168.9.1:10000\","
+ "\"database\":\"im\",\"jdbcUrl\":\"jdbc:hive2://192.168.9.1:10000/im;principal=hive/hdfs-mycluster@ESZ.COM\",\"principal\":\"hive/hdfs-mycluster@ESZ.COM\","
+ "\"javaSecurityKrb5Conf\":\"/opt/krb5.conf\",\"loginUserKeytabUsername\":\"test2/hdfs-mycluster@ESZ.COM\",\"loginUserKeytabPath\":\"/opt/hdfs.headless.keytab\"}";
Assert.assertEquals(expected, JSONUtils.toJsonString(connectionParam));
}
@Test
public void buildParameterWithDecodePassword() {
PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE, "true");
String other = "{\"autoDeserialize\":\"yes\",\"allowUrlInLocalInfile\":\"true\"}";
String param = dataSourceService.buildParameter(DbType.MYSQL, "192.168.9.1", "1521", "im"
, "", "test", "123456", null, other, "", "", "");
String expected = "{\"address\":\"jdbc:mysql://192.168.9.1:1521\",\"database\":\"im\",\"jdbcUrl\":\"jdbc:mysql://192.168.9.1:1521/im\","
+ "\"user\":\"test\",\"password\":\"IUAjJCVeJipNVEl6TkRVMg==\"}";
Assert.assertEquals(expected, param);
Map<String, String> other = new HashMap<>();
other.put("autoDeserialize", "yes");
other.put("allowUrlInLocalInfile", "true");
MysqlDatasourceParamDTO mysqlDatasourceParamDTO = new MysqlDatasourceParamDTO();
mysqlDatasourceParamDTO.setHost("192.168.9.1");
mysqlDatasourceParamDTO.setPort(1521);
mysqlDatasourceParamDTO.setDatabase("im");
mysqlDatasourceParamDTO.setUserName("test");
mysqlDatasourceParamDTO.setPassword("123456");
mysqlDatasourceParamDTO.setOther(other);
ConnectionParam connectionParam = DatasourceUtil.buildConnectionParams(mysqlDatasourceParamDTO);
String expected = "{\"user\":\"test\",\"password\":\"IUAjJCVeJipNVEl6TkRVMg==\",\"address\":\"jdbc:mysql://192.168.9.1:1521\","
+ "\"database\":\"im\",\"jdbcUrl\":\"jdbc:mysql://192.168.9.1:1521/im\"}";
Assert.assertEquals(expected, JSONUtils.toJsonString(connectionParam));
PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE, "false");
param = dataSourceService.buildParameter(DbType.MYSQL, "192.168.9.1", "1521", "im"
, "", "test", "123456", null, "", "", "", "");
expected = "{\"address\":\"jdbc:mysql://192.168.9.1:1521\",\"database\":\"im\",\"jdbcUrl\":\"jdbc:mysql://192.168.9.1:1521/im\","
+ "\"user\":\"test\",\"password\":\"123456\"}";
Assert.assertEquals(expected, param);
mysqlDatasourceParamDTO = new MysqlDatasourceParamDTO();
mysqlDatasourceParamDTO.setHost("192.168.9.1");
mysqlDatasourceParamDTO.setPort(1521);
mysqlDatasourceParamDTO.setDatabase("im");
mysqlDatasourceParamDTO.setUserName("test");
mysqlDatasourceParamDTO.setPassword("123456");
connectionParam = DatasourceUtil.buildConnectionParams(mysqlDatasourceParamDTO);
expected = "{\"user\":\"test\",\"password\":\"123456\",\"address\":\"jdbc:mysql://192.168.9.1:1521\","
+ "\"database\":\"im\",\"jdbcUrl\":\"jdbc:mysql://192.168.9.1:1521/im\"}";
Assert.assertEquals(expected, JSONUtils.toJsonString(connectionParam));
}
/**
@ -334,22 +382,27 @@ public class DataSourceServiceTest {
@Test
public void testCheckConnection() throws Exception {
DbType dataSourceType = DbType.POSTGRESQL;
String parameter = dataSourceService.buildParameter(dataSourceType, "172.16.133.200", "5432", "dolphinscheduler", null, "postgres", "", null, null, null, null, null);
String dataSourceName = "dataSource01";
String dataSourceDesc = "test dataSource";
PowerMockito.mockStatic(DataSourceFactory.class);
PowerMockito.when(DataSourceFactory.getDatasource(Mockito.any(), Mockito.anyString())).thenReturn(null);
Result result = dataSourceService.checkConnection(dataSourceType, parameter);
Assert.assertEquals(Status.DATASOURCE_TYPE_NOT_EXIST.getCode(), result.getCode().intValue());
PostgreSqlDatasourceParamDTO postgreSqlDatasourceParam = new PostgreSqlDatasourceParamDTO();
postgreSqlDatasourceParam.setDatabase(dataSourceName);
postgreSqlDatasourceParam.setNote(dataSourceDesc);
postgreSqlDatasourceParam.setHost("172.16.133.200");
postgreSqlDatasourceParam.setPort(5432);
postgreSqlDatasourceParam.setDatabase("dolphinscheduler");
postgreSqlDatasourceParam.setUserName("postgres");
postgreSqlDatasourceParam.setPassword("");
ConnectionParam connectionParam = DatasourceUtil.buildConnectionParams(postgreSqlDatasourceParam);
BaseDataSource dataSource = PowerMockito.mock(BaseDataSource.class);
PowerMockito.when(DataSourceFactory.getDatasource(Mockito.any(), Mockito.anyString())).thenReturn(dataSource);
PowerMockito.when(dataSource.getConnection()).thenReturn(null);
result = dataSourceService.checkConnection(dataSourceType, parameter);
PowerMockito.mockStatic(DatasourceUtil.class);
Result result = dataSourceService.checkConnection(dataSourceType, connectionParam);
Assert.assertEquals(Status.CONNECTION_TEST_FAILURE.getCode(), result.getCode().intValue());
Connection connection = PowerMockito.mock(Connection.class);
PowerMockito.when(dataSource.getConnection()).thenReturn(connection);
result = dataSourceService.checkConnection(dataSourceType, parameter);
PowerMockito.when(DatasourceUtil.getConnection(Mockito.any(), Mockito.any())).thenReturn(connection);
result = dataSourceService.checkConnection(dataSourceType, connectionParam);
Assert.assertEquals(Status.SUCCESS.getCode(), result.getCode().intValue());
}

10
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java
View File

@ -380,21 +380,11 @@ public final class Constants {
*/
public static final int DEFAULT_MASTER_CPU_LOAD = Runtime.getRuntime().availableProcessors() * 2;
/**
* master reserved memory
*/
public static final double DEFAULT_MASTER_RESERVED_MEMORY = OSUtils.totalMemorySize() / 10;
/**
* worker cpu load
*/
public static final int DEFAULT_WORKER_CPU_LOAD = Runtime.getRuntime().availableProcessors() * 2;
/**
* worker reserved memory
*/
public static final double DEFAULT_WORKER_RESERVED_MEMORY = OSUtils.totalMemorySize() / 10;
/**
* worker host weight
*/

79
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/AbstractDatasourceProcessor.java Normal file
View File

@ -0,0 +1,79 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.datasource;
import org.apache.commons.collections4.MapUtils;
import java.util.Map;
import java.util.regex.Pattern;
public abstract class AbstractDatasourceProcessor implements DatasourceProcessor {
private static final Pattern IPV4_PATTERN = Pattern.compile("^[a-zA-Z0-9\\_\\-\\.]+$");
private static final Pattern IPV6_PATTERN = Pattern.compile("^[a-zA-Z0-9\\_\\-\\.\\:\\[\\]]+$");
private static final Pattern DATABASE_PATTER = Pattern.compile("^[a-zA-Z0-9\\_\\-\\.]+$");
private static final Pattern PARAMS_PATTER = Pattern.compile("^[a-zA-Z0-9]+$");
@Override
public void checkDatasourceParam(BaseDataSourceParamDTO baseDataSourceParamDTO) {
checkHost(baseDataSourceParamDTO.getHost());
checkDatasourcePatter(baseDataSourceParamDTO.getDatabase());
checkOther(baseDataSourceParamDTO.getOther());
}
/**
* Check the host is valid
*
* @param host datasource host
*/
protected void checkHost(String host) {
if (!IPV4_PATTERN.matcher(host).matches() || !IPV6_PATTERN.matcher(host).matches()) {
throw new IllegalArgumentException("datasource host illegal");
}
}
/**
* check database name is valid
*
* @param database database name
*/
protected void checkDatasourcePatter(String database) {
if (!DATABASE_PATTER.matcher(database).matches()) {
throw new IllegalArgumentException("datasource name illegal");
}
}
/**
* check other is valid
*
* @param other other
*/
protected void checkOther(Map<String, String> other) {
if (MapUtils.isEmpty(other)) {
return;
}
boolean paramsCheck = other.entrySet().stream().allMatch(p -> PARAMS_PATTER.matcher(p.getValue()).matches());
if (!paramsCheck) {
throw new IllegalArgumentException("datasource other params illegal");
}
}
}

98
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/BaseConnectionParam.java Normal file
View File

@ -0,0 +1,98 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.datasource;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonInclude.Include;
/**
* The base model of connection param
* <p>
* {@link org.apache.dolphinscheduler.common.datasource.clickhouse.ClickhouseConnectionParam}
* {@link org.apache.dolphinscheduler.common.datasource.db2.Db2ConnectionParam}
* {@link org.apache.dolphinscheduler.common.datasource.hive.HiveConnectionParam}
* {@link org.apache.dolphinscheduler.common.datasource.mysql.MysqlConnectionParam}
* {@link org.apache.dolphinscheduler.common.datasource.oracle.OracleConnectionParam}
* {@link org.apache.dolphinscheduler.common.datasource.postgresql.PostgreSqlConnectionParam}
* {@link org.apache.dolphinscheduler.common.datasource.presto.PrestoConnectionParam}
* {@link org.apache.dolphinscheduler.common.datasource.spark.SparkConnectionParam}
* {@link org.apache.dolphinscheduler.common.datasource.sqlserver.SqlServerConnectionParam}
*/
@JsonInclude(Include.NON_NULL)
public abstract class BaseConnectionParam implements ConnectionParam {
protected String user;
protected String password;
protected String address;
protected String database;
protected String jdbcUrl;
protected String other;
public String getUser() {
return user;
}
public void setUser(String user) {
this.user = user;
}
public String getPassword() {
return password;
}
public void setPassword(String password) {
this.password = password;
}
public String getAddress() {
return address;
}
public void setAddress(String address) {
this.address = address;
}
public String getDatabase() {
return database;
}
public void setDatabase(String database) {
this.database = database;
}
public String getJdbcUrl() {
return jdbcUrl;
}
public void setJdbcUrl(String jdbcUrl) {
this.jdbcUrl = jdbcUrl;
}
public String getOther() {
return other;
}
public void setOther(String other) {
this.other = other;
}
}

161
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/BaseDataSourceParamDTO.java Normal file
View File

@ -0,0 +1,161 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.datasource;
import org.apache.dolphinscheduler.common.datasource.clickhouse.ClickHouseDatasourceParamDTO;
import org.apache.dolphinscheduler.common.datasource.db2.Db2DatasourceParamDTO;
import org.apache.dolphinscheduler.common.datasource.hive.HiveDataSourceParamDTO;
import org.apache.dolphinscheduler.common.datasource.mysql.MysqlDatasourceParamDTO;
import org.apache.dolphinscheduler.common.datasource.oracle.OracleDatasourceParamDTO;
import org.apache.dolphinscheduler.common.datasource.postgresql.PostgreSqlDatasourceParamDTO;
import org.apache.dolphinscheduler.common.datasource.presto.PrestoDatasourceParamDTO;
import org.apache.dolphinscheduler.common.datasource.spark.SparkDatasourceParamDTO;
import org.apache.dolphinscheduler.common.datasource.sqlserver.SqlServerDatasourceParamDTO;
import org.apache.dolphinscheduler.common.enums.DbType;
import java.io.Serializable;
import java.util.Map;
import com.fasterxml.jackson.annotation.JsonSubTypes;
import com.fasterxml.jackson.annotation.JsonTypeInfo;
/**
* Basic datasource params submitted to api.
* <p>
* see {@link MysqlDatasourceParamDTO}
* see {@link PostgreSqlDatasourceParamDTO}
* see {@link HiveDataSourceParamDTO}
* see {@link SparkDatasourceParamDTO}
* see {@link ClickHouseDatasourceParamDTO}
* see {@link OracleDatasourceParamDTO}
* see {@link SqlServerDatasourceParamDTO}
* see {@link Db2DatasourceParamDTO}
* see {@link PrestoDatasourceParamDTO}
*/
@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type")
@JsonSubTypes(value = {
@JsonSubTypes.Type(value = MysqlDatasourceParamDTO.class, name = "MYSQL"),
@JsonSubTypes.Type(value = PostgreSqlDatasourceParamDTO.class, name = "POSTGRESQL"),
@JsonSubTypes.Type(value = HiveDataSourceParamDTO.class, name = "HIVE"),
@JsonSubTypes.Type(value = SparkDatasourceParamDTO.class, name = "SPARK"),
@JsonSubTypes.Type(value = ClickHouseDatasourceParamDTO.class, name = "CLICKHOUSE"),
@JsonSubTypes.Type(value = OracleDatasourceParamDTO.class, name = "ORACLE"),
@JsonSubTypes.Type(value = SqlServerDatasourceParamDTO.class, name = "SQLSERVER"),
@JsonSubTypes.Type(value = Db2DatasourceParamDTO.class, name = "DB2"),
@JsonSubTypes.Type(value = PrestoDatasourceParamDTO.class, name = "PRESTO"),
})
public abstract class BaseDataSourceParamDTO implements Serializable {
protected Integer id;
protected String name;
protected String note;
protected String host;
protected Integer port;
protected String database;
protected String userName;
protected String password;
protected Map<String, String> other;
public Integer getId() {
return id;
}
public void setId(Integer id) {
this.id = id;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public String getNote() {
return note;
}
public void setNote(String note) {
this.note = note;
}
public String getHost() {
return host;
}
public void setHost(String host) {
this.host = host;
}
public Integer getPort() {
return port;
}
public void setPort(Integer port) {
this.port = port;
}
public String getDatabase() {
return database;
}
public void setDatabase(String database) {
this.database = database;
}
public String getUserName() {
return userName;
}
public void setUserName(String userName) {
this.userName = userName;
}
public String getPassword() {
return password;
}
public void setPassword(String password) {
this.password = password;
}
public Map<String, String> getOther() {
return other;
}
public void setOther(Map<String, String> other) {
this.other = other;
}
/**
* Get the datasource type
* see{@link DbType}
*
* @return datasource type code
*/
public abstract DbType getType();
}

57
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/BaseHdfsConnectionParam.java Normal file
View File

@ -0,0 +1,57 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.datasource;
public class BaseHdfsConnectionParam extends BaseConnectionParam {
protected String principal;
protected String javaSecurityKrb5Conf;
protected String loginUserKeytabUsername;
protected String loginUserKeytabPath;
public String getPrincipal() {
return principal;
}
public void setPrincipal(String principal) {
this.principal = principal;
}
public String getJavaSecurityKrb5Conf() {
return javaSecurityKrb5Conf;
}
public void setJavaSecurityKrb5Conf(String javaSecurityKrb5Conf) {
this.javaSecurityKrb5Conf = javaSecurityKrb5Conf;
}
public String getLoginUserKeytabUsername() {
return loginUserKeytabUsername;
}
public void setLoginUserKeytabUsername(String loginUserKeytabUsername) {
this.loginUserKeytabUsername = loginUserKeytabUsername;
}
public String getLoginUserKeytabPath() {
return loginUserKeytabPath;
}
public void setLoginUserKeytabPath(String loginUserKeytabPath) {
this.loginUserKeytabPath = loginUserKeytabPath;
}
}

61
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/BaseHdfsDatasourceParamDTO.java Normal file
View File

@ -0,0 +1,61 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.datasource;
public abstract class BaseHdfsDatasourceParamDTO extends BaseDataSourceParamDTO {
protected String principal;
protected String javaSecurityKrb5Conf;
protected String loginUserKeytabUsername;
protected String loginUserKeytabPath;
public String getPrincipal() {
return principal;
}
public void setPrincipal(String principal) {
this.principal = principal;
}
public String getLoginUserKeytabUsername() {
return loginUserKeytabUsername;
}
public void setLoginUserKeytabUsername(String loginUserKeytabUsername) {
this.loginUserKeytabUsername = loginUserKeytabUsername;
}
public String getLoginUserKeytabPath() {
return loginUserKeytabPath;
}
public void setLoginUserKeytabPath(String loginUserKeytabPath) {
this.loginUserKeytabPath = loginUserKeytabPath;
}
public String getJavaSecurityKrb5Conf() {
return javaSecurityKrb5Conf;
}
public void setJavaSecurityKrb5Conf(String javaSecurityKrb5Conf) {
this.javaSecurityKrb5Conf = javaSecurityKrb5Conf;
}
}

25
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/PrestoDataSource.java → dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/ConnectionParam.java
View File

@ -14,26 +14,13 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.dao.datasource;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.DbType;
package org.apache.dolphinscheduler.common.datasource;
public class PrestoDataSource extends BaseDataSource {
import java.io.Serializable;
/**
* @return driver class
*/
@Override
public String driverClassSelector() {
return Constants.COM_PRESTO_JDBC_DRIVER;
}
/**
* @return db type
*/
@Override
public DbType dbTypeSelector() {
return DbType.PRESTO;
}
/**
* The model of Datasource Connection param
*/
public interface ConnectionParam extends Serializable {
}

81
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/DatasourceProcessor.java Normal file
View File

@ -0,0 +1,81 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.datasource;
import org.apache.dolphinscheduler.common.enums.DbType;
import java.io.IOException;
import java.sql.Connection;
import java.sql.SQLException;
public interface DatasourceProcessor {
/**
* check datasource param is valid
*/
void checkDatasourceParam(BaseDataSourceParamDTO datasourceParam);
/**
* create BaseDataSourceParamDTO by connectionJson
*
* @param connectionJson see{@link org.apache.dolphinscheduler.dao.entity.Datasource}
* @return {@link BaseDataSourceParamDTO}
*/
BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson);
/**
* create datasource connection parameter which will be stored at DataSource
* <p>
* see {@code org.apache.dolphinscheduler.dao.entity.DataSource.connectionParams}
*/
ConnectionParam createConnectionParams(BaseDataSourceParamDTO datasourceParam);
/**
* deserialize json to datasource connection param
*
* @param connectionJson {@code org.apache.dolphinscheduler.dao.entity.DataSource.connectionParams}
* @return {@link BaseConnectionParam}
*/
ConnectionParam createConnectionParams(String connectionJson);
/**
* get datasource Driver
*/
String getDatasourceDriver();
/**
* get jdbcUrl by connection param, the jdbcUrl is different with ConnectionParam.jdbcUrl, this method will inject
* other to jdbcUrl
*
* @param connectionParam connection param
*/
String getJdbcUrl(ConnectionParam connectionParam);
/**
* get connection by connectionParam
*
* @param connectionParam connectionParam
* @return {@link Connection}
*/
Connection getConnection(ConnectionParam connectionParam) throws ClassNotFoundException, SQLException, IOException;
/**
* @return {@link DbType}
*/
DbType getDbType();
}

121
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/DatasourceUtil.java Normal file
View File

@ -0,0 +1,121 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.datasource;
import org.apache.dolphinscheduler.common.datasource.clickhouse.ClickHouseDatasourceProcessor;
import org.apache.dolphinscheduler.common.datasource.db2.Db2DatasourceProcessor;
import org.apache.dolphinscheduler.common.datasource.hive.HiveDatasourceProcessor;
import org.apache.dolphinscheduler.common.datasource.mysql.MysqlDatasourceProcessor;
import org.apache.dolphinscheduler.common.datasource.oracle.OracleDatasourceProcessor;
import org.apache.dolphinscheduler.common.datasource.postgresql.PostgreSqlDatasourceProcessor;
import org.apache.dolphinscheduler.common.datasource.presto.PrestoDatasourceProcessor;
import org.apache.dolphinscheduler.common.datasource.spark.SparkDatasourceProcessor;
import org.apache.dolphinscheduler.common.datasource.sqlserver.SqlServerDatasourceProcessor;
import org.apache.dolphinscheduler.common.enums.DbType;
import java.sql.Connection;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class DatasourceUtil {
private DatasourceUtil() {
}
private static final Logger logger = LoggerFactory.getLogger(DatasourceUtil.class);
private static final DatasourceProcessor mysqlProcessor = new MysqlDatasourceProcessor();
private static final DatasourceProcessor postgreSqlProcessor = new PostgreSqlDatasourceProcessor();
private static final DatasourceProcessor hiveProcessor = new HiveDatasourceProcessor();
private static final DatasourceProcessor sparkProcessor = new SparkDatasourceProcessor();
private static final DatasourceProcessor clickhouseProcessor = new ClickHouseDatasourceProcessor();
private static final DatasourceProcessor oracleProcessor = new OracleDatasourceProcessor();
private static final DatasourceProcessor sqlServerProcessor = new SqlServerDatasourceProcessor();
private static final DatasourceProcessor db2PROCESSOR = new Db2DatasourceProcessor();
private static final DatasourceProcessor prestoPROCESSOR = new PrestoDatasourceProcessor();
/**
* check datasource param
*
* @param baseDataSourceParamDTO datasource param
*/
public static void checkDatasourceParam(BaseDataSourceParamDTO baseDataSourceParamDTO) {
getDatasourceProcessor(baseDataSourceParamDTO.getType()).checkDatasourceParam(baseDataSourceParamDTO);
}
/**
* build connection url
*
* @param baseDataSourceParamDTO datasourceParam
*/
public static ConnectionParam buildConnectionParams(BaseDataSourceParamDTO baseDataSourceParamDTO) {
ConnectionParam connectionParams = getDatasourceProcessor(baseDataSourceParamDTO.getType())
.createConnectionParams(baseDataSourceParamDTO);
if (logger.isDebugEnabled()) {
logger.info("parameters map:{}", connectionParams);
}
return connectionParams;
}
public static ConnectionParam buildConnectionParams(DbType dbType, String connectionJson) {
return getDatasourceProcessor(dbType).createConnectionParams(connectionJson);
}
public static Connection getConnection(DbType dbType, ConnectionParam connectionParam) {
try {
return getDatasourceProcessor(dbType).getConnection(connectionParam);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
public static String getJdbcUrl(DbType dbType, ConnectionParam baseConnectionParam) {
return getDatasourceProcessor(dbType).getJdbcUrl(baseConnectionParam);
}
public static BaseDataSourceParamDTO buildDatasourceParamDTO(DbType dbType, String connectionParams) {
return getDatasourceProcessor(dbType).createDatasourceParamDTO(connectionParams);
}
public static DatasourceProcessor getDatasourceProcessor(DbType dbType) {
switch (dbType) {
case MYSQL:
return mysqlProcessor;
case POSTGRESQL:
return postgreSqlProcessor;
case HIVE:
return hiveProcessor;
case SPARK:
return sparkProcessor;
case CLICKHOUSE:
return clickhouseProcessor;
case ORACLE:
return oracleProcessor;
case SQLSERVER:
return sqlServerProcessor;
case DB2:
return db2PROCESSOR;
case PRESTO:
return prestoPROCESSOR;
default:
throw new IllegalArgumentException("datasource type illegal:" + dbType);
}
}
}

29
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/ClickHouseDataSource.java → dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/clickhouse/ClickHouseDatasourceParamDTO.java
View File

@ -14,29 +14,28 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.dao.datasource;
import org.apache.dolphinscheduler.common.Constants;
package org.apache.dolphinscheduler.common.datasource.clickhouse;
import org.apache.dolphinscheduler.common.datasource.BaseDataSourceParamDTO;
import org.apache.dolphinscheduler.common.enums.DbType;
/**
* data source of ClickHouse
*/
public class ClickHouseDataSource extends BaseDataSource {
public class ClickHouseDatasourceParamDTO extends BaseDataSourceParamDTO {
/**
* @return driver class
*/
@Override
public String driverClassSelector() {
return Constants.COM_CLICKHOUSE_JDBC_DRIVER;
public String toString() {
return "ClickHouseDatasourceParamDTO{"
+ "host='" + host + '\''
+ ", port=" + port
+ ", database='" + database + '\''
+ ", userName='" + userName + '\''
+ ", password='" + password + '\''
+ ", other='" + other + '\''
+ '}';
}
/**
* @return db type
*/
@Override
public DbType dbTypeSelector() {
public DbType getType() {
return DbType.CLICKHOUSE;
}
}

125
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/clickhouse/ClickHouseDatasourceProcessor.java Normal file
View File

@ -0,0 +1,125 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.datasource.clickhouse;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.datasource.AbstractDatasourceProcessor;
import org.apache.dolphinscheduler.common.datasource.BaseDataSourceParamDTO;
import org.apache.dolphinscheduler.common.datasource.ConnectionParam;
import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.common.utils.CommonUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.commons.collections4.MapUtils;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.util.LinkedHashMap;
import java.util.Map;
public class ClickHouseDatasourceProcessor extends AbstractDatasourceProcessor {
@Override
public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) {
ClickhouseConnectionParam connectionParams = (ClickhouseConnectionParam) createConnectionParams(connectionJson);
ClickHouseDatasourceParamDTO clickHouseDatasourceParamDTO = new ClickHouseDatasourceParamDTO();
clickHouseDatasourceParamDTO.setDatabase(connectionParams.getDatabase());
clickHouseDatasourceParamDTO.setUserName(connectionParams.getUser());
clickHouseDatasourceParamDTO.setOther(parseOther(connectionParams.getOther()));
String[] hostSeperator = connectionParams.getAddress().split(Constants.DOUBLE_SLASH);
String[] hostPortArray = hostSeperator[hostSeperator.length - 1].split(Constants.COMMA);
clickHouseDatasourceParamDTO.setPort(Integer.parseInt(hostPortArray[0].split(Constants.COLON)[1]));
clickHouseDatasourceParamDTO.setHost(hostPortArray[0].split(Constants.COLON)[0]);
return clickHouseDatasourceParamDTO;
}
@Override
public ConnectionParam createConnectionParams(BaseDataSourceParamDTO datasourceParam) {
ClickHouseDatasourceParamDTO clickHouseParam = (ClickHouseDatasourceParamDTO) datasourceParam;
String address = String.format("%s%s:%s", Constants.JDBC_CLICKHOUSE, clickHouseParam.getHost(), clickHouseParam.getPort());
String jdbcUrl = address + "/" + clickHouseParam.getDatabase();
ClickhouseConnectionParam clickhouseConnectionParam = new ClickhouseConnectionParam();
clickhouseConnectionParam.setDatabase(clickHouseParam.getDatabase());
clickhouseConnectionParam.setAddress(address);
clickhouseConnectionParam.setJdbcUrl(jdbcUrl);
clickhouseConnectionParam.setUser(clickHouseParam.getUserName());
clickhouseConnectionParam.setPassword(CommonUtils.encodePassword(clickHouseParam.getPassword()));
clickhouseConnectionParam.setOther(transformOther(clickHouseParam.getOther()));
return clickhouseConnectionParam;
}
@Override
public ConnectionParam createConnectionParams(String connectionJson) {
return JSONUtils.parseObject(connectionJson, ClickhouseConnectionParam.class);
}
@Override
public String getDatasourceDriver() {
return Constants.COM_CLICKHOUSE_JDBC_DRIVER;
}
@Override
public String getJdbcUrl(ConnectionParam connectionParam) {
ClickhouseConnectionParam clickhouseConnectionParam = (ClickhouseConnectionParam) connectionParam;
String jdbcUrl = clickhouseConnectionParam.getJdbcUrl();
if (StringUtils.isNotEmpty(clickhouseConnectionParam.getOther())) {
jdbcUrl = String.format("%s?%s", jdbcUrl, clickhouseConnectionParam.getOther());
}
return jdbcUrl;
}
@Override
public Connection getConnection(ConnectionParam connectionParam) throws ClassNotFoundException, SQLException {
ClickhouseConnectionParam clickhouseConnectionParam = (ClickhouseConnectionParam) connectionParam;
Class.forName(getDatasourceDriver());
return DriverManager.getConnection(getJdbcUrl(clickhouseConnectionParam),
clickhouseConnectionParam.getUser(), CommonUtils.decodePassword(clickhouseConnectionParam.getPassword()));
}
@Override
public DbType getDbType() {
return DbType.CLICKHOUSE;
}
private String transformOther(Map<String, String> otherMap) {
if (MapUtils.isEmpty(otherMap)) {
return null;
}
StringBuilder stringBuilder = new StringBuilder();
otherMap.forEach((key, value) -> stringBuilder.append(String.format("%s=%s%s", key, value, "&")));
return stringBuilder.toString();
}
private Map<String, String> parseOther(String other) {
if (other == null) {
return null;
}
Map<String, String> otherMap = new LinkedHashMap<>();
String[] configs = other.split("&");
for (String config : configs) {
otherMap.put(config.split("=")[0], config.split("=")[1]);
}
return otherMap;
}
}

34
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/clickhouse/ClickhouseConnectionParam.java Normal file
View File

@ -0,0 +1,34 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.datasource.clickhouse;
import org.apache.dolphinscheduler.common.datasource.BaseConnectionParam;
public class ClickhouseConnectionParam extends BaseConnectionParam {
@Override
public String toString() {
return "ClickhouseConnectionParam{"
+ "user='" + user + '\''
+ ", password='" + password + '\''
+ ", address='" + address + '\''
+ ", database='" + database + '\''
+ ", jdbcUrl='" + jdbcUrl + '\''
+ ", other='" + other + '\''
+ '}';
}
}

33
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/DB2ServerDataSource.java → dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/db2/Db2ConnectionParam.java
View File

@ -14,30 +14,21 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.dao.datasource;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.DbType;
package org.apache.dolphinscheduler.common.datasource.db2;
/**
* data source of DB2 Server
*/
public class DB2ServerDataSource extends BaseDataSource {
import org.apache.dolphinscheduler.common.datasource.BaseConnectionParam;
/**
* gets the JDBC url for the data source connection
* @return jdbc url
*/
public class Db2ConnectionParam extends BaseConnectionParam {
@Override
public String driverClassSelector() {
return Constants.COM_DB2_JDBC_DRIVER;
}
/**
* @return db type
*/
@Override
public DbType dbTypeSelector() {
return DbType.DB2;
public String toString() {
return "Db2ConnectionParam{"
+ "user='" + user + '\''
+ ", password='" + password + '\''
+ ", address='" + address + '\''
+ ", database='" + database + '\''
+ ", jdbcUrl='" + jdbcUrl + '\''
+ ", other='" + other + '\''
+ '}';
}
}

43
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/db2/Db2DatasourceParamDTO.java Normal file
View File

@ -0,0 +1,43 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.datasource.db2;
import org.apache.dolphinscheduler.common.datasource.BaseDataSourceParamDTO;
import org.apache.dolphinscheduler.common.enums.DbType;
public class Db2DatasourceParamDTO extends BaseDataSourceParamDTO {
@Override
public String toString() {
return "Db2DatasourceParamDTO{"
+ "name='" + name + '\''
+ ", note='" + note + '\''
+ ", host='" + host + '\''
+ ", port=" + port
+ ", database='" + database + '\''
+ ", userName='" + userName + '\''
+ ", password='" + password + '\''
+ ", other='" + other + '\''
+ '}';
}
@Override
public DbType getType() {
return DbType.DB2;
}
}

126
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/db2/Db2DatasourceProcessor.java Normal file
View File

@ -0,0 +1,126 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.datasource.db2;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.datasource.AbstractDatasourceProcessor;
import org.apache.dolphinscheduler.common.datasource.BaseConnectionParam;
import org.apache.dolphinscheduler.common.datasource.BaseDataSourceParamDTO;
import org.apache.dolphinscheduler.common.datasource.ConnectionParam;
import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.common.utils.CommonUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.commons.collections4.MapUtils;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.util.LinkedHashMap;
import java.util.Map;
public class Db2DatasourceProcessor extends AbstractDatasourceProcessor {
@Override
public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) {
Db2ConnectionParam connectionParams = (Db2ConnectionParam) createConnectionParams(connectionJson);
Db2DatasourceParamDTO db2DatasourceParamDTO = new Db2DatasourceParamDTO();
db2DatasourceParamDTO.setDatabase(connectionParams.getDatabase());
db2DatasourceParamDTO.setOther(parseOther(connectionParams.getOther()));
db2DatasourceParamDTO.setUserName(db2DatasourceParamDTO.getUserName());
String[] hostSeperator = connectionParams.getAddress().split(Constants.DOUBLE_SLASH);
String[] hostPortArray = hostSeperator[hostSeperator.length - 1].split(Constants.COMMA);
db2DatasourceParamDTO.setHost(hostPortArray[0].split(Constants.COLON)[0]);
db2DatasourceParamDTO.setPort(Integer.parseInt(hostPortArray[0].split(Constants.COLON)[1]));
return db2DatasourceParamDTO;
}
@Override
public BaseConnectionParam createConnectionParams(BaseDataSourceParamDTO datasourceParam) {
Db2DatasourceParamDTO db2Param = (Db2DatasourceParamDTO) datasourceParam;
String address = String.format("%s%s:%s", Constants.JDBC_DB2, db2Param.getHost(), db2Param.getPort());
String jdbcUrl = String.format("%s/%s", address, db2Param.getDatabase());
Db2ConnectionParam db2ConnectionParam = new Db2ConnectionParam();
db2ConnectionParam.setAddress(address);
db2ConnectionParam.setDatabase(db2Param.getDatabase());
db2ConnectionParam.setJdbcUrl(jdbcUrl);
db2ConnectionParam.setUser(db2Param.getUserName());
db2ConnectionParam.setPassword(CommonUtils.encodePassword(db2Param.getPassword()));
db2ConnectionParam.setOther(transformOther(db2Param.getOther()));
return db2ConnectionParam;
}
@Override
public ConnectionParam createConnectionParams(String connectionJson) {
return JSONUtils.parseObject(connectionJson, Db2ConnectionParam.class);
}
@Override
public String getDatasourceDriver() {
return Constants.COM_DB2_JDBC_DRIVER;
}
@Override
public String getJdbcUrl(ConnectionParam connectionParam) {
Db2ConnectionParam db2ConnectionParam = (Db2ConnectionParam) connectionParam;
if (StringUtils.isNotEmpty(db2ConnectionParam.getOther())) {
return String.format("%s;%s", db2ConnectionParam.getJdbcUrl(), db2ConnectionParam.getOther());
}
return db2ConnectionParam.getJdbcUrl();
}
@Override
public Connection getConnection(ConnectionParam connectionParam) throws ClassNotFoundException, SQLException {
Db2ConnectionParam db2ConnectionParam = (Db2ConnectionParam) connectionParam;
Class.forName(getDatasourceDriver());
return DriverManager.getConnection(getJdbcUrl(db2ConnectionParam),
db2ConnectionParam.getUser(), CommonUtils.decodePassword(db2ConnectionParam.getPassword()));
}
@Override
public DbType getDbType() {
return DbType.DB2;
}
private String transformOther(Map<String, String> otherMap) {
if (MapUtils.isEmpty(otherMap)) {
return null;
}
StringBuilder stringBuilder = new StringBuilder();
otherMap.forEach((key, value) -> stringBuilder.append(String.format("%s=%s%s", key, value, ";")));
stringBuilder.deleteCharAt(stringBuilder.length() - 1);
return stringBuilder.toString();
}
private Map<String, String> parseOther(String other) {
if (other == null) {
return null;
}
Map<String, String> otherMap = new LinkedHashMap<>();
for (String config : other.split("&")) {
otherMap.put(config.split("=")[0], config.split("=")[1]);
}
return otherMap;
}
}

38
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/hive/HiveConnectionParam.java Normal file
View File

@ -0,0 +1,38 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.datasource.hive;
import org.apache.dolphinscheduler.common.datasource.BaseHdfsConnectionParam;
public class HiveConnectionParam extends BaseHdfsConnectionParam {
@Override
public String toString() {
return "HiveConnectionParam{"
+ "user='" + user + '\''
+ ", password='" + password + '\''
+ ", address='" + address + '\''
+ ", database='" + database + '\''
+ ", jdbcUrl='" + jdbcUrl + '\''
+ ", other='" + other + '\''
+ ", principal='" + principal + '\''
+ ", javaSecurityKrb5Conf='" + javaSecurityKrb5Conf + '\''
+ ", loginUserKeytabUsername='" + loginUserKeytabUsername + '\''
+ ", loginUserKeytabPath='" + loginUserKeytabPath + '\''
+ '}';
}
}

45
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/hive/HiveDataSourceParamDTO.java Normal file
View File

@ -0,0 +1,45 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.datasource.hive;
import org.apache.dolphinscheduler.common.datasource.BaseHdfsDatasourceParamDTO;
import org.apache.dolphinscheduler.common.enums.DbType;
public class HiveDataSourceParamDTO extends BaseHdfsDatasourceParamDTO {
@Override
public String toString() {
return "HiveDataSourceParamDTO{"
+ "host='" + host + '\''
+ ", port=" + port
+ ", database='" + database + '\''
+ ", principal='" + principal + '\''
+ ", userName='" + userName + '\''
+ ", password='" + password + '\''
+ ", other='" + other + '\''
+ ", javaSecurityKrb5Conf='" + javaSecurityKrb5Conf + '\''
+ ", loginUserKeytabUsername='" + loginUserKeytabUsername + '\''
+ ", loginUserKeytabPath='" + loginUserKeytabPath + '\''
+ '}';
}
@Override
public DbType getType() {
return DbType.HIVE;
}
}

185
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/hive/HiveDatasourceProcessor.java Normal file
View File

@ -0,0 +1,185 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.datasource.hive;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.datasource.AbstractDatasourceProcessor;
import org.apache.dolphinscheduler.common.datasource.BaseConnectionParam;
import org.apache.dolphinscheduler.common.datasource.BaseDataSourceParamDTO;
import org.apache.dolphinscheduler.common.datasource.ConnectionParam;
import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.common.utils.CommonUtils;
import org.apache.dolphinscheduler.common.utils.HiveConfUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.commons.collections4.MapUtils;
import java.io.IOException;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.util.LinkedHashMap;
import java.util.Map;
public class HiveDatasourceProcessor extends AbstractDatasourceProcessor {
@Override
public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) {
HiveDataSourceParamDTO hiveDataSourceParamDTO = new HiveDataSourceParamDTO();
HiveConnectionParam hiveConnectionParam = (HiveConnectionParam) createConnectionParams(connectionJson);
hiveDataSourceParamDTO.setDatabase(hiveConnectionParam.getDatabase());
hiveDataSourceParamDTO.setUserName(hiveConnectionParam.getUser());
hiveDataSourceParamDTO.setOther(parseOther(hiveConnectionParam.getOther()));
hiveDataSourceParamDTO.setLoginUserKeytabUsername(hiveConnectionParam.getLoginUserKeytabUsername());
hiveDataSourceParamDTO.setLoginUserKeytabPath(hiveConnectionParam.getLoginUserKeytabPath());
hiveDataSourceParamDTO.setJavaSecurityKrb5Conf(hiveConnectionParam.getJavaSecurityKrb5Conf());
String[] tmpArray = hiveConnectionParam.getAddress().split(Constants.DOUBLE_SLASH);
StringBuilder hosts = new StringBuilder();
String[] hostPortArray = tmpArray[tmpArray.length - 1].split(Constants.COMMA);
for (String hostPort : hostPortArray) {
hosts.append(hostPort.split(Constants.COLON)[0]).append(Constants.COMMA);
}
hosts.deleteCharAt(hosts.length() - 1);
hiveDataSourceParamDTO.setHost(hosts.toString());
hiveDataSourceParamDTO.setPort(Integer.parseInt(hostPortArray[0].split(Constants.COLON)[1]));
return hiveDataSourceParamDTO;
}
@Override
public BaseConnectionParam createConnectionParams(BaseDataSourceParamDTO datasourceParam) {
HiveDataSourceParamDTO hiveParam = (HiveDataSourceParamDTO) datasourceParam;
StringBuilder address = new StringBuilder();
address.append(Constants.JDBC_HIVE_2);
for (String zkHost : hiveParam.getHost().split(",")) {
address.append(String.format("%s:%s,", zkHost, hiveParam.getPort()));
}
address.deleteCharAt(address.length() - 1);
String jdbcUrl = address.toString() + "/" + hiveParam.getDatabase();
if (CommonUtils.getKerberosStartupState()) {
jdbcUrl += ";principal=" + hiveParam.getPrincipal();
}
HiveConnectionParam hiveConnectionParam = new HiveConnectionParam();
hiveConnectionParam.setDatabase(hiveParam.getDatabase());
hiveConnectionParam.setAddress(address.toString());
hiveConnectionParam.setJdbcUrl(jdbcUrl);
hiveConnectionParam.setUser(hiveParam.getUserName());
hiveConnectionParam.setPassword(CommonUtils.encodePassword(hiveParam.getPassword()));
if (CommonUtils.getKerberosStartupState()) {
hiveConnectionParam.setPrincipal(hiveParam.getPrincipal());
hiveConnectionParam.setJavaSecurityKrb5Conf(hiveParam.getJavaSecurityKrb5Conf());
hiveConnectionParam.setLoginUserKeytabPath(hiveParam.getLoginUserKeytabPath());
hiveConnectionParam.setLoginUserKeytabUsername(hiveParam.getLoginUserKeytabUsername());
}
hiveConnectionParam.setOther(transformOther(hiveParam.getOther()));
return hiveConnectionParam;
}
@Override
public ConnectionParam createConnectionParams(String connectionJson) {
return JSONUtils.parseObject(connectionJson, HiveConnectionParam.class);
}
@Override
public String getDatasourceDriver() {
return Constants.ORG_APACHE_HIVE_JDBC_HIVE_DRIVER;
}
@Override
public String getJdbcUrl(ConnectionParam connectionParam) {
HiveConnectionParam hiveConnectionParam = (HiveConnectionParam) connectionParam;
String jdbcUrl = hiveConnectionParam.getJdbcUrl();
String otherParams = filterOther(hiveConnectionParam.getOther());
if (StringUtils.isNotEmpty(otherParams) && !"?".equals(otherParams.substring(0, 1))) {
jdbcUrl += ";";
}
return jdbcUrl + otherParams;
}
@Override
public Connection getConnection(ConnectionParam connectionParam) throws IOException, ClassNotFoundException, SQLException {
HiveConnectionParam hiveConnectionParam = (HiveConnectionParam) connectionParam;
CommonUtils.loadKerberosConf(hiveConnectionParam.getJavaSecurityKrb5Conf(),
hiveConnectionParam.getLoginUserKeytabUsername(), hiveConnectionParam.getLoginUserKeytabPath());
Class.forName(getDatasourceDriver());
return DriverManager.getConnection(getJdbcUrl(connectionParam),
hiveConnectionParam.getUser(), CommonUtils.decodePassword(hiveConnectionParam.getPassword()));
}
@Override
public DbType getDbType() {
return DbType.HIVE;
}
private String transformOther(Map<String, String> otherMap) {
if (MapUtils.isEmpty(otherMap)) {
return null;
}
StringBuilder stringBuilder = new StringBuilder();
otherMap.forEach((key, value) -> stringBuilder.append(String.format("%s=%s;", key, value)));
return stringBuilder.toString();
}
private String filterOther(String otherParams) {
if (StringUtils.isBlank(otherParams)) {
return "";
}
StringBuilder hiveConfListSb = new StringBuilder();
hiveConfListSb.append("?");
StringBuilder sessionVarListSb = new StringBuilder();
String[] otherArray = otherParams.split(";", -1);
for (String conf : otherArray) {
if (HiveConfUtils.isHiveConfVar(conf)) {
hiveConfListSb.append(conf).append(";");
} else {
sessionVarListSb.append(conf).append(";");
}
}
// remove the last ";"
if (sessionVarListSb.length() > 0) {
sessionVarListSb.deleteCharAt(sessionVarListSb.length() - 1);
}
if (hiveConfListSb.length() > 0) {
hiveConfListSb.deleteCharAt(hiveConfListSb.length() - 1);
}
return sessionVarListSb.toString() + hiveConfListSb.toString();
}
private Map<String, String> parseOther(String other) {
if (other == null) {
return null;
}
Map<String, String> otherMap = new LinkedHashMap<>();
String[] configs = other.split(";");
for (String config : configs) {
otherMap.put(config.split("=")[0], config.split("=")[1]);
}
return otherMap;
}
}

37
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/PostgreDataSource.java → dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/mysql/MysqlConnectionParam.java
View File

@ -14,30 +14,21 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.dao.datasource;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.DbType;
package org.apache.dolphinscheduler.common.datasource.mysql;
/**
* data source of postgreSQL
*/
public class PostgreDataSource extends BaseDataSource {
import org.apache.dolphinscheduler.common.datasource.BaseConnectionParam;
/**
* gets the JDBC url for the data source connection
* @return jdbc url
*/
@Override
public String driverClassSelector() {
return Constants.ORG_POSTGRESQL_DRIVER;
}
/**
* @return db type
*/
@Override
public DbType dbTypeSelector() {
return DbType.POSTGRESQL;
}
public class MysqlConnectionParam extends BaseConnectionParam {
@Override
public String toString() {
return "MysqlConnectionParam{"
+ "user='" + user + '\''
+ ", password='" + password + '\''
+ ", address='" + address + '\''
+ ", database='" + database + '\''
+ ", jdbcUrl='" + jdbcUrl + '\''
+ ", other='" + other + '\''
+ '}';
}
}

43
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/mysql/MysqlDatasourceParamDTO.java Normal file
View File

@ -0,0 +1,43 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.datasource.mysql;
import org.apache.dolphinscheduler.common.datasource.BaseDataSourceParamDTO;
import org.apache.dolphinscheduler.common.enums.DbType;
public class MysqlDatasourceParamDTO extends BaseDataSourceParamDTO {
@Override
public String toString() {
return "MysqlDatasourceParamDTO{"
+ "name='" + name + '\''
+ ", note='" + note + '\''
+ ", host='" + host + '\''
+ ", port=" + port
+ ", database='" + database + '\''
+ ", userName='" + userName + '\''
+ ", password='" + password + '\''
+ ", other='" + other + '\''
+ '}';
}
@Override
public DbType getType() {
return DbType.MYSQL;
}
}

170
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/mysql/MysqlDatasourceProcessor.java Normal file
View File

@ -0,0 +1,170 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.datasource.mysql;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.datasource.AbstractDatasourceProcessor;
import org.apache.dolphinscheduler.common.datasource.BaseConnectionParam;
import org.apache.dolphinscheduler.common.datasource.BaseDataSourceParamDTO;
import org.apache.dolphinscheduler.common.datasource.ConnectionParam;
import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.common.utils.CommonUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.commons.collections4.MapUtils;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.Map;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class MysqlDatasourceProcessor extends AbstractDatasourceProcessor {
private final Logger logger = LoggerFactory.getLogger(MysqlDatasourceProcessor.class);
private static final String ALLOW_LOAD_LOCAL_IN_FILE_NAME = "allowLoadLocalInfile";
private static final String AUTO_DESERIALIZE = "autoDeserialize";
private static final String ALLOW_LOCAL_IN_FILE_NAME = "allowLocalInfile";
private static final String ALLOW_URL_IN_LOCAL_IN_FILE_NAME = "allowUrlInLocalInfile";
private static final String APPEND_PARAMS = "allowLoadLocalInfile=false&autoDeserialize=false&allowLocalInfile=false&allowUrlInLocalInfile=false";
@Override
public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) {
MysqlConnectionParam connectionParams = (MysqlConnectionParam) createConnectionParams(connectionJson);
MysqlDatasourceParamDTO mysqlDatasourceParamDTO = new MysqlDatasourceParamDTO();
mysqlDatasourceParamDTO.setUserName(connectionParams.getUser());
mysqlDatasourceParamDTO.setDatabase(connectionParams.getDatabase());
mysqlDatasourceParamDTO.setOther(parseOther(connectionParams.getOther()));
String address = connectionParams.getAddress();
String[] hostSeperator = address.split(Constants.DOUBLE_SLASH);
String[] hostPortArray = hostSeperator[hostSeperator.length - 1].split(Constants.COMMA);
mysqlDatasourceParamDTO.setPort(Integer.parseInt(hostPortArray[0].split(Constants.COLON)[1]));
mysqlDatasourceParamDTO.setHost(hostPortArray[0].split(Constants.COLON)[0]);
return mysqlDatasourceParamDTO;
}
@Override
public BaseConnectionParam createConnectionParams(BaseDataSourceParamDTO dataSourceParam) {
MysqlDatasourceParamDTO mysqlDatasourceParam = (MysqlDatasourceParamDTO) dataSourceParam;
String address = String.format("%s%s:%s", Constants.JDBC_MYSQL, mysqlDatasourceParam.getHost(), mysqlDatasourceParam.getPort());
String jdbcUrl = String.format("%s/%s", address, mysqlDatasourceParam.getDatabase());
MysqlConnectionParam mysqlConnectionParam = new MysqlConnectionParam();
mysqlConnectionParam.setJdbcUrl(jdbcUrl);
mysqlConnectionParam.setDatabase(mysqlDatasourceParam.getDatabase());
mysqlConnectionParam.setAddress(address);
mysqlConnectionParam.setUser(mysqlDatasourceParam.getUserName());
mysqlConnectionParam.setPassword(CommonUtils.encodePassword(mysqlDatasourceParam.getPassword()));
mysqlConnectionParam.setOther(transformOther(mysqlDatasourceParam.getOther()));
return mysqlConnectionParam;
}
@Override
public ConnectionParam createConnectionParams(String connectionJson) {
return JSONUtils.parseObject(connectionJson, MysqlConnectionParam.class);
}
@Override
public String getDatasourceDriver() {
return Constants.COM_MYSQL_JDBC_DRIVER;
}
@Override
public String getJdbcUrl(ConnectionParam connectionParam) {
MysqlConnectionParam mysqlConnectionParam = (MysqlConnectionParam) connectionParam;
String jdbcUrl = mysqlConnectionParam.getJdbcUrl();
if (StringUtils.isNotEmpty(mysqlConnectionParam.getOther())) {
return String.format("%s?%s&%s", jdbcUrl, mysqlConnectionParam.getOther(), APPEND_PARAMS);
}
return String.format("%s?%s", jdbcUrl, APPEND_PARAMS);
}
@Override
public Connection getConnection(ConnectionParam connectionParam) throws ClassNotFoundException, SQLException {
MysqlConnectionParam mysqlConnectionParam = (MysqlConnectionParam) connectionParam;
Class.forName(getDatasourceDriver());
String user = mysqlConnectionParam.getUser();
if (user.contains(AUTO_DESERIALIZE)) {
logger.warn("sensitive param : {} in username field is filtered", AUTO_DESERIALIZE);
user = user.replace(AUTO_DESERIALIZE, "");
}
String password = CommonUtils.decodePassword(mysqlConnectionParam.getPassword());
if (password.contains(AUTO_DESERIALIZE)) {
logger.warn("sensitive param : {} in password field is filtered", AUTO_DESERIALIZE);
password = password.replace(AUTO_DESERIALIZE, "");
}
return DriverManager.getConnection(getJdbcUrl(connectionParam), user, password);
}
@Override
public DbType getDbType() {
return DbType.MYSQL;
}
private String transformOther(Map<String, String> paramMap) {
if (MapUtils.isEmpty(paramMap)) {
return null;
}
Map<String, String> otherMap = new HashMap<>();
paramMap.forEach((k, v) -> {
if (!checkKeyIsLegitimate(k)) {
return;
}
otherMap.put(k, v);
});
if (MapUtils.isEmpty(otherMap)) {
return null;
}
StringBuilder stringBuilder = new StringBuilder();
otherMap.forEach((key, value) -> stringBuilder.append(String.format("%s=%s&", key, value)));
return stringBuilder.toString();
}
private static boolean checkKeyIsLegitimate(String key) {
return !key.contains(ALLOW_LOAD_LOCAL_IN_FILE_NAME)
&& !key.contains(AUTO_DESERIALIZE)
&& !key.contains(ALLOW_LOCAL_IN_FILE_NAME)
&& !key.contains(ALLOW_URL_IN_LOCAL_IN_FILE_NAME);
}
private Map<String, String> parseOther(String other) {
if (StringUtils.isEmpty(other)) {
return null;
}
Map<String, String> otherMap = new LinkedHashMap<>();
for (String config : other.split("&")) {
otherMap.put(config.split("=")[0], config.split("=")[1]);
}
return otherMap;
}
}

46
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/oracle/OracleConnectionParam.java Normal file
View File

@ -0,0 +1,46 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.datasource.oracle;
import org.apache.dolphinscheduler.common.datasource.BaseConnectionParam;
import org.apache.dolphinscheduler.common.enums.DbConnectType;
public class OracleConnectionParam extends BaseConnectionParam {
protected DbConnectType connectType;
public DbConnectType getConnectType() {
return connectType;
}
public void setConnectType(DbConnectType connectType) {
this.connectType = connectType;
}
@Override
public String toString() {
return "OracleConnectionParam{"
+ "user='" + user + '\''
+ ", password='" + password + '\''
+ ", address='" + address + '\''
+ ", database='" + database + '\''
+ ", jdbcUrl='" + jdbcUrl + '\''
+ ", other='" + other + '\''
+ ", connectType=" + connectType
+ '}';
}
}

46
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/OracleDataSource.java → dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/oracle/OracleDatasourceParamDTO.java
View File

@ -14,16 +14,14 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.dao.datasource;
import org.apache.dolphinscheduler.common.Constants;
package org.apache.dolphinscheduler.common.datasource.oracle;
import org.apache.dolphinscheduler.common.datasource.BaseDataSourceParamDTO;
import org.apache.dolphinscheduler.common.enums.DbConnectType;
import org.apache.dolphinscheduler.common.enums.DbType;
/**
* data source of Oracle
*/
public class OracleDataSource extends BaseDataSource {
public class OracleDatasourceParamDTO extends BaseDataSourceParamDTO {
private DbConnectType connectType;
@ -35,33 +33,23 @@ public class OracleDataSource extends BaseDataSource {
this.connectType = connectType;
}
/**
* @return driver class
*/
@Override
public String driverClassSelector() {
return Constants.COM_ORACLE_JDBC_DRIVER;
public String toString() {
return "OracleDatasourceParamDTO{"
+ "name='" + name + '\''
+ ", note='" + note + '\''
+ ", host='" + host + '\''
+ ", port=" + port
+ ", database='" + database + '\''
+ ", userName='" + userName + '\''
+ ", password='" + password + '\''
+ ", connectType=" + connectType
+ ", other='" + other + '\''
+ '}';
}
/**
* append service name or SID
*/
@Override
protected void appendDatabase(StringBuilder jdbcUrl) {
if (getConnectType() == DbConnectType.ORACLE_SID) {
jdbcUrl.append(":");
} else {
jdbcUrl.append("/");
}
jdbcUrl.append(getDatabase());
}
/**
* @return db type
*/
@Override
public DbType dbTypeSelector() {
public DbType getType() {
return DbType.ORACLE;
}
}

141
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/oracle/OracleDatasourceProcessor.java Normal file
View File

@ -0,0 +1,141 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.datasource.oracle;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.datasource.AbstractDatasourceProcessor;
import org.apache.dolphinscheduler.common.datasource.BaseConnectionParam;
import org.apache.dolphinscheduler.common.datasource.BaseDataSourceParamDTO;
import org.apache.dolphinscheduler.common.datasource.ConnectionParam;
import org.apache.dolphinscheduler.common.enums.DbConnectType;
import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.common.utils.CommonUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.commons.collections4.MapUtils;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
public class OracleDatasourceProcessor extends AbstractDatasourceProcessor {
@Override
public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) {
OracleConnectionParam connectionParams = (OracleConnectionParam) createConnectionParams(connectionJson);
OracleDatasourceParamDTO oracleDatasourceParamDTO = new OracleDatasourceParamDTO();
oracleDatasourceParamDTO.setDatabase(connectionParams.getDatabase());
oracleDatasourceParamDTO.setUserName(connectionParams.getUser());
oracleDatasourceParamDTO.setOther(parseOther(connectionParams.getOther()));
String hostSeperator = Constants.DOUBLE_SLASH;
if (DbConnectType.ORACLE_SID.equals(connectionParams.connectType)) {
hostSeperator = Constants.AT_SIGN;
}
String[] hostPort = connectionParams.getAddress().split(hostSeperator);
String[] hostPortArray = hostPort[hostPort.length - 1].split(Constants.COMMA);
oracleDatasourceParamDTO.setPort(Integer.parseInt(hostPortArray[0].split(Constants.COLON)[1]));
oracleDatasourceParamDTO.setHost(hostPortArray[0].split(Constants.COLON)[0]);
return oracleDatasourceParamDTO;
}
@Override
public BaseConnectionParam createConnectionParams(BaseDataSourceParamDTO datasourceParam) {
OracleDatasourceParamDTO oracleParam = (OracleDatasourceParamDTO) datasourceParam;
String address;
if (DbConnectType.ORACLE_SID.equals(oracleParam.getConnectType())) {
address = String.format("%s%s:%s",
Constants.JDBC_ORACLE_SID, oracleParam.getHost(), oracleParam.getPort());
} else {
address = String.format("%s%s:%s",
Constants.JDBC_ORACLE_SERVICE_NAME, oracleParam.getHost(), oracleParam.getPort());
}
String jdbcUrl = address + "/" + oracleParam.getDatabase();
OracleConnectionParam oracleConnectionParam = new OracleConnectionParam();
oracleConnectionParam.setUser(oracleParam.getUserName());
oracleConnectionParam.setPassword(CommonUtils.encodePassword(oracleParam.getPassword()));
oracleConnectionParam.setAddress(address);
oracleConnectionParam.setJdbcUrl(jdbcUrl);
oracleConnectionParam.setDatabase(oracleParam.getDatabase());
oracleConnectionParam.setConnectType(oracleParam.getConnectType());
oracleConnectionParam.setOther(transformOther(oracleParam.getOther()));
return oracleConnectionParam;
}
@Override
public ConnectionParam createConnectionParams(String connectionJson) {
return JSONUtils.parseObject(connectionJson, OracleConnectionParam.class);
}
@Override
public String getDatasourceDriver() {
return Constants.COM_ORACLE_JDBC_DRIVER;
}
@Override
public String getJdbcUrl(ConnectionParam connectionParam) {
OracleConnectionParam oracleConnectionParam = (OracleConnectionParam) connectionParam;
if (StringUtils.isNotEmpty(oracleConnectionParam.getOther())) {
return String.format("%s?%s", oracleConnectionParam.getJdbcUrl(), oracleConnectionParam.getOther());
}
return oracleConnectionParam.getJdbcUrl();
}
@Override
public Connection getConnection(ConnectionParam connectionParam) throws ClassNotFoundException, SQLException {
OracleConnectionParam oracleConnectionParam = (OracleConnectionParam) connectionParam;
Class.forName(getDatasourceDriver());
return DriverManager.getConnection(getJdbcUrl(connectionParam),
oracleConnectionParam.getUser(), CommonUtils.decodePassword(oracleConnectionParam.getPassword()));
}
@Override
public DbType getDbType() {
return DbType.ORACLE;
}
private String transformOther(Map<String, String> otherMap) {
if (MapUtils.isEmpty(otherMap)) {
return null;
}
List<String> list = new ArrayList<>();
otherMap.forEach((key, value) -> list.add(String.format("%s=%s", key, value)));
return String.join("&", list);
}
private Map<String, String> parseOther(String other) {
if (StringUtils.isEmpty(other)) {
return null;
}
Map<String, String> otherMap = new LinkedHashMap<>();
String[] configs = other.split("&");
for (String config : configs) {
otherMap.put(config.split("=")[0], config.split("=")[1]);
}
return otherMap;
}
}

34
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/postgresql/PostgreSqlConnectionParam.java Normal file
View File

@ -0,0 +1,34 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.datasource.postgresql;
import org.apache.dolphinscheduler.common.datasource.BaseConnectionParam;
public class PostgreSqlConnectionParam extends BaseConnectionParam {
@Override
public String toString() {
return "PostgreSqlConnectionParam{"
+ "user='" + user + '\''
+ ", password='" + password + '\''
+ ", address='" + address + '\''
+ ", database='" + database + '\''
+ ", jdbcUrl='" + jdbcUrl + '\''
+ ", other='" + other + '\''
+ '}';
}
}

41
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/postgresql/PostgreSqlDatasourceParamDTO.java Normal file
View File

@ -0,0 +1,41 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.datasource.postgresql;
import org.apache.dolphinscheduler.common.datasource.BaseDataSourceParamDTO;
import org.apache.dolphinscheduler.common.enums.DbType;
public class PostgreSqlDatasourceParamDTO extends BaseDataSourceParamDTO {
@Override
public String toString() {
return "PostgreSqlDatasourceParamDTO{"
+ "host='" + host + '\''
+ ", port=" + port
+ ", database='" + database + '\''
+ ", userName='" + userName + '\''
+ ", password='" + password + '\''
+ ", other='" + other + '\''
+ '}';
}
@Override
public DbType getType() {
return DbType.POSTGRESQL;
}
}

126
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/postgresql/PostgreSqlDatasourceProcessor.java Normal file
View File

@ -0,0 +1,126 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.datasource.postgresql;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.datasource.AbstractDatasourceProcessor;
import org.apache.dolphinscheduler.common.datasource.BaseConnectionParam;
import org.apache.dolphinscheduler.common.datasource.BaseDataSourceParamDTO;
import org.apache.dolphinscheduler.common.datasource.ConnectionParam;
import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.common.utils.CommonUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.commons.collections4.MapUtils;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.util.LinkedHashMap;
import java.util.Map;
public class PostgreSqlDatasourceProcessor extends AbstractDatasourceProcessor {
@Override
public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) {
PostgreSqlConnectionParam connectionParams = (PostgreSqlConnectionParam) createConnectionParams(connectionJson);
PostgreSqlDatasourceParamDTO postgreSqlDatasourceParamDTO = new PostgreSqlDatasourceParamDTO();
postgreSqlDatasourceParamDTO.setDatabase(connectionParams.getDatabase());
postgreSqlDatasourceParamDTO.setUserName(connectionParams.getUser());
postgreSqlDatasourceParamDTO.setOther(parseOther(connectionParams.getOther()));
String address = connectionParams.getAddress();
String[] hostSeperator = address.split(Constants.DOUBLE_SLASH);
String[] hostPortArray = hostSeperator[hostSeperator.length - 1].split(Constants.COMMA);
postgreSqlDatasourceParamDTO.setHost(hostPortArray[0].split(Constants.COLON)[0]);
postgreSqlDatasourceParamDTO.setPort(Integer.parseInt(hostPortArray[0].split(Constants.COLON)[1]));
return postgreSqlDatasourceParamDTO;
}
@Override
public BaseConnectionParam createConnectionParams(BaseDataSourceParamDTO datasourceParam) {
PostgreSqlDatasourceParamDTO postgreSqlParam = (PostgreSqlDatasourceParamDTO) datasourceParam;
String address = String.format("%s%s:%s", Constants.JDBC_POSTGRESQL, postgreSqlParam.getHost(), postgreSqlParam.getPort());
String jdbcUrl = String.format("%s/%s", address, postgreSqlParam.getDatabase());
PostgreSqlConnectionParam postgreSqlConnectionParam = new PostgreSqlConnectionParam();
postgreSqlConnectionParam.setJdbcUrl(jdbcUrl);
postgreSqlConnectionParam.setAddress(address);
postgreSqlConnectionParam.setDatabase(postgreSqlParam.getDatabase());
postgreSqlConnectionParam.setUser(postgreSqlParam.getUserName());
postgreSqlConnectionParam.setPassword(CommonUtils.encodePassword(postgreSqlParam.getPassword()));
postgreSqlConnectionParam.setOther(transformOther(postgreSqlParam.getOther()));
return postgreSqlConnectionParam;
}
@Override
public ConnectionParam createConnectionParams(String connectionJson) {
return JSONUtils.parseObject(connectionJson, PostgreSqlConnectionParam.class);
}
@Override
public String getDatasourceDriver() {
return Constants.ORG_POSTGRESQL_DRIVER;
}
@Override
public String getJdbcUrl(ConnectionParam connectionParam) {
PostgreSqlConnectionParam postgreSqlConnectionParam = (PostgreSqlConnectionParam) connectionParam;
if (StringUtils.isNotEmpty(postgreSqlConnectionParam.getOther())) {
return String.format("%s?%s", postgreSqlConnectionParam.getJdbcUrl(), postgreSqlConnectionParam.getOther());
}
return postgreSqlConnectionParam.getJdbcUrl();
}
@Override
public Connection getConnection(ConnectionParam connectionParam) throws ClassNotFoundException, SQLException {
PostgreSqlConnectionParam postgreSqlConnectionParam = (PostgreSqlConnectionParam) connectionParam;
Class.forName(getDatasourceDriver());
return DriverManager.getConnection(getJdbcUrl(postgreSqlConnectionParam),
postgreSqlConnectionParam.getUser(), CommonUtils.decodePassword(postgreSqlConnectionParam.getPassword()));
}
@Override
public DbType getDbType() {
return DbType.POSTGRESQL;
}
private String transformOther(Map<String, String> otherMap) {
if (MapUtils.isEmpty(otherMap)) {
return null;
}
StringBuilder stringBuilder = new StringBuilder();
otherMap.forEach((key, value) -> stringBuilder.append(String.format("%s=%s&", key, value)));
return stringBuilder.toString();
}
private Map<String, String> parseOther(String other) {
if (StringUtils.isEmpty(other)) {
return null;
}
Map<String, String> otherMap = new LinkedHashMap<>();
for (String config : other.split("&")) {
String[] split = config.split("=");
otherMap.put(split[0], split[1]);
}
return otherMap;
}
}

34
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/presto/PrestoConnectionParam.java Normal file
View File

@ -0,0 +1,34 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.datasource.presto;
import org.apache.dolphinscheduler.common.datasource.BaseConnectionParam;
public class PrestoConnectionParam extends BaseConnectionParam {
@Override
public String toString() {
return "PrestoConnectionParam{"
+ "user='" + user + '\''
+ ", password='" + password + '\''
+ ", address='" + address + '\''
+ ", database='" + database + '\''
+ ", jdbcUrl='" + jdbcUrl + '\''
+ ", other='" + other + '\''
+ '}';
}
}

43
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/presto/PrestoDatasourceParamDTO.java Normal file
View File

@ -0,0 +1,43 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.datasource.presto;
import org.apache.dolphinscheduler.common.datasource.BaseDataSourceParamDTO;
import org.apache.dolphinscheduler.common.enums.DbType;
public class PrestoDatasourceParamDTO extends BaseDataSourceParamDTO {
@Override
public String toString() {
return "PrestoDatasourceParamDTO{"
+ "name='" + name + '\''
+ ", note='" + note + '\''
+ ", host='" + host + '\''
+ ", port=" + port
+ ", database='" + database + '\''
+ ", userName='" + userName + '\''
+ ", password='" + password + '\''
+ ", other='" + other + '\''
+ '}';
}
@Override
public DbType getType() {
return DbType.PRESTO;
}
}

128
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/presto/PrestoDatasourceProcessor.java Normal file
View File

@ -0,0 +1,128 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.datasource.presto;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.datasource.AbstractDatasourceProcessor;
import org.apache.dolphinscheduler.common.datasource.BaseConnectionParam;
import org.apache.dolphinscheduler.common.datasource.BaseDataSourceParamDTO;
import org.apache.dolphinscheduler.common.datasource.ConnectionParam;
import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.common.utils.CommonUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.commons.collections4.MapUtils;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
public class PrestoDatasourceProcessor extends AbstractDatasourceProcessor {
@Override
public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) {
PrestoConnectionParam connectionParams = (PrestoConnectionParam) createConnectionParams(connectionJson);
String[] hostSeperator = connectionParams.getAddress().split(Constants.DOUBLE_SLASH);
String[] hostPortArray = hostSeperator[hostSeperator.length - 1].split(Constants.COMMA);
PrestoDatasourceParamDTO prestoDatasourceParamDTO = new PrestoDatasourceParamDTO();
prestoDatasourceParamDTO.setPort(Integer.parseInt(hostPortArray[0].split(Constants.COLON)[1]));
prestoDatasourceParamDTO.setHost(hostPortArray[0].split(Constants.COLON)[0]);
prestoDatasourceParamDTO.setDatabase(connectionParams.getDatabase());
prestoDatasourceParamDTO.setUserName(connectionParams.getUser());
prestoDatasourceParamDTO.setOther(parseOther(connectionParams.getOther()));
return prestoDatasourceParamDTO;
}
@Override
public BaseConnectionParam createConnectionParams(BaseDataSourceParamDTO datasourceParam) {
PrestoDatasourceParamDTO prestoParam = (PrestoDatasourceParamDTO) datasourceParam;
String address = String.format("%s%s:%s", Constants.JDBC_PRESTO, prestoParam.getHost(), prestoParam.getPort());
String jdbcUrl = address + "/" + prestoParam.getDatabase();
PrestoConnectionParam prestoConnectionParam = new PrestoConnectionParam();
prestoConnectionParam.setUser(prestoParam.getUserName());
prestoConnectionParam.setPassword(CommonUtils.encodePassword(prestoParam.getPassword()));
prestoConnectionParam.setOther(transformOther(prestoParam.getOther()));
prestoConnectionParam.setAddress(address);
prestoConnectionParam.setJdbcUrl(jdbcUrl);
prestoConnectionParam.setDatabase(prestoParam.getDatabase());
return prestoConnectionParam;
}
@Override
public ConnectionParam createConnectionParams(String connectionJson) {
return JSONUtils.parseObject(connectionJson, PrestoConnectionParam.class);
}
@Override
public String getDatasourceDriver() {
return Constants.COM_PRESTO_JDBC_DRIVER;
}
@Override
public String getJdbcUrl(ConnectionParam connectionParam) {
PrestoConnectionParam prestoConnectionParam = (PrestoConnectionParam) connectionParam;
if (StringUtils.isNotEmpty(prestoConnectionParam.getOther())) {
return String.format("%s?%s", prestoConnectionParam.getJdbcUrl(), prestoConnectionParam.getOther());
}
return prestoConnectionParam.getJdbcUrl();
}
@Override
public Connection getConnection(ConnectionParam connectionParam) throws ClassNotFoundException, SQLException {
PrestoConnectionParam prestoConnectionParam = (PrestoConnectionParam) connectionParam;
Class.forName(getDatasourceDriver());
return DriverManager.getConnection(getJdbcUrl(connectionParam),
prestoConnectionParam.getUser(), CommonUtils.decodePassword(prestoConnectionParam.getPassword()));
}
@Override
public DbType getDbType() {
return DbType.PRESTO;
}
private String transformOther(Map<String, String> otherMap) {
if (MapUtils.isNotEmpty(otherMap)) {
List<String> list = new ArrayList<>();
otherMap.forEach((key, value) -> list.add(String.format("%s=%s", key, value)));
return String.join("&", list);
}
return null;
}
private Map<String, String> parseOther(String other) {
if (StringUtils.isEmpty(other)) {
return null;
}
Map<String, String> otherMap = new LinkedHashMap<>();
String[] configs = other.split("&");
for (String config : configs) {
otherMap.put(config.split("=")[0], config.split("=")[1]);
}
return otherMap;
}
}

38
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/spark/SparkConnectionParam.java Normal file
View File

@ -0,0 +1,38 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.datasource.spark;
import org.apache.dolphinscheduler.common.datasource.BaseHdfsConnectionParam;
public class SparkConnectionParam extends BaseHdfsConnectionParam {
@Override
public String toString() {
return "SparkConnectionParam{"
+ "user='" + user + '\''
+ ", password='" + password + '\''
+ ", address='" + address + '\''
+ ", database='" + database + '\''
+ ", jdbcUrl='" + jdbcUrl + '\''
+ ", other='" + other + '\''
+ ", principal='" + principal + '\''
+ ", javaSecurityKrb5Conf='" + javaSecurityKrb5Conf + '\''
+ ", loginUserKeytabUsername='" + loginUserKeytabUsername + '\''
+ ", loginUserKeytabPath='" + loginUserKeytabPath + '\''
+ '}';
}
}

47
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/SparkDataSource.java → dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/spark/SparkDatasourceParamDTO.java
View File

@ -15,44 +15,31 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.dao.datasource;
package org.apache.dolphinscheduler.common.datasource.spark;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.datasource.BaseHdfsDatasourceParamDTO;
import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.common.utils.CommonUtils;
import java.sql.Connection;
public class SparkDatasourceParamDTO extends BaseHdfsDatasourceParamDTO {
/**
* data source of spark
*/
public class SparkDataSource extends BaseDataSource {
/**
* gets the JDBC url for the data source connection
* @return jdbc url
*/
@Override
public String driverClassSelector() {
return Constants.ORG_APACHE_HIVE_JDBC_HIVE_DRIVER;
public String toString() {
return "SparkDatasourceParamDTO{"
+ "host='" + host + '\''
+ ", port=" + port
+ ", database='" + database + '\''
+ ", principal='" + principal + '\''
+ ", userName='" + userName + '\''
+ ", password='" + password + '\''
+ ", other='" + other + '\''
+ ", javaSecurityKrb5Conf='" + javaSecurityKrb5Conf + '\''
+ ", loginUserKeytabUsername='" + loginUserKeytabUsername + '\''
+ ", loginUserKeytabPath='" + loginUserKeytabPath + '\''
+ '}';
}
/**
* @return db type
*/
@Override
public DbType dbTypeSelector() {
public DbType getType() {
return DbType.SPARK;
}
/**
* the data source test connection
* @return Connection Connection
* @throws Exception Exception
*/
@Override
public Connection getConnection() throws Exception {
CommonUtils.loadKerberosConf(getJavaSecurityKrb5Conf(), getLoginUserKeytabUsername(), getLoginUserKeytabPath());
return super.getConnection();
}
}

154
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/spark/SparkDatasourceProcessor.java Normal file
View File

@ -0,0 +1,154 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.datasource.spark;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.datasource.AbstractDatasourceProcessor;
import org.apache.dolphinscheduler.common.datasource.BaseConnectionParam;
import org.apache.dolphinscheduler.common.datasource.BaseDataSourceParamDTO;
import org.apache.dolphinscheduler.common.datasource.ConnectionParam;
import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.common.utils.CommonUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.commons.collections4.MapUtils;
import java.io.IOException;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.util.Arrays;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
public class SparkDatasourceProcessor extends AbstractDatasourceProcessor {
@Override
public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) {
SparkConnectionParam connectionParams = (SparkConnectionParam) createConnectionParams(connectionJson);
SparkDatasourceParamDTO sparkDatasourceParamDTO = new SparkDatasourceParamDTO();
sparkDatasourceParamDTO.setDatabase(connectionParams.getDatabase());
sparkDatasourceParamDTO.setUserName(connectionParams.getUser());
sparkDatasourceParamDTO.setOther(parseOther(connectionParams.getOther()));
sparkDatasourceParamDTO.setJavaSecurityKrb5Conf(connectionParams.getJavaSecurityKrb5Conf());
sparkDatasourceParamDTO.setLoginUserKeytabPath(connectionParams.getLoginUserKeytabPath());
sparkDatasourceParamDTO.setLoginUserKeytabUsername(connectionParams.getLoginUserKeytabUsername());
StringBuilder hosts = new StringBuilder();
String[] tmpArray = connectionParams.getAddress().split(Constants.DOUBLE_SLASH);
String[] hostPortArray = tmpArray[tmpArray.length - 1].split(Constants.COMMA);
Arrays.stream(hostPortArray).forEach(hostPort -> hosts.append(hostPort.split(Constants.COLON)[0]).append(Constants.COMMA));
hosts.deleteCharAt(hosts.length() - 1);
sparkDatasourceParamDTO.setHost(hosts.toString());
sparkDatasourceParamDTO.setPort(Integer.parseInt(hostPortArray[0].split(Constants.COLON)[1]));
return sparkDatasourceParamDTO;
}
@Override
public BaseConnectionParam createConnectionParams(BaseDataSourceParamDTO dataSourceParam) {
StringBuilder address = new StringBuilder();
SparkDatasourceParamDTO sparkDatasourceParam = (SparkDatasourceParamDTO) dataSourceParam;
address.append(Constants.JDBC_HIVE_2);
for (String zkHost : sparkDatasourceParam.getHost().split(",")) {
address.append(String.format("%s:%s,", zkHost, sparkDatasourceParam.getPort()));
}
address.deleteCharAt(address.length() - 1);
String jdbcUrl = address + "/" + sparkDatasourceParam.getDatabase();
if (CommonUtils.getKerberosStartupState()) {
jdbcUrl += ";principal=" + sparkDatasourceParam.getPrincipal();
}
SparkConnectionParam sparkConnectionParam = new SparkConnectionParam();
sparkConnectionParam.setPassword(CommonUtils.encodePassword(sparkDatasourceParam.getPassword()));
sparkConnectionParam.setUser(sparkDatasourceParam.getUserName());
sparkConnectionParam.setOther(transformOther(sparkDatasourceParam.getOther()));
sparkConnectionParam.setDatabase(sparkDatasourceParam.getDatabase());
sparkConnectionParam.setAddress(address.toString());
sparkConnectionParam.setJdbcUrl(jdbcUrl);
if (CommonUtils.getKerberosStartupState()) {
sparkConnectionParam.setPrincipal(sparkDatasourceParam.getPrincipal());
sparkConnectionParam.setJavaSecurityKrb5Conf(sparkDatasourceParam.getJavaSecurityKrb5Conf());
sparkConnectionParam.setLoginUserKeytabPath(sparkDatasourceParam.getLoginUserKeytabPath());
sparkConnectionParam.setLoginUserKeytabUsername(sparkDatasourceParam.getLoginUserKeytabUsername());
}
return sparkConnectionParam;
}
@Override
public ConnectionParam createConnectionParams(String connectionJson) {
return JSONUtils.parseObject(connectionJson, SparkConnectionParam.class);
}
@Override
public String getDatasourceDriver() {
return Constants.ORG_APACHE_HIVE_JDBC_HIVE_DRIVER;
}
@Override
public String getJdbcUrl(ConnectionParam connectionParam) {
SparkConnectionParam sparkConnectionParam = (SparkConnectionParam) connectionParam;
if (StringUtils.isNotEmpty(sparkConnectionParam.getOther())) {
return String.format("%s;%s", sparkConnectionParam.getJdbcUrl(), sparkConnectionParam.getOther());
}
return sparkConnectionParam.getJdbcUrl();
}
@Override
public Connection getConnection(ConnectionParam connectionParam) throws IOException, ClassNotFoundException, SQLException {
SparkConnectionParam sparkConnectionParam = (SparkConnectionParam) connectionParam;
CommonUtils.loadKerberosConf(sparkConnectionParam.getJavaSecurityKrb5Conf(),
sparkConnectionParam.getLoginUserKeytabUsername(), sparkConnectionParam.getLoginUserKeytabPath());
Class.forName(getDatasourceDriver());
return DriverManager.getConnection(getJdbcUrl(sparkConnectionParam),
sparkConnectionParam.getUser(), CommonUtils.decodePassword(sparkConnectionParam.getPassword()));
}
@Override
public DbType getDbType() {
return DbType.SPARK;
}
private String transformOther(Map<String, String> otherMap) {
if (MapUtils.isEmpty(otherMap)) {
return null;
}
List<String> stringBuilder = otherMap.entrySet().stream()
.map(entry -> String.format("%s=%s", entry.getKey(), entry.getValue())).collect(Collectors.toList());
return String.join(";", stringBuilder);
}
private Map<String, String> parseOther(String other) {
if (StringUtils.isEmpty(other)) {
return null;
}
Map<String, String> otherMap = new LinkedHashMap<>();
String[] configs = other.split(";");
for (String config : configs) {
otherMap.put(config.split("=")[0], config.split("=")[1]);
}
return otherMap;
}
}

34
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/sqlserver/SqlServerConnectionParam.java Normal file
View File

@ -0,0 +1,34 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.datasource.sqlserver;
import org.apache.dolphinscheduler.common.datasource.BaseConnectionParam;
public class SqlServerConnectionParam extends BaseConnectionParam {
@Override
public String toString() {
return "SqlServerConnectionParam{"
+ "user='" + user + '\''
+ ", password='" + password + '\''
+ ", address='" + address + '\''
+ ", database='" + database + '\''
+ ", jdbcUrl='" + jdbcUrl + '\''
+ ", other='" + other + '\''
+ '}';
}
}

47
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/SQLServerDataSource.java → dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/sqlserver/SqlServerDatasourceParamDTO.java
View File

@ -15,46 +15,29 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.dao.datasource;
package org.apache.dolphinscheduler.common.datasource.sqlserver;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.datasource.BaseDataSourceParamDTO;
import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.common.utils.StringUtils;
/**
* data source of SQL Server
*/
public class SQLServerDataSource extends BaseDataSource {
public class SqlServerDatasourceParamDTO extends BaseDataSourceParamDTO {
/**
* gets the JDBC url for the data source connection
* @return jdbc url
*/
@Override
public String getJdbcUrl() {
String jdbcUrl = getAddress();
jdbcUrl += ";databaseName=" + getDatabase();
if (StringUtils.isNotEmpty(getOther())) {
jdbcUrl += ";" + getOther();
}
return jdbcUrl;
public String toString() {
return "SqlServerDatasourceParamDTO{"
+ "name='" + name + '\''
+ ", note='" + note + '\''
+ ", host='" + host + '\''
+ ", port=" + port
+ ", database='" + database + '\''
+ ", userName='" + userName + '\''
+ ", password='" + password + '\''
+ ", other='" + other + '\''
+ '}';
}
/**
* @return driver class
*/
@Override
public String driverClassSelector() {
return Constants.COM_SQLSERVER_JDBC_DRIVER;
}
/**
* @return db type
*/
@Override
public DbType dbTypeSelector() {
public DbType getType() {
return DbType.SQLSERVER;
}
}

123
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/sqlserver/SqlServerDatasourceProcessor.java Normal file
View File

@ -0,0 +1,123 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.datasource.sqlserver;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.datasource.AbstractDatasourceProcessor;
import org.apache.dolphinscheduler.common.datasource.BaseConnectionParam;
import org.apache.dolphinscheduler.common.datasource.BaseDataSourceParamDTO;
import org.apache.dolphinscheduler.common.datasource.ConnectionParam;
import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.common.utils.CommonUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.commons.collections4.MapUtils;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.util.LinkedHashMap;
import java.util.Map;
public class SqlServerDatasourceProcessor extends AbstractDatasourceProcessor {
@Override
public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) {
SqlServerConnectionParam connectionParams = (SqlServerConnectionParam) createConnectionParams(connectionJson);
String[] hostSeperator = connectionParams.getAddress().split(Constants.DOUBLE_SLASH);
String[] hostPortArray = hostSeperator[hostSeperator.length - 1].split(Constants.COMMA);
SqlServerDatasourceParamDTO sqlServerDatasourceParamDTO = new SqlServerDatasourceParamDTO();
sqlServerDatasourceParamDTO.setDatabase(connectionParams.getDatabase());
sqlServerDatasourceParamDTO.setUserName(connectionParams.getUser());
sqlServerDatasourceParamDTO.setOther(parseOther(connectionParams.getOther()));
sqlServerDatasourceParamDTO.setPort(Integer.parseInt(hostPortArray[0].split(Constants.COLON)[1]));
sqlServerDatasourceParamDTO.setHost(hostPortArray[0].split(Constants.COLON)[0]);
return sqlServerDatasourceParamDTO;
}
@Override
public BaseConnectionParam createConnectionParams(BaseDataSourceParamDTO datasourceParam) {
SqlServerDatasourceParamDTO sqlServerParam = (SqlServerDatasourceParamDTO) datasourceParam;
String address = String.format("%s%s:%s", Constants.JDBC_SQLSERVER, sqlServerParam.getHost(), sqlServerParam.getPort());
String jdbcUrl = address + ";databaseName=" + sqlServerParam.getDatabase();
SqlServerConnectionParam sqlServerConnectionParam = new SqlServerConnectionParam();
sqlServerConnectionParam.setAddress(address);
sqlServerConnectionParam.setDatabase(sqlServerParam.getDatabase());
sqlServerConnectionParam.setJdbcUrl(jdbcUrl);
sqlServerConnectionParam.setOther(transformOther(sqlServerParam.getOther()));
sqlServerConnectionParam.setUser(sqlServerParam.getUserName());
sqlServerConnectionParam.setPassword(CommonUtils.encodePassword(sqlServerParam.getPassword()));
return sqlServerConnectionParam;
}
@Override
public BaseConnectionParam createConnectionParams(String connectionJson) {
return JSONUtils.parseObject(connectionJson, SqlServerConnectionParam.class);
}
@Override
public String getDatasourceDriver() {
return Constants.COM_SQLSERVER_JDBC_DRIVER;
}
@Override
public String getJdbcUrl(ConnectionParam connectionParam) {
SqlServerConnectionParam sqlServerConnectionParam = (SqlServerConnectionParam) connectionParam;
if (StringUtils.isNotEmpty(sqlServerConnectionParam.getOther())) {
return String.format("%s;%s", sqlServerConnectionParam.getJdbcUrl(), sqlServerConnectionParam.getOther());
}
return sqlServerConnectionParam.getJdbcUrl();
}
@Override
public Connection getConnection(ConnectionParam connectionParam) throws ClassNotFoundException, SQLException {
SqlServerConnectionParam sqlServerConnectionParam = (SqlServerConnectionParam) connectionParam;
Class.forName(getDatasourceDriver());
return DriverManager.getConnection(getJdbcUrl(connectionParam), sqlServerConnectionParam.getUser(),
CommonUtils.decodePassword(sqlServerConnectionParam.getPassword()));
}
@Override
public DbType getDbType() {
return DbType.SQLSERVER;
}
private String transformOther(Map<String, String> otherMap) {
if (MapUtils.isEmpty(otherMap)) {
return null;
}
StringBuilder stringBuilder = new StringBuilder();
otherMap.forEach((key, value) -> stringBuilder.append(String.format("%s=%s;", key, value)));
return stringBuilder.toString();
}
private Map<String, String> parseOther(String other) {
if (StringUtils.isEmpty(other)) {
return null;
}
Map<String, String> otherMap = new LinkedHashMap<>();
for (String config : other.split(";")) {
otherMap.put(config.split("=")[0], config.split("=")[1]);
}
return otherMap;
}
}

1
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/DbConnectType.java
View File

@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.enums;
import com.baomidou.mybatisplus.annotation.EnumValue;

3
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/DependResult.java
View File

@ -26,6 +26,7 @@ public enum DependResult {
* 0 success
* 1 waiting
* 2 failed
* 3 non execution
*/
SUCCESS, WAITING, FAILED
SUCCESS, WAITING, FAILED, NON_EXEC
}

5
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CommonUtils.java
View File

@ -24,6 +24,7 @@ import org.apache.commons.codec.binary.Base64;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
import java.io.IOException;
import java.net.URL;
import java.nio.charset.StandardCharsets;
@ -103,9 +104,9 @@ public class CommonUtils {
* @param javaSecurityKrb5Conf javaSecurityKrb5Conf
* @param loginUserKeytabUsername loginUserKeytabUsername
* @param loginUserKeytabPath loginUserKeytabPath
* @throws Exception errors
* @throws IOException errors
*/
public static void loadKerberosConf(String javaSecurityKrb5Conf, String loginUserKeytabUsername, String loginUserKeytabPath) throws Exception {
public static void loadKerberosConf(String javaSecurityKrb5Conf, String loginUserKeytabUsername, String loginUserKeytabPath) throws IOException {
if (CommonUtils.getKerberosStartupState()) {
System.setProperty(Constants.JAVA_SECURITY_KRB5_CONF, StringUtils.defaultIfBlank(javaSecurityKrb5Conf, PropertyUtils.getString(Constants.JAVA_SECURITY_KRB5_CONF_PATH)));
Configuration configuration = new Configuration();

63
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/IpUtils.java
View File

@ -1,63 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.utils;
/**
* http utils
*/
public class IpUtils {
private IpUtils() {
throw new UnsupportedOperationException("Construct IpUtils");
}
public static final String DOT = ".";
/**
* ip str to long <p>
*
* @param ipStr ip string
* @return ip to long
*/
public static Long ipToLong(String ipStr) {
String[] ipSet = ipStr.split("\\" + DOT);
return Long.parseLong(ipSet[0]) << 24 | Long.parseLong(ipSet[1]) << 16 | Long.parseLong(ipSet[2]) << 8 | Long.parseLong(ipSet[3]);
}
/**
* long to ip
*
* @param ipLong the long number converted from IP
* @return String
*/
public static String longToIp(long ipLong) {
long[] ipNumbers = new long[4];
long tmp = 0xFF;
ipNumbers[0] = ipLong >> 24 & tmp;
ipNumbers[1] = ipLong >> 16 & tmp;
ipNumbers[2] = ipLong >> 8 & tmp;
ipNumbers[3] = ipLong & tmp;
return ipNumbers[0] + DOT
+ ipNumbers[1] + DOT
+ ipNumbers[2] + DOT
+ ipNumbers[3];
}
}

3
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/NetUtils.java
View File

@ -86,6 +86,9 @@ public class NetUtils {
if (STS_PATTERN.matcher(host).find()) {
return String.format("%s.%s", host, host.replaceFirst("\\d+$", "headless"));
}
} else if (canonicalHost.contains(".")) {
String[] items = canonicalHost.split("\\.");
return String.format("%s.%s", items[0], items[1]);
}
return canonicalHost;
}

47
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java
View File

@ -17,11 +17,8 @@
package org.apache.dolphinscheduler.common.utils;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.shell.ShellExecutor;
import org.apache.commons.configuration.Configuration;
import java.io.BufferedReader;
import java.io.FileInputStream;
import java.io.IOException;
@ -85,7 +82,7 @@ public class OSUtils {
*/
public static double memoryUsage() {
GlobalMemory memory = hal.getMemory();
double memoryUsage = (memory.getTotal() - memory.getAvailable() - memory.getSwapUsed()) * 0.1 / memory.getTotal() * 10;
double memoryUsage = (memory.getTotal() - memory.getAvailable()) * 1.0 / memory.getTotal();
DecimalFormat df = new DecimalFormat(TWO_DECIMAL);
df.setRoundingMode(RoundingMode.HALF_UP);
@ -101,12 +98,11 @@ public class OSUtils {
*/
public static double availablePhysicalMemorySize() {
GlobalMemory memory = hal.getMemory();
double availablePhysicalMemorySize = (memory.getAvailable() + memory.getSwapUsed()) / 1024.0 / 1024 / 1024;
double availablePhysicalMemorySize = memory.getAvailable() / 1024.0 / 1024 / 1024;
DecimalFormat df = new DecimalFormat(TWO_DECIMAL);
df.setRoundingMode(RoundingMode.HALF_UP);
return Double.parseDouble(df.format(availablePhysicalMemorySize));
}
/**
@ -116,13 +112,13 @@ public class OSUtils {
*
* @return available Physical Memory Size, unit: G
*/
public static double totalMemorySize() {
public static double totalPhysicalMemorySize() {
GlobalMemory memory = hal.getMemory();
double availablePhysicalMemorySize = memory.getTotal() / 1024.0 / 1024 / 1024;
double totalPhysicalMemorySize = memory.getTotal() / 1024.0 / 1024 / 1024;
DecimalFormat df = new DecimalFormat(TWO_DECIMAL);
df.setRoundingMode(RoundingMode.HALF_UP);
return Double.parseDouble(df.format(availablePhysicalMemorySize));
return Double.parseDouble(df.format(totalPhysicalMemorySize));
}
/**
@ -486,43 +482,22 @@ public class OSUtils {
/**
* check memory and cpu usage
*
* @param systemCpuLoad systemCpuLoad
* @param systemReservedMemory systemReservedMemory
* @param maxCpuloadAvg maxCpuloadAvg
* @param reservedMemory reservedMemory
* @return check memory and cpu usage
*/
public static Boolean checkResource(double systemCpuLoad, double systemReservedMemory) {
public static Boolean checkResource(double maxCpuloadAvg, double reservedMemory) {
// system load average
double loadAverage = loadAverage();
// system available physical memory
double availablePhysicalMemorySize = availablePhysicalMemorySize();
if (loadAverage > systemCpuLoad || availablePhysicalMemorySize < systemReservedMemory) {
logger.warn("load is too high or availablePhysicalMemorySize(G) is too low, it's availablePhysicalMemorySize(G):{},loadAvg:{}", availablePhysicalMemorySize, loadAverage);
if (loadAverage > maxCpuloadAvg || availablePhysicalMemorySize < reservedMemory) {
logger.warn("current cpu load average {} is too high or available memory {}G is too low, under max.cpuload.avg={} and reserved.memory={}G",
loadAverage, availablePhysicalMemorySize, maxCpuloadAvg, reservedMemory);
return false;
} else {
return true;
}
}
/**
* check memory and cpu usage
*
* @param conf conf
* @param isMaster is master
* @return check memory and cpu usage
*/
public static Boolean checkResource(Configuration conf, Boolean isMaster) {
double systemCpuLoad;
double systemReservedMemory;
if (Boolean.TRUE.equals(isMaster)) {
systemCpuLoad = conf.getDouble(Constants.MASTER_MAX_CPULOAD_AVG, Constants.DEFAULT_MASTER_CPU_LOAD);
systemReservedMemory = conf.getDouble(Constants.MASTER_RESERVED_MEMORY, Constants.DEFAULT_MASTER_RESERVED_MEMORY);
} else {
systemCpuLoad = conf.getDouble(Constants.WORKER_MAX_CPULOAD_AVG, Constants.DEFAULT_WORKER_CPU_LOAD);
systemReservedMemory = conf.getDouble(Constants.WORKER_RESERVED_MEMORY, Constants.DEFAULT_WORKER_RESERVED_MEMORY);
}
return checkResource(systemCpuLoad, systemReservedMemory);
}
}

2
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ParameterUtils.java
View File

@ -45,7 +45,7 @@ public class ParameterUtils {
private static final Logger logger = LoggerFactory.getLogger(ParameterUtils.class);
private static final String DATE_PARSE_PATTERN = "\\$\\[([^\\]]+)]";
private static final String DATE_PARSE_PATTERN = "\\$\\[([^\\$\\]]+)]";
private static final String DATE_START_PATTERN = "^[0-9]";

9
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java
View File

@ -57,19 +57,12 @@ public class PropertyUtils {
*/
public static synchronized void loadPropertyFile(String... propertyFiles) {
for (String fileName : propertyFiles) {
InputStream fis = null;
try {
fis = PropertyUtils.class.getResourceAsStream(fileName);
try (InputStream fis = PropertyUtils.class.getResourceAsStream(fileName);) {
properties.load(fis);
} catch (IOException e) {
logger.error(e.getMessage(), e);
if (fis != null) {
IOUtils.closeQuietly(fis);
}
System.exit(1);
} finally {
IOUtils.closeQuietly(fis);
}
}
}

130
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/datasource/DatasourceUtilTest.java Normal file
View File

@ -0,0 +1,130 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.datasource;
import org.apache.dolphinscheduler.common.datasource.mysql.MysqlConnectionParam;
import org.apache.dolphinscheduler.common.datasource.mysql.MysqlDatasourceParamDTO;
import org.apache.dolphinscheduler.common.datasource.mysql.MysqlDatasourceProcessor;
import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mockito;
import org.powermock.api.mockito.PowerMockito;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
@RunWith(PowerMockRunner.class)
@PrepareForTest({Class.class, DriverManager.class, MysqlDatasourceProcessor.class})
public class DatasourceUtilTest {
@Test
public void testCheckDatasourceParam() {
MysqlDatasourceParamDTO mysqlDatasourceParamDTO = new MysqlDatasourceParamDTO();
mysqlDatasourceParamDTO.setHost("localhost");
mysqlDatasourceParamDTO.setDatabase("default");
mysqlDatasourceParamDTO.setOther(null);
DatasourceUtil.checkDatasourceParam(mysqlDatasourceParamDTO);
Assert.assertTrue(true);
}
@Test
public void testBuildConnectionParams() {
MysqlDatasourceParamDTO mysqlDatasourceParamDTO = new MysqlDatasourceParamDTO();
mysqlDatasourceParamDTO.setHost("localhost");
mysqlDatasourceParamDTO.setDatabase("default");
mysqlDatasourceParamDTO.setUserName("root");
mysqlDatasourceParamDTO.setPort(3306);
mysqlDatasourceParamDTO.setPassword("123456");
ConnectionParam connectionParam = DatasourceUtil.buildConnectionParams(mysqlDatasourceParamDTO);
Assert.assertNotNull(connectionParam);
}
@Test
public void testBuildConnectionParams2() {
MysqlDatasourceParamDTO mysqlDatasourceParamDTO = new MysqlDatasourceParamDTO();
mysqlDatasourceParamDTO.setHost("localhost");
mysqlDatasourceParamDTO.setDatabase("default");
mysqlDatasourceParamDTO.setUserName("root");
mysqlDatasourceParamDTO.setPort(3306);
mysqlDatasourceParamDTO.setPassword("123456");
ConnectionParam connectionParam = DatasourceUtil.buildConnectionParams(DbType.MYSQL, JSONUtils.toJsonString(mysqlDatasourceParamDTO));
Assert.assertNotNull(connectionParam);
}
@Test
public void testGetConnection() throws ClassNotFoundException, SQLException {
PowerMockito.mockStatic(Class.class);
PowerMockito.when(Class.forName(Mockito.any())).thenReturn(null);
PowerMockito.mockStatic(DriverManager.class);
PowerMockito.when(DriverManager.getConnection(Mockito.any(), Mockito.any(), Mockito.any())).thenReturn(null);
MysqlConnectionParam connectionParam = new MysqlConnectionParam();
connectionParam.setUser("root");
connectionParam.setPassword("123456");
Connection connection = DatasourceUtil.getConnection(DbType.MYSQL, connectionParam);
Assert.assertNull(connection);
}
@Test
public void testGetJdbcUrl() {
MysqlConnectionParam mysqlConnectionParam = new MysqlConnectionParam();
mysqlConnectionParam.setJdbcUrl("jdbc:mysql://localhost:3308");
String jdbcUrl = DatasourceUtil.getJdbcUrl(DbType.MYSQL, mysqlConnectionParam);
Assert.assertEquals("jdbc:mysql://localhost:3308?allowLoadLocalInfile=false&autoDeserialize=false&allowLocalInfile=false&allowUrlInLocalInfile=false",
jdbcUrl);
}
@Test
public void testBuildDatasourceParamDTO() {
MysqlConnectionParam connectionParam = new MysqlConnectionParam();
connectionParam.setJdbcUrl("jdbc:mysql://localhost:3308?allowLoadLocalInfile=false&autoDeserialize=false&allowLocalInfile=false&allowUrlInLocalInfile=false");
connectionParam.setAddress("jdbc:mysql://localhost:3308");
connectionParam.setUser("root");
connectionParam.setPassword("123456");
Assert.assertNotNull(DatasourceUtil.buildDatasourceParamDTO(DbType.MYSQL, JSONUtils.toJsonString(connectionParam)));
}
@Test
public void testGetDatasourceProcessor() {
Assert.assertNotNull(DatasourceUtil.getDatasourceProcessor(DbType.MYSQL));
Assert.assertNotNull(DatasourceUtil.getDatasourceProcessor(DbType.POSTGRESQL));
Assert.assertNotNull(DatasourceUtil.getDatasourceProcessor(DbType.HIVE));
Assert.assertNotNull(DatasourceUtil.getDatasourceProcessor(DbType.SPARK));
Assert.assertNotNull(DatasourceUtil.getDatasourceProcessor(DbType.CLICKHOUSE));
Assert.assertNotNull(DatasourceUtil.getDatasourceProcessor(DbType.ORACLE));
Assert.assertNotNull(DatasourceUtil.getDatasourceProcessor(DbType.SQLSERVER));
Assert.assertNotNull(DatasourceUtil.getDatasourceProcessor(DbType.DB2));
Assert.assertNotNull(DatasourceUtil.getDatasourceProcessor(DbType.PRESTO));
}
@Test(expected = Exception.class)
public void testGetDatasourceProcessorError() {
DatasourceUtil.getDatasourceProcessor(null);
}
}

83
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/datasource/clickhouse/ClickHouseDatasourceProcessorTest.java Normal file
View File

@ -0,0 +1,83 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.datasource.clickhouse;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.DbType;
import java.sql.DriverManager;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
@RunWith(PowerMockRunner.class)
@PrepareForTest({Class.class, DriverManager.class})
public class ClickHouseDatasourceProcessorTest {
private ClickHouseDatasourceProcessor clickHouseDatasourceProcessor = new ClickHouseDatasourceProcessor();
@Test
public void testCreateConnectionParams() {
ClickHouseDatasourceParamDTO clickhouseConnectionParam = new ClickHouseDatasourceParamDTO();
clickhouseConnectionParam.setUserName("user");
clickhouseConnectionParam.setPassword("password");
clickhouseConnectionParam.setHost("localhost");
clickhouseConnectionParam.setPort(8123);
clickhouseConnectionParam.setDatabase("default");
ClickhouseConnectionParam connectionParams = (ClickhouseConnectionParam) clickHouseDatasourceProcessor
.createConnectionParams(clickhouseConnectionParam);
Assert.assertNotNull(connectionParams);
Assert.assertEquals("jdbc:clickhouse://localhost:8123", connectionParams.getAddress());
Assert.assertEquals("jdbc:clickhouse://localhost:8123/default", connectionParams.getJdbcUrl());
}
@Test
public void testCreateConnectionParams2() {
String connectionParamJson = "{\"address\":\"jdbc:clickhouse://localhost:8123\",\"database\":\"default\","
+ "\"jdbcUrl\":\"jdbc:clickhouse://localhost:8123/default\",\"user\":\"default\",\"password\":\"123456\"}";
ClickhouseConnectionParam clickhouseConnectionParam = (ClickhouseConnectionParam) clickHouseDatasourceProcessor
.createConnectionParams(connectionParamJson);
Assert.assertNotNull(clickhouseConnectionParam);
Assert.assertEquals("default", clickhouseConnectionParam.getUser());
Assert.assertEquals("123456", clickhouseConnectionParam.getPassword());
}
@Test
public void testGetDatasourceDriver() {
Assert.assertNotNull(clickHouseDatasourceProcessor.getDatasourceDriver());
Assert.assertEquals(Constants.COM_CLICKHOUSE_JDBC_DRIVER, clickHouseDatasourceProcessor.getDatasourceDriver());
}
@Test
public void testGetJdbcUrl() {
ClickhouseConnectionParam connectionParam = new ClickhouseConnectionParam();
connectionParam.setUser("default");
connectionParam.setJdbcUrl("jdbc:clickhouse://localhost:8123/default");
connectionParam.setOther("other=other1");
String jdbcUrl = clickHouseDatasourceProcessor.getJdbcUrl(connectionParam);
Assert.assertEquals("jdbc:clickhouse://localhost:8123/default?other=other1", jdbcUrl);
}
@Test
public void testGetDbType() {
Assert.assertEquals(DbType.CLICKHOUSE, clickHouseDatasourceProcessor.getDbType());
}
}

83
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/datasource/db2/Db2DatasourceProcessorTest.java Normal file
View File

@ -0,0 +1,83 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.datasource.db2;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.DbType;
import java.sql.DriverManager;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
@RunWith(PowerMockRunner.class)
@PrepareForTest({Class.class, DriverManager.class})
public class Db2DatasourceProcessorTest {
private Db2DatasourceProcessor db2DatasourceProcessor = new Db2DatasourceProcessor();
@Test
public void testCreateConnectionParams() {
Db2DatasourceParamDTO db2DatasourceParamDTO = new Db2DatasourceParamDTO();
db2DatasourceParamDTO.setUserName("root");
db2DatasourceParamDTO.setPassword("123456");
db2DatasourceParamDTO.setHost("localhost");
db2DatasourceParamDTO.setPort(5142);
db2DatasourceParamDTO.setDatabase("default");
Db2ConnectionParam connectionParams = (Db2ConnectionParam) db2DatasourceProcessor
.createConnectionParams(db2DatasourceParamDTO);
Assert.assertNotNull(connectionParams);
Assert.assertEquals("jdbc:db2://localhost:5142", connectionParams.getAddress());
Assert.assertEquals("jdbc:db2://localhost:5142/default", connectionParams.getJdbcUrl());
}
@Test
public void testCreateConnectionParams2() {
String connectionJson = "{\"user\":\"root\",\"password\":\"123456\",\"address\":\"jdbc:db2://localhost:5142\""
+ ",\"database\":\"default\",\"jdbcUrl\":\"jdbc:db2://localhost:5142/default\"}";
Db2ConnectionParam connectionParams = (Db2ConnectionParam) db2DatasourceProcessor
.createConnectionParams(connectionJson);
Assert.assertNotNull(connectionJson);
Assert.assertEquals("root", connectionParams.getUser());
}
@Test
public void testGetDatasourceDriver() {
Assert.assertEquals(Constants.COM_DB2_JDBC_DRIVER, db2DatasourceProcessor.getDatasourceDriver());
}
@Test
public void testGetJdbcUrl() {
Db2ConnectionParam db2ConnectionParam = new Db2ConnectionParam();
db2ConnectionParam.setJdbcUrl("jdbc:db2://localhost:5142/default");
db2ConnectionParam.setOther("other=other");
String jdbcUrl = db2DatasourceProcessor.getJdbcUrl(db2ConnectionParam);
Assert.assertEquals("jdbc:db2://localhost:5142/default;other=other", jdbcUrl);
}
@Test
public void testGetDbType() {
Assert.assertEquals(DbType.DB2, db2DatasourceProcessor.getDbType());
}
}

79
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/datasource/hive/HiveDatasourceProcessorTest.java Normal file
View File

@ -0,0 +1,79 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.datasource.hive;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import java.sql.DriverManager;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
@RunWith(PowerMockRunner.class)
@PrepareForTest({Class.class, DriverManager.class})
public class HiveDatasourceProcessorTest {
private HiveDatasourceProcessor hiveDatasourceProcessor = new HiveDatasourceProcessor();
@Test
public void testCreateConnectionParams() {
HiveDataSourceParamDTO hiveDataSourceParamDTO = new HiveDataSourceParamDTO();
hiveDataSourceParamDTO.setHost("localhost1,localhost2");
hiveDataSourceParamDTO.setPort(5142);
hiveDataSourceParamDTO.setUserName("default");
hiveDataSourceParamDTO.setDatabase("default");
HiveConnectionParam connectionParams = (HiveConnectionParam) hiveDatasourceProcessor
.createConnectionParams(hiveDataSourceParamDTO);
System.out.println(JSONUtils.toJsonString(connectionParams));
Assert.assertNotNull(connectionParams);
Assert.assertEquals("jdbc:hive2://localhost1:5142,localhost2:5142", connectionParams.getAddress());
}
@Test
public void testCreateConnectionParams2() {
String connectionParam = "{\"user\":\"default\",\"address\":\"jdbc:hive2://localhost1:5142,localhost2:5142\""
+ ",\"jdbcUrl\":\"jdbc:hive2://localhost1:5142,localhost2:5142/default\"}";
HiveConnectionParam connectionParams = (HiveConnectionParam) hiveDatasourceProcessor
.createConnectionParams(connectionParam);
Assert.assertNotNull(connectionParam);
Assert.assertEquals("default", connectionParams.getUser());
}
@Test
public void testGetDatasourceDriver() {
Assert.assertEquals(Constants.ORG_APACHE_HIVE_JDBC_HIVE_DRIVER, hiveDatasourceProcessor.getDatasourceDriver());
}
@Test
public void testGetJdbcUrl() {
HiveConnectionParam connectionParam = new HiveConnectionParam();
connectionParam.setJdbcUrl("jdbc:hive2://localhost1:5142,localhost2:5142/default");
Assert.assertEquals("jdbc:hive2://localhost1:5142,localhost2:5142/default",
hiveDatasourceProcessor.getJdbcUrl(connectionParam));
}
@Test
public void testGetDbType() {
Assert.assertEquals(DbType.HIVE, hiveDatasourceProcessor.getDbType());
}
}

81
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/datasource/mysql/MysqlDatasourceProcessorTest.java Normal file
View File

@ -0,0 +1,81 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.datasource.mysql;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import java.sql.DriverManager;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
@RunWith(PowerMockRunner.class)
@PrepareForTest({Class.class, DriverManager.class})
public class MysqlDatasourceProcessorTest {
private MysqlDatasourceProcessor mysqlDatasourceProcessor = new MysqlDatasourceProcessor();
@Test
public void testCreateConnectionParams() {
MysqlDatasourceParamDTO mysqlDatasourceParamDTO = new MysqlDatasourceParamDTO();
mysqlDatasourceParamDTO.setUserName("root");
mysqlDatasourceParamDTO.setPassword("123456");
mysqlDatasourceParamDTO.setHost("localhost");
mysqlDatasourceParamDTO.setPort(3306);
mysqlDatasourceParamDTO.setDatabase("default");
MysqlConnectionParam connectionParams = (MysqlConnectionParam) mysqlDatasourceProcessor
.createConnectionParams(mysqlDatasourceParamDTO);
System.out.println(JSONUtils.toJsonString(connectionParams));
Assert.assertEquals("jdbc:mysql://localhost:3306", connectionParams.getAddress());
Assert.assertEquals("jdbc:mysql://localhost:3306/default", connectionParams.getJdbcUrl());
}
@Test
public void testCreateConnectionParams2() {
String connectionJson = "{\"user\":\"root\",\"password\":\"123456\",\"address\":\"jdbc:mysql://localhost:3306\""
+ ",\"database\":\"default\",\"jdbcUrl\":\"jdbc:mysql://localhost:3306/default\"}";
MysqlConnectionParam connectionParams = (MysqlConnectionParam) mysqlDatasourceProcessor
.createConnectionParams(connectionJson);
Assert.assertNotNull(connectionJson);
Assert.assertEquals("root", connectionParams.getUser());
}
@Test
public void testGetDatasourceDriver() {
Assert.assertEquals(Constants.COM_MYSQL_JDBC_DRIVER, mysqlDatasourceProcessor.getDatasourceDriver());
}
@Test
public void testGetJdbcUrl() {
MysqlConnectionParam mysqlConnectionParam = new MysqlConnectionParam();
mysqlConnectionParam.setJdbcUrl("jdbc:mysql://localhost:3306/default");
Assert.assertEquals("jdbc:mysql://localhost:3306/default?allowLoadLocalInfile=false&autoDeserialize=false&allowLocalInfile=false&allowUrlInLocalInfile=false",
mysqlDatasourceProcessor.getJdbcUrl(mysqlConnectionParam));
}
@Test
public void testGetDbType() {
Assert.assertEquals(DbType.MYSQL, mysqlDatasourceProcessor.getDbType());
}
}

83
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/datasource/oracle/OracleDatasourceProcessorTest.java Normal file
View File

@ -0,0 +1,83 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.datasource.oracle;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.DbConnectType;
import org.apache.dolphinscheduler.common.enums.DbType;
import java.sql.DriverManager;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
@RunWith(PowerMockRunner.class)
@PrepareForTest({Class.class, DriverManager.class})
public class OracleDatasourceProcessorTest {
private OracleDatasourceProcessor oracleDatasourceProcessor = new OracleDatasourceProcessor();
@Test
public void testCreateConnectionParams() {
OracleDatasourceParamDTO oracleDatasourceParamDTO = new OracleDatasourceParamDTO();
oracleDatasourceParamDTO.setConnectType(DbConnectType.ORACLE_SID);
oracleDatasourceParamDTO.setHost("localhost");
oracleDatasourceParamDTO.setPort(3308);
oracleDatasourceParamDTO.setUserName("root");
oracleDatasourceParamDTO.setPassword("123456");
oracleDatasourceParamDTO.setDatabase("default");
OracleConnectionParam connectionParams = (OracleConnectionParam) oracleDatasourceProcessor
.createConnectionParams(oracleDatasourceParamDTO);
Assert.assertNotNull(connectionParams);
Assert.assertEquals("jdbc:oracle:thin:@localhost:3308", connectionParams.getAddress());
Assert.assertEquals("jdbc:oracle:thin:@localhost:3308/default", connectionParams.getJdbcUrl());
}
@Test
public void testCreateConnectionParams2() {
String connectionJson = "{\"user\":\"root\",\"password\":\"123456\",\"address\":\"jdbc:oracle:thin:@localhost:3308\""
+ ",\"database\":\"default\",\"jdbcUrl\":\"jdbc:oracle:thin:@localhost:3308/default\",\"connectType\":\"ORACLE_SID\"}";
OracleConnectionParam connectionParams = (OracleConnectionParam) oracleDatasourceProcessor
.createConnectionParams(connectionJson);
Assert.assertNotNull(connectionParams);
Assert.assertEquals("root", connectionParams.getUser());
}
@Test
public void testGetDatasourceDriver() {
Assert.assertEquals(Constants.COM_ORACLE_JDBC_DRIVER, oracleDatasourceProcessor.getDatasourceDriver());
}
@Test
public void testGetJdbcUrl() {
OracleConnectionParam oracleConnectionParam = new OracleConnectionParam();
oracleConnectionParam.setJdbcUrl("jdbc:oracle:thin:@localhost:3308/default");
oracleConnectionParam.setOther("other=other");
Assert.assertEquals("jdbc:oracle:thin:@localhost:3308/default?other=other",
oracleDatasourceProcessor.getJdbcUrl(oracleConnectionParam));
}
@Test
public void getDbType() {
Assert.assertEquals(DbType.ORACLE, oracleDatasourceProcessor.getDbType());
}
}

83
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/datasource/postgresql/PostgreSqlDatasourceProcessorTest.java Normal file
View File

@ -0,0 +1,83 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.datasource.postgresql;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.DbType;
import java.sql.DriverManager;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
@RunWith(PowerMockRunner.class)
@PrepareForTest({Class.class, DriverManager.class})
public class PostgreSqlDatasourceProcessorTest {
private PostgreSqlDatasourceProcessor postgreSqlDatasourceProcessor = new PostgreSqlDatasourceProcessor();
@Test
public void testCreateConnectionParams() {
PostgreSqlDatasourceParamDTO postgreSqlDatasourceParamDTO = new PostgreSqlDatasourceParamDTO();
postgreSqlDatasourceParamDTO.setUserName("root");
postgreSqlDatasourceParamDTO.setPassword("123456");
postgreSqlDatasourceParamDTO.setHost("localhost");
postgreSqlDatasourceParamDTO.setPort(3308);
postgreSqlDatasourceParamDTO.setDatabase("default");
PostgreSqlConnectionParam connectionParams = (PostgreSqlConnectionParam) postgreSqlDatasourceProcessor
.createConnectionParams(postgreSqlDatasourceParamDTO);
Assert.assertEquals("jdbc:postgresql://localhost:3308", connectionParams.getAddress());
Assert.assertEquals("jdbc:postgresql://localhost:3308/default", connectionParams.getJdbcUrl());
Assert.assertEquals("root", connectionParams.getUser());
}
@Test
public void testCreateConnectionParams2() {
String connectionJson = "{\"user\":\"root\",\"password\":\"123456\",\"address\":\"jdbc:postgresql://localhost:3308\""
+ ",\"database\":\"default\",\"jdbcUrl\":\"jdbc:postgresql://localhost:3308/default\"}";
PostgreSqlConnectionParam connectionParams = (PostgreSqlConnectionParam) postgreSqlDatasourceProcessor
.createConnectionParams(connectionJson);
Assert.assertNotNull(connectionParams);
Assert.assertEquals("root", connectionParams.getUser());
}
@Test
public void testGetDatasourceDriver() {
Assert.assertEquals(Constants.ORG_POSTGRESQL_DRIVER, postgreSqlDatasourceProcessor.getDatasourceDriver());
}
@Test
public void testGetJdbcUrl() {
PostgreSqlConnectionParam postgreSqlConnectionParam = new PostgreSqlConnectionParam();
postgreSqlConnectionParam.setJdbcUrl("jdbc:postgresql://localhost:3308/default");
postgreSqlConnectionParam.setOther("other");
String jdbcUrl = postgreSqlDatasourceProcessor.getJdbcUrl(postgreSqlConnectionParam);
Assert.assertEquals("jdbc:postgresql://localhost:3308/default?other", jdbcUrl);
}
@Test
public void testGetDbType() {
Assert.assertEquals(DbType.POSTGRESQL, postgreSqlDatasourceProcessor.getDbType());
}
}

81
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/datasource/presto/PrestoDatasourceProcessorTest.java Normal file
View File

@ -0,0 +1,81 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.datasource.presto;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.DbType;
import java.sql.DriverManager;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
@RunWith(PowerMockRunner.class)
@PrepareForTest({Class.class, DriverManager.class})
public class PrestoDatasourceProcessorTest {
private PrestoDatasourceProcessor prestoDatasourceProcessor = new PrestoDatasourceProcessor();
@Test
public void testCreateConnectionParams() {
PrestoDatasourceParamDTO prestoDatasourceParamDTO = new PrestoDatasourceParamDTO();
prestoDatasourceParamDTO.setHost("localhost");
prestoDatasourceParamDTO.setPort(1234);
prestoDatasourceParamDTO.setDatabase("default");
prestoDatasourceParamDTO.setUserName("root");
prestoDatasourceParamDTO.setPassword("123456");
PrestoConnectionParam connectionParams = (PrestoConnectionParam) prestoDatasourceProcessor
.createConnectionParams(prestoDatasourceParamDTO);
Assert.assertEquals("jdbc:presto://localhost:1234", connectionParams.getAddress());
Assert.assertEquals("jdbc:presto://localhost:1234/default", connectionParams.getJdbcUrl());
}
@Test
public void testCreateConnectionParams2() {
String connectionJson = "{\"user\":\"root\",\"password\":\"123456\",\"address\":\"jdbc:presto://localhost:1234\""
+ ",\"database\":\"default\",\"jdbcUrl\":\"jdbc:presto://localhost:1234/default\"}";
PrestoConnectionParam connectionParams = (PrestoConnectionParam) prestoDatasourceProcessor
.createConnectionParams(connectionJson);
Assert.assertNotNull(connectionParams);
Assert.assertEquals("root", connectionParams.getUser());
}
@Test
public void testGetDatasourceDriver() {
Assert.assertEquals(Constants.COM_PRESTO_JDBC_DRIVER, prestoDatasourceProcessor.getDatasourceDriver());
}
@Test
public void testGetJdbcUrl() {
PrestoConnectionParam prestoConnectionParam = new PrestoConnectionParam();
prestoConnectionParam.setJdbcUrl("jdbc:postgresql://localhost:1234/default");
prestoConnectionParam.setOther("other");
Assert.assertEquals("jdbc:postgresql://localhost:1234/default?other",
prestoDatasourceProcessor.getJdbcUrl(prestoConnectionParam));
}
@Test
public void testGetDbType() {
Assert.assertEquals(DbType.PRESTO, prestoDatasourceProcessor.getDbType());
}
}

80
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/datasource/spark/SparkDatasourceProcessorTest.java Normal file
View File

@ -0,0 +1,80 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.datasource.spark;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.DbType;
import java.sql.DriverManager;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
@RunWith(PowerMockRunner.class)
@PrepareForTest({Class.class, DriverManager.class})
public class SparkDatasourceProcessorTest {
private SparkDatasourceProcessor sparkDatasourceProcessor = new SparkDatasourceProcessor();
@Test
public void testCreateConnectionParams() {
SparkDatasourceParamDTO sparkDatasourceParamDTO = new SparkDatasourceParamDTO();
sparkDatasourceParamDTO.setUserName("root");
sparkDatasourceParamDTO.setPassword("12345");
sparkDatasourceParamDTO.setHost("localhost1,localhost2");
sparkDatasourceParamDTO.setPort(1234);
sparkDatasourceParamDTO.setDatabase("default");
SparkConnectionParam connectionParams = (SparkConnectionParam) sparkDatasourceProcessor
.createConnectionParams(sparkDatasourceParamDTO);
Assert.assertEquals("jdbc:hive2://localhost1:1234,localhost2:1234", connectionParams.getAddress());
Assert.assertEquals("jdbc:hive2://localhost1:1234,localhost2:1234/default", connectionParams.getJdbcUrl());
}
@Test
public void testCreateConnectionParams2() {
String connectionJson = "{\"user\":\"root\",\"password\":\"12345\",\"address\":\"jdbc:hive2://localhost1:1234,localhost2:1234\""
+ ",\"database\":\"default\",\"jdbcUrl\":\"jdbc:hive2://localhost1:1234,localhost2:1234/default\"}";
SparkConnectionParam connectionParams = (SparkConnectionParam) sparkDatasourceProcessor
.createConnectionParams(connectionJson);
Assert.assertNotNull(connectionParams);
Assert.assertEquals("root", connectionParams.getUser());
}
@Test
public void testGetDatasourceDriver() {
Assert.assertEquals(Constants.ORG_APACHE_HIVE_JDBC_HIVE_DRIVER, sparkDatasourceProcessor.getDatasourceDriver());
}
@Test
public void testGetJdbcUrl() {
SparkConnectionParam sparkConnectionParam = new SparkConnectionParam();
sparkConnectionParam.setJdbcUrl("jdbc:hive2://localhost1:1234,localhost2:1234/default");
sparkConnectionParam.setOther("other");
Assert.assertEquals("jdbc:hive2://localhost1:1234,localhost2:1234/default;other",
sparkDatasourceProcessor.getJdbcUrl(sparkConnectionParam));
}
@Test
public void testGetDbType() {
Assert.assertEquals(DbType.SPARK, sparkDatasourceProcessor.getDbType());
}
}

Some files were not shown because too many files have changed in this diff Show More