diff --git a/.github/actions/comment-on-issue b/.github/actions/comment-on-issue new file mode 160000 index 0000000000..966bbbd211 --- /dev/null +++ b/.github/actions/comment-on-issue @@ -0,0 +1 @@ +Subproject commit 966bbbd2113de1f8008322fd1cc7674981a6b1ab diff --git a/.github/actions/lable-on-issue b/.github/actions/lable-on-issue new file mode 160000 index 0000000000..978f846c4c --- /dev/null +++ b/.github/actions/lable-on-issue @@ -0,0 +1 @@ +Subproject commit 978f846c4ca6299fd136f465b42c5e87aca28cac diff --git a/.github/actions/translation-on-issue b/.github/actions/translation-on-issue new file mode 160000 index 0000000000..4b50621337 --- /dev/null +++ b/.github/actions/translation-on-issue @@ -0,0 +1 @@ +Subproject commit 4b5062133773227695d089319f0374ff0868c9fb diff --git a/.github/workflows/issue_robot.yml b/.github/workflows/issue_robot.yml index cf940c15b5..9ed33065aa 100644 --- a/.github/workflows/issue_robot.yml +++ b/.github/workflows/issue_robot.yml @@ -22,24 +22,29 @@ on: types: [opened] jobs: - comment: + issueRobot: runs-on: ubuntu-latest steps: - - uses: ben-z/actions-comment-on-issue@1.0.2 - with: - message: "Hi:\n* Thank you for your feedback, we have received your issue, Please wait patiently for a reply.\n* In order to let us know your request as soon as possible, please provide detailed information、version or pictures.\n* If you haven't received a reply for a long time, you can subscribe to the developer's email，Mail subscription steps reference https://dolphinscheduler.apache.org/zh-cn/community/development/subscribe.html ,then send questions to dev@dolphinscheduler.apache.org." - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: "Checkout ${{ github.ref }}" + uses: actions/checkout@v2 + with: + persist-credentials: false + submodules: true - autoTranslate: - runs-on: ubuntu-latest - steps: - - uses: actions-cool/translation-helper@v1.1.1 + - name: "Translation into English in issue" + uses: ./.github/actions/translation-on-issue + with: + translate-title: true + translate-body: false - automate-issues-labels: - runs-on: ubuntu-latest - steps: - - name: initial labeling - uses: andymckay/labeler@master + - name: "Comment in issue" + uses: ./.github/actions/comment-on-issue + with: + message: "Hi:\n* Thank you for your feedback, we have received your issue, Please wait patiently for a reply.\n* In order for us to understand your request as soon as possible, please provide detailed information、version or pictures.\n* If you haven't received a reply for a long time, you can subscribe to the developer's email，Mail subscription steps reference https://dolphinscheduler.apache.org/zh-cn/community/development/subscribe.html ,Then write the issue URL in the email content and send question to dev@dolphinscheduler.apache.org." + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: "Add lable in issue" + uses: ./.github/actions/lable-on-issue with: add-labels: "Waiting for reply" ignore-if-labeled: true diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000000..6a7832aa8a --- /dev/null +++ b/.gitmodules @@ -0,0 +1,26 @@ +# Licensed to Apache Software Foundation (ASF) under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Apache Software Foundation (ASF) licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[submodule ".github/actions/comment-on-issue"] + path = .github/actions/comment-on-issue + url = https://github.com/xingchun-chen/actions-comment-on-issue +[submodule ".github/actions/lable-on-issue"] + path = .github/actions/lable-on-issue + url = https://github.com/xingchun-chen/labeler +[submodule ".github/actions/translation-on-issue"] + path = .github/actions/translation-on-issue + url = https://github.com/xingchun-chen/translation-helper diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f26b06e850..5b372e4863 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -14,12 +14,12 @@ There are three branches in the remote repository currently: So, you should fork the `dev` branch. -After forking the [dolphinscheduler upstream source repository](https://github.com/apache/incubator-dolphinscheduler/fork) to your personal repository, you can set your personal development environment. +After forking the [dolphinscheduler upstream source repository](https://github.com/apache/dolphinscheduler/fork) to your personal repository, you can set your personal development environment. ```sh $ cd $ git clone < your personal forked dolphinscheduler repo> -$ cd incubator-dolphinscheduler +$ cd dolphinscheduler ``` ## Set git remote as ``upstream`` @@ -27,7 +27,7 @@ $ cd incubator-dolphinscheduler Add remote repository address, named upstream ```sh -git remote add upstream https://github.com/apache/incubator-dolphinscheduler.git +git remote add upstream https://github.com/apache/dolphinscheduler.git ``` View repository: diff --git a/DISCLAIMER b/DISCLAIMER deleted file mode 100644 index 7abee7c705..0000000000 --- a/DISCLAIMER +++ /dev/null @@ -1,5 +0,0 @@ -Apache DolphinScheduler is an effort undergoing incubation at The Apache Software Foundation (ASF), sponsored by the Apache Incubator. -Incubation is required of all newly accepted projects until a further review indicates that the infrastructure, -communications, and decision making process have stabilized in a manner consistent with other successful ASF projects. -While incubation status is not necessarily a reflection of the completeness or stability of the code, -it does indicate that the project has yet to be fully endorsed by the ASF. diff --git a/LICENSE b/LICENSE index 5d7c9f4cf7..8c9beb882f 100644 --- a/LICENSE +++ b/LICENSE @@ -200,9 +200,9 @@ See the License for the specific language governing permissions and limitations under the License. ======================================================================= -Apache DolphinScheduler (incubating) Subcomponents: +Apache DolphinScheduler Subcomponents: -The Apache DolphinScheduler (incubating) project contains subcomponents with separate copyright +The Apache DolphinScheduler project contains subcomponents with separate copyright notices and license terms. Your use of the source code for the these subcomponents is subject to the terms and conditions of the following licenses. diff --git a/NOTICE b/NOTICE index 702a09cc63..b07c3dd4ca 100644 --- a/NOTICE +++ b/NOTICE @@ -1,4 +1,4 @@ -Apache DolphinScheduler (incubating) +Apache DolphinScheduler Copyright 2019-2021 The Apache Software Foundation This product includes software developed at diff --git a/README.md b/README.md index e4be878d3c..5c982c283e 100644 --- a/README.md +++ b/README.md @@ -3,15 +3,15 @@ Dolphin Scheduler Official Website ============ [![License](https://img.shields.io/badge/license-Apache%202-4EB1BA.svg)](https://www.apache.org/licenses/LICENSE-2.0.html) -[![Total Lines](https://tokei.rs/b1/github/apache/Incubator-DolphinScheduler?category=lines)](https://github.com/apache/Incubator-DolphinScheduler) -[![codecov](https://codecov.io/gh/apache/incubator-dolphinscheduler/branch/dev/graph/badge.svg)](https://codecov.io/gh/apache/incubator-dolphinscheduler/branch/dev) +[![Total Lines](https://tokei.rs/b1/github/apache/dolphinscheduler?category=lines)](https://github.com/apache/dolphinscheduler) +[![codecov](https://codecov.io/gh/apache/dolphinscheduler/branch/dev/graph/badge.svg)](https://codecov.io/gh/apache/dolphinscheduler/branch/dev) [![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=apache-dolphinscheduler&metric=alert_status)](https://sonarcloud.io/dashboard?id=apache-dolphinscheduler) [![Twitter Follow](https://img.shields.io/twitter/follow/dolphinschedule.svg?style=social&label=Follow)](https://twitter.com/dolphinschedule) [![Slack Status](https://img.shields.io/badge/slack-join_chat-white.svg?logo=slack&style=social)](https://join.slack.com/t/asf-dolphinscheduler/shared_invite/zt-omtdhuio-_JISsxYhiVsltmC5h38yfw) -[![Stargazers over time](https://starchart.cc/apache/incubator-dolphinscheduler.svg)](https://starchart.cc/apache/incubator-dolphinscheduler) +[![Stargazers over time](https://starchart.cc/apache/dolphinscheduler.svg)](https://starchart.cc/apache/dolphinscheduler) [![EN doc](https://img.shields.io/badge/document-English-blue.svg)](README.md) [![CN doc](https://img.shields.io/badge/文档-中文版-blue.svg)](README_zh_CN.md) @@ -73,8 +73,8 @@ Please referer the official website document: [QuickStart in Kubernetes](https:/ Artifact: ``` -dolphinscheduler-dist/target/apache-dolphinscheduler-incubating-${latest.release.version}-dolphinscheduler-bin.tar.gz: Binary package of DolphinScheduler -dolphinscheduler-dist/target/apache-dolphinscheduler-incubating-${latest.release.version}-src.zip: Source code package of DolphinScheduler +dolphinscheduler-dist/target/apache-dolphinscheduler-${latest.release.version}-bin.tar.gz: Binary package of DolphinScheduler +dolphinscheduler-dist/target/apache-dolphinscheduler-${latest.release.version}-src.tar.gz: Source code package of DolphinScheduler ``` ## Thanks @@ -84,7 +84,7 @@ We would like to express our deep gratitude to all the open-source projects used ## Get Help -1. Submit an [issue](https://github.com/apache/incubator-dolphinscheduler/issues/new/choose) +1. Submit an [issue](https://github.com/apache/dolphinscheduler/issues/new/choose) 1. Subscribe to this mailing list: https://dolphinscheduler.apache.org/en-us/community/development/subscribe.html, then email dev@dolphinscheduler.apache.org ## Community @@ -99,4 +99,4 @@ The community welcomes everyone to contribute, please refer to this page to find ## License -Please refer to the [LICENSE](https://github.com/apache/incubator-dolphinscheduler/blob/dev/LICENSE) file. +Please refer to the [LICENSE](https://github.com/apache/dolphinscheduler/blob/dev/LICENSE) file. diff --git a/README_zh_CN.md b/README_zh_CN.md index 502bff55be..39c0892eaa 100644 --- a/README_zh_CN.md +++ b/README_zh_CN.md @@ -3,12 +3,12 @@ Dolphin Scheduler Official Website ============ [![License](https://img.shields.io/badge/license-Apache%202-4EB1BA.svg)](https://www.apache.org/licenses/LICENSE-2.0.html) -[![Total Lines](https://tokei.rs/b1/github/apache/Incubator-DolphinScheduler?category=lines)](https://github.com/apache/Incubator-DolphinScheduler) -[![codecov](https://codecov.io/gh/apache/incubator-dolphinscheduler/branch/dev/graph/badge.svg)](https://codecov.io/gh/apache/incubator-dolphinscheduler/branch/dev) +[![Total Lines](https://tokei.rs/b1/github/apache/dolphinscheduler?category=lines)](https://github.com/apache/dolphinscheduler) +[![codecov](https://codecov.io/gh/apache/dolphinscheduler/branch/dev/graph/badge.svg)](https://codecov.io/gh/apache/dolphinscheduler/branch/dev) [![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=apache-dolphinscheduler&metric=alert_status)](https://sonarcloud.io/dashboard?id=apache-dolphinscheduler) -[![Stargazers over time](https://starchart.cc/apache/incubator-dolphinscheduler.svg)](https://starchart.cc/apache/incubator-dolphinscheduler) +[![Stargazers over time](https://starchart.cc/apache/dolphinscheduler.svg)](https://starchart.cc/apache/dolphinscheduler) [![CN doc](https://img.shields.io/badge/文档-中文版-blue.svg)](README_zh_CN.md) [![EN doc](https://img.shields.io/badge/document-English-blue.svg)](README.md) @@ -52,7 +52,7 @@ Dolphin Scheduler Official Website ## 近期研发计划 -DolphinScheduler的工作计划：研发计划 ，其中 In Develop卡片下是正在研发的功能，TODO卡片是待做事项(包括 feature ideas) +DolphinScheduler的工作计划：研发计划 ，其中 In Develop卡片下是正在研发的功能，TODO卡片是待做事项(包括 feature ideas) ## 参与贡献 @@ -76,8 +76,8 @@ DolphinScheduler的工作计划： customizeTenantQpsRate; + + public boolean isTrafficGlobalControlSwitch() { + return trafficGlobalControlSwitch; + } + + public void setTrafficGlobalControlSwitch(boolean trafficGlobalControlSwitch) { + this.trafficGlobalControlSwitch = trafficGlobalControlSwitch; + } + + public Integer getMaxGlobalQpsRate() { + return maxGlobalQpsRate; + } + + public void setMaxGlobalQpsRate(Integer maxGlobalQpsRate) { + this.maxGlobalQpsRate = maxGlobalQpsRate; + } + + public boolean isTrafficTenantControlSwitch() { + return trafficTenantControlSwitch; + } + + public void setTrafficTenantControlSwitch(boolean trafficTenantControlSwitch) { + this.trafficTenantControlSwitch = trafficTenantControlSwitch; + } + + public Integer getDefaultTenantQpsRate() { + return defaultTenantQpsRate; + } + + public void setDefaultTenantQpsRate(Integer defaultTenantQpsRate) { + this.defaultTenantQpsRate = defaultTenantQpsRate; + } + + public Map getCustomizeTenantQpsRate() { + return customizeTenantQpsRate; + } + + public void setCustomizeTenantQpsRate(Map customizeTenantQpsRate) { + this.customizeTenantQpsRate = customizeTenantQpsRate; + } +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataSourceController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataSourceController.java index 05f78f8db2..98951f26a7 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataSourceController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataSourceController.java @@ -31,9 +31,12 @@ import static org.apache.dolphinscheduler.api.enums.Status.VERIFY_DATASOURCE_NAM import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.exceptions.ApiException; import org.apache.dolphinscheduler.api.service.DataSourceService; +import org.apache.dolphinscheduler.api.utils.RegexUtils; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.common.enums.DbConnectType; +import org.apache.dolphinscheduler.common.datasource.BaseDataSourceParamDTO; +import org.apache.dolphinscheduler.common.datasource.ConnectionParam; +import org.apache.dolphinscheduler.common.datasource.DatasourceUtil; import org.apache.dolphinscheduler.common.enums.DbType; import org.apache.dolphinscheduler.common.utils.CommonUtils; import org.apache.dolphinscheduler.common.utils.ParameterUtils; @@ -48,6 +51,7 @@ import org.springframework.http.HttpStatus; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestAttribute; +import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.ResponseStatus; @@ -57,6 +61,7 @@ import io.swagger.annotations.Api; import io.swagger.annotations.ApiImplicitParam; import io.swagger.annotations.ApiImplicitParams; import io.swagger.annotations.ApiOperation; +import io.swagger.annotations.ApiParam; import springfox.documentation.annotations.ApiIgnore; /** @@ -76,58 +81,19 @@ public class DataSourceController extends BaseController { * create data source * * @param loginUser login user - * @param name data source name - * @param note data source description - * @param type data source type - * @param host host - * @param port port - * @param database data base - * @param principal principal - * @param userName user name - * @param password password - * @param other other arguments + * @param dataSourceParam datasource param * @return create result code */ @ApiOperation(value = "createDataSource", notes = "CREATE_DATA_SOURCE_NOTES") - @ApiImplicitParams({ - @ApiImplicitParam(name = "name", value = "DATA_SOURCE_NAME", required = true, dataType = "String"), - @ApiImplicitParam(name = "note", value = "DATA_SOURCE_NOTE", dataType = "String"), - @ApiImplicitParam(name = "type", value = "DB_TYPE", required = true, dataType = "DbType"), - @ApiImplicitParam(name = "host", value = "DATA_SOURCE_HOST", required = true, dataType = "String"), - @ApiImplicitParam(name = "port", value = "DATA_SOURCE_PORT", required = true, dataType = "String"), - @ApiImplicitParam(name = "database", value = "DATABASE_NAME", required = true, dataType = "String"), - @ApiImplicitParam(name = "principal", value = "DATA_SOURCE_PRINCIPAL", dataType = "String"), - @ApiImplicitParam(name = "userName", value = "USER_NAME", required = true, dataType = "String"), - @ApiImplicitParam(name = "password", value = "PASSWORD", dataType = "String"), - @ApiImplicitParam(name = "connectType", value = "CONNECT_TYPE", dataType = "DbConnectType"), - @ApiImplicitParam(name = "other", value = "DATA_SOURCE_OTHER", dataType = "String"), - @ApiImplicitParam(name = "javaSecurityKrb5Conf", value = "DATA_SOURCE_KERBEROS_KRB5_CONF", dataType = "String"), - @ApiImplicitParam(name = "loginUserKeytabUsername", value = "DATA_SOURCE_KERBEROS_KEYTAB_USERNAME", dataType = "String"), - @ApiImplicitParam(name = "loginUserKeytabPath", value = "DATA_SOURCE_KERBEROS_KEYTAB_PATH", dataType = "String") - }) @PostMapping(value = "/create") @ResponseStatus(HttpStatus.CREATED) @ApiException(CREATE_DATASOURCE_ERROR) public Result createDataSource(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser, - @RequestParam("name") String name, - @RequestParam(value = "note", required = false) String note, - @RequestParam(value = "type") DbType type, - @RequestParam(value = "host") String host, - @RequestParam(value = "port") String port, - @RequestParam(value = "database") String database, - @RequestParam(value = "principal") String principal, - @RequestParam(value = "userName") String userName, - @RequestParam(value = "password") String password, - @RequestParam(value = "connectType") DbConnectType connectType, - @RequestParam(value = "other") String other, - @RequestParam(value = "javaSecurityKrb5Conf", required = false) String javaSecurityKrb5Conf, - @RequestParam(value = "loginUserKeytabUsername", required = false) String loginUserKeytabUsername, - @RequestParam(value = "loginUserKeytabPath", required = false) String loginUserKeytabPath) { - logger.info("login user {} create datasource name: {}, note: {}, type: {}, host: {}, port: {}, database : {}, principal: {}, userName : {}, connectType: {}, other: {}", - loginUser.getUserName(), name, note, type, host, port, database, principal, userName, connectType, other); - String parameter = dataSourceService.buildParameter(type, host, port, database, principal, userName, password, connectType, other, - javaSecurityKrb5Conf, loginUserKeytabUsername, loginUserKeytabPath); - return dataSourceService.createDataSource(loginUser, name, note, type, parameter); + @ApiParam(name = "DATA_SOURCE_PARAM", required = true) + @RequestBody BaseDataSourceParamDTO dataSourceParam) { + String userName = RegexUtils.escapeNRT(loginUser.getUserName()); + logger.info("login user {} create datasource : {}", userName, dataSourceParam); + return dataSourceService.createDataSource(loginUser, dataSourceParam); } @@ -135,61 +101,21 @@ public class DataSourceController extends BaseController { * updateProcessInstance data source * * @param loginUser login user - * @param name data source name - * @param note description - * @param type data source type - * @param other other arguments - * @param id data source di - * @param host host - * @param port port - * @param database database - * @param principal principal - * @param userName user name - * @param password password + * @param dataSourceParam datasource param * @return update result code */ @ApiOperation(value = "updateDataSource", notes = "UPDATE_DATA_SOURCE_NOTES") @ApiImplicitParams({ @ApiImplicitParam(name = "id", value = "DATA_SOURCE_ID", required = true, dataType = "Int", example = "100"), - @ApiImplicitParam(name = "name", value = "DATA_SOURCE_NAME", required = true, dataType = "String"), - @ApiImplicitParam(name = "note", value = "DATA_SOURCE_NOTE", dataType = "String"), - @ApiImplicitParam(name = "type", value = "DB_TYPE", required = true, dataType = "DbType"), - @ApiImplicitParam(name = "host", value = "DATA_SOURCE_HOST", required = true, dataType = "String"), - @ApiImplicitParam(name = "port", value = "DATA_SOURCE_PORT", required = true, dataType = "String"), - @ApiImplicitParam(name = "database", value = "DATABASE_NAME", required = true, dataType = "String"), - @ApiImplicitParam(name = "principal", value = "DATA_SOURCE_PRINCIPAL", dataType = "String"), - @ApiImplicitParam(name = "userName", value = "USER_NAME", required = true, dataType = "String"), - @ApiImplicitParam(name = "password", value = "PASSWORD", dataType = "String"), - @ApiImplicitParam(name = "connectType", value = "CONNECT_TYPE", dataType = "DbConnectType"), - @ApiImplicitParam(name = "other", value = "DATA_SOURCE_OTHER", dataType = "String"), - @ApiImplicitParam(name = "javaSecurityKrb5Conf", value = "DATA_SOURCE_KERBEROS_KRB5_CONF", dataType = "String"), - @ApiImplicitParam(name = "loginUserKeytabUsername", value = "DATA_SOURCE_KERBEROS_KEYTAB_USERNAME", dataType = "String"), - @ApiImplicitParam(name = "loginUserKeytabPath", value = "DATA_SOURCE_KERBEROS_KEYTAB_PATH", dataType = "String") }) @PostMapping(value = "/update") @ResponseStatus(HttpStatus.OK) @ApiException(UPDATE_DATASOURCE_ERROR) public Result updateDataSource(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser, - @RequestParam("id") int id, - @RequestParam("name") String name, - @RequestParam(value = "note", required = false) String note, - @RequestParam(value = "type") DbType type, - @RequestParam(value = "host") String host, - @RequestParam(value = "port") String port, - @RequestParam(value = "database") String database, - @RequestParam(value = "principal") String principal, - @RequestParam(value = "userName") String userName, - @RequestParam(value = "password") String password, - @RequestParam(value = "connectType") DbConnectType connectType, - @RequestParam(value = "other") String other, - @RequestParam(value = "javaSecurityKrb5Conf", required = false) String javaSecurityKrb5Conf, - @RequestParam(value = "loginUserKeytabUsername", required = false) String loginUserKeytabUsername, - @RequestParam(value = "loginUserKeytabPath", required = false) String loginUserKeytabPath) { - logger.info("login user {} updateProcessInstance datasource name: {}, note: {}, type: {}, connectType: {}, other: {}", - loginUser.getUserName(), name, note, type, connectType, other); - String parameter = dataSourceService.buildParameter(type, host, port, database, principal, userName, password, connectType, other, - javaSecurityKrb5Conf, loginUserKeytabUsername, loginUserKeytabPath); - return dataSourceService.updateDataSource(id, loginUser, name, note, type, parameter); + @RequestBody BaseDataSourceParamDTO dataSourceParam) { + String userName = RegexUtils.escapeNRT(loginUser.getUserName()); + logger.info("login user {} updateProcessInstance datasource : {}", userName, dataSourceParam); + return dataSourceService.updateDataSource(dataSourceParam.getId(), loginUser, dataSourceParam); } /** @@ -270,58 +196,23 @@ public class DataSourceController extends BaseController { * connect datasource * * @param loginUser login user - * @param name data source name - * @param note data soruce description - * @param type data source type - * @param other other parameters - * @param host host - * @param port port - * @param database data base - * @param principal principal - * @param userName user name - * @param password password + * @param dataSourceParam datasource param * @return connect result code */ @ApiOperation(value = "connectDataSource", notes = "CONNECT_DATA_SOURCE_NOTES") @ApiImplicitParams({ @ApiImplicitParam(name = "name", value = "DATA_SOURCE_NAME", required = true, dataType = "String"), - @ApiImplicitParam(name = "note", value = "DATA_SOURCE_NOTE", dataType = "String"), - @ApiImplicitParam(name = "type", value = "DB_TYPE", required = true, dataType = "DbType"), - @ApiImplicitParam(name = "host", value = "DATA_SOURCE_HOST", required = true, dataType = "String"), - @ApiImplicitParam(name = "port", value = "DATA_SOURCE_PORT", required = true, dataType = "String"), - @ApiImplicitParam(name = "database", value = "DATABASE_NAME", required = true, dataType = "String"), - @ApiImplicitParam(name = "principal", value = "DATA_SOURCE_PRINCIPAL", dataType = "String"), - @ApiImplicitParam(name = "userName", value = "USER_NAME", required = true, dataType = "String"), - @ApiImplicitParam(name = "password", value = "PASSWORD", dataType = "String"), - @ApiImplicitParam(name = "connectType", value = "CONNECT_TYPE", dataType = "DbConnectType"), - @ApiImplicitParam(name = "other", value = "DATA_SOURCE_OTHER", dataType = "String"), - @ApiImplicitParam(name = "javaSecurityKrb5Conf", value = "DATA_SOURCE_KERBEROS_KRB5_CONF", dataType = "String"), - @ApiImplicitParam(name = "loginUserKeytabUsername", value = "DATA_SOURCE_KERBEROS_KEYTAB_USERNAME", dataType = "String"), - @ApiImplicitParam(name = "loginUserKeytabPath", value = "DATA_SOURCE_KERBEROS_KEYTAB_PATH", dataType = "String") }) @PostMapping(value = "/connect") @ResponseStatus(HttpStatus.OK) @ApiException(CONNECT_DATASOURCE_FAILURE) public Result connectDataSource(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser, - @RequestParam("name") String name, - @RequestParam(value = "note", required = false) String note, - @RequestParam(value = "type") DbType type, - @RequestParam(value = "host") String host, - @RequestParam(value = "port") String port, - @RequestParam(value = "database") String database, - @RequestParam(value = "principal") String principal, - @RequestParam(value = "userName") String userName, - @RequestParam(value = "password") String password, - @RequestParam(value = "connectType") DbConnectType connectType, - @RequestParam(value = "other") String other, - @RequestParam(value = "javaSecurityKrb5Conf", required = false) String javaSecurityKrb5Conf, - @RequestParam(value = "loginUserKeytabUsername", required = false) String loginUserKeytabUsername, - @RequestParam(value = "loginUserKeytabPath", required = false) String loginUserKeytabPath) { - logger.info("login user {}, connect datasource: {}, note: {}, type: {}, connectType: {}, other: {}", - loginUser.getUserName(), name, note, type, connectType, other); - String parameter = dataSourceService.buildParameter(type, host, port, database, principal, userName, password, connectType, other, - javaSecurityKrb5Conf, loginUserKeytabUsername, loginUserKeytabPath); - return dataSourceService.checkConnection(type, parameter); + @RequestBody BaseDataSourceParamDTO dataSourceParam) { + String userName = RegexUtils.escapeNRT(loginUser.getUserName()); + logger.info("login user {}, connect datasource: {}", userName, dataSourceParam); + DatasourceUtil.checkDatasourceParam(dataSourceParam); + ConnectionParam connectionParams = DatasourceUtil.buildConnectionParams(dataSourceParam); + return dataSourceService.checkConnection(dataSourceParam.getType(), connectionParams); } /** diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessDefinitionController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessDefinitionController.java index a003317e4f..0ccdca15cb 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessDefinitionController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessDefinitionController.java @@ -122,7 +122,13 @@ public class ProcessDefinitionController extends BaseController { logger.info("login user {}, create process definition, project name: {}, process definition name: {}, " + "process_definition_json: {}, desc: {} locations:{}, connects:{}", - loginUser.getUserName(), projectName, name, json, description, locations, connects); + RegexUtils.escapeNRT(loginUser.getUserName()), + RegexUtils.escapeNRT(projectName), + RegexUtils.escapeNRT(name), + RegexUtils.escapeNRT(json), + RegexUtils.escapeNRT(description), + RegexUtils.escapeNRT(locations), + RegexUtils.escapeNRT(connects)); Map result = processDefinitionService.createProcessDefinition(loginUser, projectName, name, json, description, locations, connects); return returnDataList(result); @@ -209,7 +215,9 @@ public class ProcessDefinitionController extends BaseController { @ApiParam(name = "projectName", value = "PROJECT_NAME", required = true) @PathVariable String projectName, @RequestParam(value = "name", required = true) String name) { logger.info("verify process definition name unique, user:{}, project name:{}, process definition name:{}", - loginUser.getUserName(), projectName, name); + RegexUtils.escapeNRT(loginUser.getUserName()), + RegexUtils.escapeNRT(projectName), + RegexUtils.escapeNRT(name)); Map result = processDefinitionService.verifyProcessDefinitionName(loginUser, projectName, name); return returnDataList(result); } @@ -251,9 +259,16 @@ public class ProcessDefinitionController extends BaseController { @RequestParam(value = "description", required = false) String description, @RequestParam(value = "releaseState", required = false, defaultValue = "OFFLINE") ReleaseState releaseState) { - logger.info("login user {}, update process define, project name: {}, process define name: {}, " + logger.info("login user {}, update process define, project name: {}, process define name: {}," + "process_definition_json: {}, desc: {}, locations:{}, connects:{}", - loginUser.getUserName(), projectName, name, processDefinitionJson, description, locations, connects); + RegexUtils.escapeNRT(loginUser.getUserName()), + RegexUtils.escapeNRT(projectName), + RegexUtils.escapeNRT(name), + RegexUtils.escapeNRT(processDefinitionJson), + RegexUtils.escapeNRT(description), + RegexUtils.escapeNRT(locations), + RegexUtils.escapeNRT(connects)); + Map result = processDefinitionService.updateProcessDefinition(loginUser, projectName, id, name, processDefinitionJson, description, locations, connects); // If the update fails, the result will be returned directly @@ -294,9 +309,21 @@ public class ProcessDefinitionController extends BaseController { @RequestParam(value = "processDefinitionCode") long processDefinitionCode) { logger.info("login user {}, query process versions, project name: {}, pageNo: {}, pageSize: {}, processDefinitionCode: {}", loginUser.getUserName(), projectName, pageNo, pageSize, processDefinitionCode); - Map result = processDefinitionService.queryProcessDefinitionVersions(loginUser - , projectName, pageNo, pageSize, processDefinitionCode); + + logger.info("query process definition versions, login user {}, project name: {}, process define id: {}, list paging, pageNo: {}, pageSize: {}", + RegexUtils.escapeNRT(loginUser.getUserName()), + RegexUtils.escapeNRT(projectName), + processDefinitionCode, + pageNo, + pageSize); + + Map result = checkPageParams(pageNo, pageSize); + if (result.get(Constants.STATUS) != Status.SUCCESS) { + return returnDataListPaging(result); + } + result = processDefinitionService.queryProcessDefinitionVersions(loginUser + , projectName, pageNo, pageSize, processDefinitionCode); return returnDataList(result); } @@ -321,8 +348,11 @@ public class ProcessDefinitionController extends BaseController { @ApiParam(name = "projectName", value = "PROJECT_NAME", required = true) @PathVariable String projectName, @RequestParam(value = "processDefinitionId") int processDefinitionId, @RequestParam(value = "version") long version) { - logger.info("login user {}, switch process version, project name: {}, processDefinitionId: {}, version: {}", - loginUser.getUserName(), projectName, processDefinitionId, version); + logger.info("switch certain process definition version, login user {}, project name:{}, process definition id:{}, version:{}", + RegexUtils.escapeNRT(loginUser.getUserName()), + RegexUtils.escapeNRT(projectName), + processDefinitionId, + version); Map result = processDefinitionService.switchProcessDefinitionVersion(loginUser, projectName , processDefinitionId, version); return returnDataList(result); @@ -349,8 +379,11 @@ public class ProcessDefinitionController extends BaseController { @ApiParam(name = "projectName", value = "PROJECT_NAME", required = true) @PathVariable String projectName, @RequestParam(value = "processDefinitionId") int processDefinitionId, @RequestParam(value = "version") long version) { - logger.info("login user {}, delete process definition, project name: {}, processDefinitionId: {}, version: {}", - loginUser.getUserName(), projectName, processDefinitionId, version); + logger.info("delete the certain process definition version by version and process definition id, login user {}, project name:{}, process definition id:{}, version:{}", + RegexUtils.escapeNRT(loginUser.getUserName()), + RegexUtils.escapeNRT(projectName), + processDefinitionId, + version); Map result = processDefinitionService.deleteByProcessDefinitionIdAndVersion(loginUser, projectName, processDefinitionId, version); return returnDataList(result); } @@ -379,13 +412,15 @@ public class ProcessDefinitionController extends BaseController { @RequestParam(value = "releaseState", required = true) ReleaseState releaseState) { logger.info("login user {}, release process definition, project name: {}, release state: {}", - loginUser.getUserName(), projectName, releaseState); + RegexUtils.escapeNRT(loginUser.getUserName()), + RegexUtils.escapeNRT(projectName), + releaseState); Map result = processDefinitionService.releaseProcessDefinition(loginUser, projectName, processId, releaseState); return returnDataList(result); } /** - * query datail of process definition by id + * query detail of process definition by id * * @param loginUser login user * @param projectName project name @@ -401,16 +436,18 @@ public class ProcessDefinitionController extends BaseController { @ApiException(QUERY_DATAIL_OF_PROCESS_DEFINITION_ERROR) public Result queryProcessDefinitionById(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @ApiParam(name = "projectName", value = "PROJECT_NAME", required = true) @PathVariable String projectName, - @RequestParam("processId") Integer processId - ) { + @RequestParam("processId") Integer processId) { + logger.info("query detail of process definition, login user:{}, project name:{}, process definition id:{}", - loginUser.getUserName(), projectName, processId); + RegexUtils.escapeNRT(loginUser.getUserName()), + RegexUtils.escapeNRT(projectName), + processId); Map result = processDefinitionService.queryProcessDefinitionById(loginUser, projectName, processId); return returnDataList(result); } /** - * query datail of process definition by name + * query detail of process definition by name * * @param loginUser login user * @param projectName project name @@ -426,8 +463,12 @@ public class ProcessDefinitionController extends BaseController { @ApiException(QUERY_DATAIL_OF_PROCESS_DEFINITION_ERROR) public Result queryProcessDefinitionByName(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @ApiParam(name = "projectName", value = "PROJECT_NAME", required = true) @PathVariable String projectName, - @RequestParam("processDefinitionName") String processDefinitionName - ) { + @RequestParam("processDefinitionName") String processDefinitionName) { + + logger.info("query detail of process definition by name, login user:{}, project name:{}, process definition id:{}", + RegexUtils.escapeNRT(loginUser.getUserName()), + RegexUtils.escapeNRT(projectName), + RegexUtils.escapeNRT(processDefinitionName)); Map result = processDefinitionService.queryProcessDefinitionByName(loginUser, projectName, processDefinitionName); return returnDataList(result); } @@ -444,10 +485,11 @@ public class ProcessDefinitionController extends BaseController { @ResponseStatus(HttpStatus.OK) @ApiException(QUERY_PROCESS_DEFINITION_LIST) public Result queryProcessDefinitionList(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser, - @ApiParam(name = "projectName", value = "PROJECT_NAME", required = true) @PathVariable String projectName - ) { + @ApiParam(name = "projectName", value = "PROJECT_NAME", required = true) @PathVariable String projectName) { + logger.info("query process definition list, login user:{}, project name:{}", - loginUser.getUserName(), projectName); + RegexUtils.escapeNRT(loginUser.getUserName()), + RegexUtils.escapeNRT(projectName)); Map result = processDefinitionService.queryProcessDefinitionList(loginUser, projectName); return returnDataList(result); } @@ -479,7 +521,13 @@ public class ProcessDefinitionController extends BaseController { @RequestParam(value = "searchVal", required = false) String searchVal, @RequestParam(value = "userId", required = false, defaultValue = "0") Integer userId, @RequestParam("pageSize") Integer pageSize) { - logger.info("query process definition list paging, login user:{}, project name:{}", loginUser.getUserName(), projectName); + + logger.info("query process definition list paging, login user:{}, project name:{}, searchVal:{}, userId:{}, pageSize:{}", + RegexUtils.escapeNRT(loginUser.getUserName()), + RegexUtils.escapeNRT(projectName), + RegexUtils.escapeNRT(searchVal), + userId, + pageSize); Map result = checkPageParams(pageNo, pageSize); if (result.get(Constants.STATUS) != Status.SUCCESS) { return returnDataListPaging(result); @@ -490,7 +538,7 @@ public class ProcessDefinitionController extends BaseController { } /** - * encapsulation treeview structure + * encapsulation tree view structure * * @param loginUser login user * @param projectName project name @@ -510,6 +558,12 @@ public class ProcessDefinitionController extends BaseController { @ApiParam(name = "projectName", value = "PROJECT_NAME", required = true) @PathVariable String projectName, @RequestParam("processId") Integer id, @RequestParam("limit") Integer limit) throws Exception { + + logger.info("encapsulation tree view structure, login user:{}, project name:{}, processId:{}, limit:{}", + RegexUtils.escapeNRT(loginUser.getUserName()), + RegexUtils.escapeNRT(projectName), + id, + limit); Map result = processDefinitionService.viewTree(id, limit); return returnDataList(result); } @@ -533,8 +587,10 @@ public class ProcessDefinitionController extends BaseController { @ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @ApiParam(name = "projectName", value = "PROJECT_NAME", required = true) @PathVariable String projectName, @RequestParam("processDefinitionCode") Long processDefinitionCode) throws Exception { - logger.info("query task node name list by definitionCode, login user:{}, project name:{}, code : {}", - loginUser.getUserName(), projectName, processDefinitionCode); + logger.info("query task node name list by definitionId, login user:{}, project name:{}, code : {}", + RegexUtils.escapeNRT(loginUser.getUserName()), + RegexUtils.escapeNRT(projectName), + processDefinitionCode); Map result = processDefinitionService.getTaskNodeListByDefinitionCode(processDefinitionCode); return returnDataList(result); } @@ -559,8 +615,10 @@ public class ProcessDefinitionController extends BaseController { @ApiParam(name = "projectName", value = "PROJECT_NAME", required = true) @PathVariable String projectName, @RequestParam("processDefinitionCodeList") String processDefinitionCodeList) { - logger.info("query task node name list by definitionId list, login user:{}, project name:{}, code list: {}", - loginUser.getUserName(), projectName, processDefinitionCodeList); + logger.info("query task node name list by definitionId list, login user:{}, project name:{}, id list: {}", + RegexUtils.escapeNRT(loginUser.getUserName()), + RegexUtils.escapeNRT(projectName), + RegexUtils.escapeNRT(processDefinitionCodeList)); Map result = processDefinitionService.getTaskNodeListByDefinitionCodeList(processDefinitionCodeList); return returnDataList(result); } @@ -584,8 +642,11 @@ public class ProcessDefinitionController extends BaseController { @ApiParam(name = "projectName", value = "PROJECT_NAME", required = true) @PathVariable String projectName, @RequestParam("processDefinitionId") Integer processDefinitionId ) { + logger.info("delete process definition by id, login user:{}, project name:{}, process definition id:{}", - loginUser.getUserName(), projectName, processDefinitionId); + RegexUtils.escapeNRT(loginUser.getUserName()), + RegexUtils.escapeNRT(projectName), + processDefinitionId); Map result = processDefinitionService.deleteProcessDefinitionById(loginUser, projectName, processDefinitionId); return returnDataList(result); } @@ -609,9 +670,11 @@ public class ProcessDefinitionController extends BaseController { @ApiParam(name = "projectName", value = "PROJECT_NAME", required = true) @PathVariable String projectName, @RequestParam("processDefinitionIds") String processDefinitionIds ) { - logger.info("delete process definition by ids, login user:{}, project name:{}, process definition ids:{}", - loginUser.getUserName(), projectName, processDefinitionIds); + logger.info("delete process definition by ids, login user:{}, project name:{}, process definition ids:{}", + RegexUtils.escapeNRT(loginUser.getUserName()), + RegexUtils.escapeNRT(projectName), + RegexUtils.escapeNRT(processDefinitionIds)); Map result = new HashMap<>(); List deleteFailedIdList = new ArrayList<>(); if (StringUtils.isNotEmpty(processDefinitionIds)) { @@ -660,8 +723,11 @@ public class ProcessDefinitionController extends BaseController { @RequestParam("processDefinitionIds") String processDefinitionIds, HttpServletResponse response) { try { + logger.info("batch export process definition by ids, login user:{}, project name:{}, process definition ids:{}", - loginUser.getUserName(), projectName, processDefinitionIds); + RegexUtils.escapeNRT(loginUser.getUserName()), + RegexUtils.escapeNRT(projectName), + RegexUtils.escapeNRT(processDefinitionIds)); processDefinitionService.batchExportProcessDefinitionByIds(loginUser, projectName, processDefinitionIds, response); } catch (Exception e) { logger.error(Status.BATCH_EXPORT_PROCESS_DEFINE_BY_IDS_ERROR.getMsg(), e); @@ -681,8 +747,10 @@ public class ProcessDefinitionController extends BaseController { @ApiException(QUERY_PROCESS_DEFINITION_LIST) public Result queryProcessDefinitionAllByProjectId(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @RequestParam("projectId") Integer projectId) { + logger.info("query process definition list, login user:{}, project id:{}", - loginUser.getUserName(), projectId); + RegexUtils.escapeNRT(loginUser.getUserName()), + projectId); Map result = processDefinitionService.queryProcessDefinitionAllByProjectId(projectId); return returnDataList(result); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessInstanceController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessInstanceController.java index ed0a98e9b2..cac3431e3f 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessInstanceController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessInstanceController.java @@ -126,8 +126,12 @@ public class ProcessInstanceController extends BaseController { "search value:{},executor name:{},state type:{},host:{},start time:{}, end time:{},page number:{}, page size:{}", loginUser.getUserName(), projectName, processDefinitionId, searchVal, executorName, stateType, host, startTime, endTime, pageNo, pageSize); + Map result = checkPageParams(pageNo, pageSize); + if (result.get(Constants.STATUS) != Status.SUCCESS) { + return returnDataListPaging(result); + } searchVal = ParameterUtils.handleEscapes(searchVal); - Map result = processInstanceService.queryProcessInstanceList( + result = processInstanceService.queryProcessInstanceList( loginUser, projectName, processDefinitionId, startTime, endTime, searchVal, executorName, stateType, host, pageNo, pageSize); return returnDataListPaging(result); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProjectController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProjectController.java index 1d45058ca7..bdd7d6d7ea 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProjectController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProjectController.java @@ -27,6 +27,7 @@ import static org.apache.dolphinscheduler.api.enums.Status.QUERY_PROJECT_DETAILS import static org.apache.dolphinscheduler.api.enums.Status.QUERY_UNAUTHORIZED_PROJECT_ERROR; import static org.apache.dolphinscheduler.api.enums.Status.UPDATE_PROJECT_ERROR; +import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.exceptions.ApiException; import org.apache.dolphinscheduler.api.service.ProcessDefinitionService; import org.apache.dolphinscheduler.api.service.ProjectService; @@ -172,8 +173,12 @@ public class ProjectController extends BaseController { ) { logger.info("login user {}, query project list paging", loginUser.getUserName()); + Map result = checkPageParams(pageNo, pageSize); + if (result.get(Constants.STATUS) != Status.SUCCESS) { + return returnDataListPaging(result); + } searchVal = ParameterUtils.handleEscapes(searchVal); - Map result = projectService.queryProjectListPaging(loginUser, pageSize, pageNo, searchVal); + result = projectService.queryProjectListPaging(loginUser, pageSize, pageNo, searchVal); return returnDataListPaging(result); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/SchedulerController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/SchedulerController.java index c3492f7993..f4848d86ed 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/SchedulerController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/SchedulerController.java @@ -27,10 +27,12 @@ import static org.apache.dolphinscheduler.api.enums.Status.QUERY_SCHEDULE_LIST_P import static org.apache.dolphinscheduler.api.enums.Status.UPDATE_SCHEDULE_ERROR; import static org.apache.dolphinscheduler.common.Constants.SESSION_USER; +import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.exceptions.ApiException; import org.apache.dolphinscheduler.api.service.SchedulerService; import org.apache.dolphinscheduler.api.utils.RegexUtils; import org.apache.dolphinscheduler.api.utils.Result; +import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.FailureStrategy; import org.apache.dolphinscheduler.common.enums.Priority; import org.apache.dolphinscheduler.common.enums.ReleaseState; @@ -245,8 +247,12 @@ public class SchedulerController extends BaseController { @RequestParam("pageSize") Integer pageSize) { logger.info("login user {}, query schedule, project name: {}, process definition id: {}", loginUser.getUserName(), projectName, processDefinitionId); + Map result = checkPageParams(pageNo, pageSize); + if (result.get(Constants.STATUS) != Status.SUCCESS) { + return returnDataListPaging(result); + } searchVal = ParameterUtils.handleEscapes(searchVal); - Map result = schedulerService.querySchedule(loginUser, projectName, processDefinitionId, searchVal, pageNo, pageSize); + result = schedulerService.querySchedule(loginUser, projectName, processDefinitionId, searchVal, pageNo, pageSize); return returnDataListPaging(result); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TaskInstanceController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TaskInstanceController.java index 1d7459c1b0..42358bd65f 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TaskInstanceController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TaskInstanceController.java @@ -20,6 +20,7 @@ package org.apache.dolphinscheduler.api.controller; import static org.apache.dolphinscheduler.api.enums.Status.FORCE_TASK_SUCCESS_ERROR; import static org.apache.dolphinscheduler.api.enums.Status.QUERY_TASK_LIST_PAGING_ERROR; +import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.exceptions.ApiException; import org.apache.dolphinscheduler.api.service.TaskInstanceService; import org.apache.dolphinscheduler.api.utils.RegexUtils; @@ -122,8 +123,13 @@ public class TaskInstanceController extends BaseController { RegexUtils.escapeNRT(host), RegexUtils.escapeNRT(startTime), RegexUtils.escapeNRT(endTime)); + + Map result = checkPageParams(pageNo, pageSize); + if (result.get(Constants.STATUS) != Status.SUCCESS) { + return returnDataListPaging(result); + } searchVal = ParameterUtils.handleEscapes(searchVal); - Map result = taskInstanceService.queryTaskListPaging( + result = taskInstanceService.queryTaskListPaging( loginUser, projectName, processInstanceId, processInstanceName, taskName, executorName, startTime, endTime, searchVal, stateType, host, pageNo, pageSize); return returnDataListPaging(result); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/WorkerGroupController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/WorkerGroupController.java index c2c74ce3da..4211feebd1 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/WorkerGroupController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/WorkerGroupController.java @@ -18,9 +18,11 @@ package org.apache.dolphinscheduler.api.controller; import static org.apache.dolphinscheduler.api.enums.Status.DELETE_WORKER_GROUP_FAIL; +import static org.apache.dolphinscheduler.api.enums.Status.QUERY_WORKER_ADDRESS_LIST_FAIL; import static org.apache.dolphinscheduler.api.enums.Status.QUERY_WORKER_GROUP_FAIL; import static org.apache.dolphinscheduler.api.enums.Status.SAVE_ERROR; +import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.exceptions.ApiException; import org.apache.dolphinscheduler.api.service.WorkerGroupService; import org.apache.dolphinscheduler.api.utils.RegexUtils; @@ -116,8 +118,12 @@ public class WorkerGroupController extends BaseController { ) { logger.info("query all worker group paging: login user {}, pageNo:{}, pageSize:{}, searchVal:{}", RegexUtils.escapeNRT(loginUser.getUserName()), pageNo, pageSize, searchVal); + Map result = checkPageParams(pageNo, pageSize); + if (result.get(Constants.STATUS) != Status.SUCCESS) { + return returnDataListPaging(result); + } searchVal = ParameterUtils.handleEscapes(searchVal); - Map result = workerGroupService.queryAllGroupPaging(loginUser, pageNo, pageSize, searchVal); + result = workerGroupService.queryAllGroupPaging(loginUser, pageNo, pageSize, searchVal); return returnDataListPaging(result); } @@ -159,4 +165,20 @@ public class WorkerGroupController extends BaseController { return returnDataList(result); } + /** + * query worker address list + * + * @param loginUser login user + * @return all worker address list + */ + @ApiOperation(value = "queryWorkerAddressList", notes = "QUERY_WORKER_ADDRESS_LIST_NOTES") + @GetMapping(value = "/worker-address-list") + @ResponseStatus(HttpStatus.OK) + @ApiException(QUERY_WORKER_ADDRESS_LIST_FAIL) + public Result queryWorkerAddressList(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser) { + logger.info("query worker address list: login user {}", RegexUtils.escapeNRT(loginUser.getUserName())); + Map result = workerGroupService.getWorkerAddressList(); + return returnDataList(result); + } + } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java index dd31fc2124..5195ac2b21 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java @@ -209,6 +209,7 @@ public enum Status { CREATE_WORKER_GROUP_FORBIDDEN_IN_DOCKER(10175, "create worker group forbidden in docker ", "创建worker分组在docker中禁止"), DELETE_WORKER_GROUP_FORBIDDEN_IN_DOCKER(10176, "delete worker group forbidden in docker ", "删除worker分组在docker中禁止"), WORKER_ADDRESS_INVALID(10177, "worker address {0} invalid", "worker地址[{0}]无效"), + QUERY_WORKER_ADDRESS_LIST_FAIL(10178, "query worker address list fail ", "查询worker地址列表失败"), UDF_FUNCTION_NOT_EXIST(20001, "UDF function not found", "UDF函数不存在"), UDF_FUNCTION_EXISTS(20002, "UDF function already exists", "UDF函数已存在"), diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/interceptor/RateLimitInterceptor.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/interceptor/RateLimitInterceptor.java new file mode 100644 index 0000000000..f8839a3560 --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/interceptor/RateLimitInterceptor.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.interceptor; + +import org.apache.dolphinscheduler.api.configuration.TrafficConfiguration; +import org.apache.dolphinscheduler.common.utils.StringUtils; + +import org.apache.commons.collections.MapUtils; + +import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.http.HttpStatus; +import org.springframework.web.servlet.HandlerInterceptor; + +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; +import com.google.common.util.concurrent.RateLimiter; + +/** + * This interceptor is used to control the traffic, consists with global traffic control and tenant-leve traffic control. + * If the current coming tenant reaches his tenant-level request quota, his request will be reject fast. + * If the current system request number reaches the global request quota, all coming request will be reject fast. + */ +public class RateLimitInterceptor implements HandlerInterceptor { + + private static final Logger logger = LoggerFactory.getLogger(RateLimitInterceptor.class); + + private TrafficConfiguration trafficConfiguration; + + private RateLimiter globalRateLimiter; + + private LoadingCache tenantRateLimiterCache = CacheBuilder.newBuilder() + .maximumSize(100) + .expireAfterAccess(10, TimeUnit.MINUTES) + .build(new CacheLoader() { + @Override + public RateLimiter load(String token) { + // use tenant customize rate limit + Map customizeTenantQpsRate = trafficConfiguration.getCustomizeTenantQpsRate(); + int tenantQuota = trafficConfiguration.getDefaultTenantQpsRate(); + if (MapUtils.isNotEmpty(customizeTenantQpsRate)) { + tenantQuota = customizeTenantQpsRate.getOrDefault(token, trafficConfiguration.getDefaultTenantQpsRate()); + } + // use tenant default rate limit + return RateLimiter.create(tenantQuota, 1, TimeUnit.SECONDS); + } + }); + + @Override + public boolean preHandle(HttpServletRequest request, HttpServletResponse response, Object handler) throws ExecutionException { + // tenant-level rate limit + if (trafficConfiguration.isTrafficTenantControlSwitch()) { + String token = request.getHeader("token"); + if (StringUtils.isNotEmpty(token)) { + RateLimiter tenantRateLimiter = tenantRateLimiterCache.get(token); + if (!tenantRateLimiter.tryAcquire()) { + response.setStatus(HttpStatus.TOO_MANY_REQUESTS.value()); + logger.warn("Too many request, reach tenant rate limit, current tenant:{} qps is {}", token, tenantRateLimiter.getRate()); + return false; + } + } + } + // global rate limit + if (trafficConfiguration.isTrafficGlobalControlSwitch()) { + if (!globalRateLimiter.tryAcquire()) { + response.setStatus(HttpStatus.TOO_MANY_REQUESTS.value()); + logger.warn("Too many request, reach global rate limit, current qps is {}", globalRateLimiter.getRate()); + return false; + } + } + return true; + } + + public RateLimitInterceptor(TrafficConfiguration trafficConfiguration) { + this.trafficConfiguration = trafficConfiguration; + if (trafficConfiguration.isTrafficGlobalControlSwitch()) { + this.globalRateLimiter = RateLimiter.create(trafficConfiguration.getMaxGlobalQpsRate(), 1, TimeUnit.SECONDS); + } + } + +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java index fbaaf4cbfa..59ada6e058 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java @@ -18,7 +18,8 @@ package org.apache.dolphinscheduler.api.service; import org.apache.dolphinscheduler.api.utils.Result; -import org.apache.dolphinscheduler.common.enums.DbConnectType; +import org.apache.dolphinscheduler.common.datasource.BaseDataSourceParamDTO; +import org.apache.dolphinscheduler.common.datasource.ConnectionParam; import org.apache.dolphinscheduler.common.enums.DbType; import org.apache.dolphinscheduler.dao.entity.User; @@ -33,26 +34,20 @@ public interface DataSourceService { * create data source * * @param loginUser login user - * @param name data source name - * @param desc data source description - * @param type data source type - * @param parameter datasource parameters + * @param datasourceParam datasource parameter * @return create result code */ - Result createDataSource(User loginUser, String name, String desc, DbType type, String parameter); + Result createDataSource(User loginUser, BaseDataSourceParamDTO datasourceParam); /** * updateProcessInstance datasource * * @param loginUser login user - * @param name data source name - * @param desc data source description - * @param type data source type - * @param parameter datasource parameters - * @param id data source id + * @param id data source id + * @param dataSourceParam data source params * @return update result code */ - Result updateDataSource(int id, User loginUser, String name, String desc, DbType type, String parameter); + Result updateDataSource(int id, User loginUser, BaseDataSourceParamDTO dataSourceParam); /** * updateProcessInstance datasource @@ -97,7 +92,7 @@ public interface DataSourceService { * @param parameter data source parameters * @return true if connect successfully, otherwise false */ - Result checkConnection(DbType type, String parameter); + Result checkConnection(DbType type, ConnectionParam parameter); /** * test connection @@ -107,24 +102,6 @@ public interface DataSourceService { */ Result connectionTest(int id); - /** - * build paramters - * - * @param type data source type - * @param host data source host - * @param port data source port - * @param database data source database name - * @param userName user name - * @param password password - * @param other other parameters - * @param principal principal - * @return datasource parameter - */ - String buildParameter(DbType type, String host, - String port, String database, String principal, String userName, - String password, DbConnectType connectType, String other, - String javaSecurityKrb5Conf, String loginUserKeytabUsername, String loginUserKeytabPath); - /** * delete datasource * diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/WorkerGroupService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/WorkerGroupService.java index a8b11f45e8..49ba33dac0 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/WorkerGroupService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/WorkerGroupService.java @@ -62,4 +62,11 @@ public interface WorkerGroupService { */ Map deleteWorkerGroupById(User loginUser, Integer id); + /** + * query all worker address list + * + * @return all worker address list + */ + Map getWorkerAddressList(); + } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataSourceServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataSourceServiceImpl.java index 1a2ea7b57d..8d86690d2b 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataSourceServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataSourceServiceImpl.java @@ -18,37 +18,30 @@ package org.apache.dolphinscheduler.api.service.impl; import org.apache.dolphinscheduler.api.enums.Status; -import org.apache.dolphinscheduler.api.exceptions.ServiceException; import org.apache.dolphinscheduler.api.service.DataSourceService; import org.apache.dolphinscheduler.api.utils.PageInfo; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.common.enums.DbConnectType; +import org.apache.dolphinscheduler.common.datasource.BaseConnectionParam; +import org.apache.dolphinscheduler.common.datasource.BaseDataSourceParamDTO; +import org.apache.dolphinscheduler.common.datasource.ConnectionParam; +import org.apache.dolphinscheduler.common.datasource.DatasourceUtil; import org.apache.dolphinscheduler.common.enums.DbType; -import org.apache.dolphinscheduler.common.utils.CommonUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.StringUtils; -import org.apache.dolphinscheduler.dao.datasource.BaseDataSource; -import org.apache.dolphinscheduler.dao.datasource.DataSourceFactory; -import org.apache.dolphinscheduler.dao.datasource.MySQLDataSource; -import org.apache.dolphinscheduler.dao.datasource.OracleDataSource; import org.apache.dolphinscheduler.dao.entity.DataSource; import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.mapper.DataSourceMapper; import org.apache.dolphinscheduler.dao.mapper.DataSourceUserMapper; -import org.apache.commons.collections4.MapUtils; - import java.sql.Connection; import java.util.ArrayList; import java.util.Date; import java.util.HashMap; import java.util.HashSet; -import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Set; -import java.util.regex.Pattern; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -68,25 +61,6 @@ public class DataSourceServiceImpl extends BaseServiceImpl implements DataSource private static final Logger logger = LoggerFactory.getLogger(DataSourceServiceImpl.class); - public static final String NAME = "name"; - public static final String NOTE = "note"; - public static final String TYPE = "type"; - public static final String HOST = "host"; - public static final String PORT = "port"; - public static final String PRINCIPAL = "principal"; - public static final String DATABASE = "database"; - public static final String USER_NAME = "userName"; - public static final String OTHER = "other"; - - private static final Pattern IPV4_PATTERN = Pattern.compile("^[a-zA-Z0-9\\_\\-\\.]+$"); - - private static final Pattern IPV6_PATTERN = Pattern.compile("^[a-zA-Z0-9\\_\\-\\.\\:\\[\\]]+$"); - - private static final Pattern DATABASE_PATTER = Pattern.compile("^[a-zA-Z0-9\\_\\-\\.]+$"); - - private static final Pattern PARAMS_PATTER = Pattern.compile("^[a-zA-Z0-9]+$"); - - @Autowired private DataSourceMapper dataSourceMapper; @@ -97,23 +71,23 @@ public class DataSourceServiceImpl extends BaseServiceImpl implements DataSource * create data source * * @param loginUser login user - * @param name data source name - * @param desc data source description - * @param type data source type - * @param parameter datasource parameters + * @param datasourceParam datasource parameters * @return create result code */ @Override - public Result createDataSource(User loginUser, String name, String desc, DbType type, String parameter) { - + public Result createDataSource(User loginUser, BaseDataSourceParamDTO datasourceParam) { + DatasourceUtil.checkDatasourceParam(datasourceParam); Result result = new Result<>(); // check name can use or not - if (checkName(name)) { + if (checkName(datasourceParam.getName())) { putMsg(result, Status.DATASOURCE_EXIST); return result; } - Result isConnection = checkConnection(type, parameter); + // check connect + ConnectionParam connectionParam = DatasourceUtil.buildConnectionParams(datasourceParam); + Result isConnection = checkConnection(datasourceParam.getType(), connectionParam); if (Status.SUCCESS.getCode() != isConnection.getCode()) { + putMsg(result, Status.DATASOURCE_CONNECT_FAILED); return result; } @@ -121,12 +95,12 @@ public class DataSourceServiceImpl extends BaseServiceImpl implements DataSource DataSource dataSource = new DataSource(); Date now = new Date(); - dataSource.setName(name.trim()); - dataSource.setNote(desc); + dataSource.setName(datasourceParam.getName().trim()); + dataSource.setNote(datasourceParam.getNote()); dataSource.setUserId(loginUser.getId()); dataSource.setUserName(loginUser.getUserName()); - dataSource.setType(type); - dataSource.setConnectionParams(parameter); + dataSource.setType(datasourceParam.getType()); + dataSource.setConnectionParams(JSONUtils.toJsonString(connectionParam)); dataSource.setCreateTime(now); dataSource.setUpdateTime(now); dataSourceMapper.insert(dataSource); @@ -148,8 +122,8 @@ public class DataSourceServiceImpl extends BaseServiceImpl implements DataSource * @return update result code */ @Override - public Result updateDataSource(int id, User loginUser, String name, String desc, DbType type, String parameter) { - + public Result updateDataSource(int id, User loginUser, BaseDataSourceParamDTO dataSourceParam) { + DatasourceUtil.checkDatasourceParam(dataSourceParam); Result result = new Result<>(); // determine whether the data source exists DataSource dataSource = dataSourceMapper.selectById(id); @@ -164,33 +138,31 @@ public class DataSourceServiceImpl extends BaseServiceImpl implements DataSource } //check name can use or not - if (!name.trim().equals(dataSource.getName()) && checkName(name)) { + if (!dataSource.getName().trim().equals(dataSource.getName()) && checkName(dataSource.getName())) { putMsg(result, Status.DATASOURCE_EXIST); return result; } //check password，if the password is not updated, set to the old password. - ObjectNode paramObject = JSONUtils.parseObject(parameter); - String password = paramObject.path(Constants.PASSWORD).asText(); + BaseConnectionParam connectionParam = (BaseConnectionParam) DatasourceUtil.buildConnectionParams(dataSourceParam); + String password = connectionParam.getPassword(); if (StringUtils.isBlank(password)) { String oldConnectionParams = dataSource.getConnectionParams(); ObjectNode oldParams = JSONUtils.parseObject(oldConnectionParams); - paramObject.put(Constants.PASSWORD, oldParams.path(Constants.PASSWORD).asText()); + connectionParam.setPassword(oldParams.path(Constants.PASSWORD).asText()); } - // connectionParams json - String connectionParams = paramObject.toString(); - Result isConnection = checkConnection(type, parameter); + Result isConnection = checkConnection(dataSource.getType(), connectionParam); if (Status.SUCCESS.getCode() != isConnection.getCode()) { return result; } Date now = new Date(); - dataSource.setName(name.trim()); - dataSource.setNote(desc); + dataSource.setName(dataSource.getName().trim()); + dataSource.setNote(dataSource.getNote()); dataSource.setUserName(loginUser.getUserName()); - dataSource.setType(type); - dataSource.setConnectionParams(connectionParams); + dataSource.setType(dataSource.getType()); + dataSource.setConnectionParams(JSONUtils.toJsonString(connectionParam)); dataSource.setUpdateTime(now); dataSourceMapper.updateById(dataSource); putMsg(result, Status.SUCCESS); @@ -218,79 +190,13 @@ public class DataSourceServiceImpl extends BaseServiceImpl implements DataSource return result; } // type - String dataSourceType = dataSource.getType().toString(); - // name - String dataSourceName = dataSource.getName(); - // desc - String desc = dataSource.getNote(); - // parameter - String parameter = dataSource.getConnectionParams(); + BaseDataSourceParamDTO baseDataSourceParamDTO = DatasourceUtil.buildDatasourceParamDTO( + dataSource.getType(), dataSource.getConnectionParams()); + baseDataSourceParamDTO.setId(dataSource.getId()); + baseDataSourceParamDTO.setName(dataSource.getName()); + baseDataSourceParamDTO.setNote(dataSource.getNote()); - BaseDataSource datasourceForm = DataSourceFactory.getDatasource(dataSource.getType(), parameter); - DbConnectType connectType = null; - String hostSeperator = Constants.DOUBLE_SLASH; - if (DbType.ORACLE.equals(dataSource.getType())) { - connectType = ((OracleDataSource) datasourceForm).getConnectType(); - if (DbConnectType.ORACLE_SID.equals(connectType)) { - hostSeperator = Constants.AT_SIGN; - } - } - String database = datasourceForm.getDatabase(); - // jdbc connection params - String other = datasourceForm.getOther(); - String address = datasourceForm.getAddress(); - - String[] hostsPorts = getHostsAndPort(address, hostSeperator); - // ip host - String host = hostsPorts[0]; - // prot - String port = hostsPorts[1]; - String separator = ""; - - switch (dataSource.getType()) { - case HIVE: - case SQLSERVER: - separator = ";"; - break; - case MYSQL: - case POSTGRESQL: - case CLICKHOUSE: - case ORACLE: - case PRESTO: - separator = "&"; - break; - default: - separator = "&"; - break; - } - - Map otherMap = new LinkedHashMap<>(); - if (other != null) { - String[] configs = other.split(separator); - for (String config : configs) { - otherMap.put(config.split("=")[0], config.split("=")[1]); - } - - } - - Map map = new HashMap<>(); - map.put(NAME, dataSourceName); - map.put(NOTE, desc); - map.put(TYPE, dataSourceType); - if (connectType != null) { - map.put(Constants.ORACLE_DB_CONNECT_TYPE, connectType); - } - - map.put(HOST, host); - map.put(PORT, port); - map.put(PRINCIPAL, datasourceForm.getPrincipal()); - map.put(Constants.KERBEROS_KRB5_CONF_PATH, datasourceForm.getJavaSecurityKrb5Conf()); - map.put(Constants.KERBEROS_KEY_TAB_USERNAME, datasourceForm.getLoginUserKeytabUsername()); - map.put(Constants.KERBEROS_KEY_TAB_PATH, datasourceForm.getLoginUserKeytabPath()); - map.put(DATABASE, database); - map.put(USER_NAME, datasourceForm.getUser()); - map.put(OTHER, otherMap); - result.put(Constants.DATA_LIST, map); + result.put(Constants.DATA_LIST, baseDataSourceParamDTO); putMsg(result, Status.SUCCESS); return result; } @@ -400,14 +306,9 @@ public class DataSourceServiceImpl extends BaseServiceImpl implements DataSource * @return true if connect successfully, otherwise false */ @Override - public Result checkConnection(DbType type, String parameter) { + public Result checkConnection(DbType type, ConnectionParam connectionParam) { Result result = new Result<>(); - BaseDataSource datasource = DataSourceFactory.getDatasource(type, parameter); - if (datasource == null) { - putMsg(result, Status.DATASOURCE_TYPE_NOT_EXIST, type); - return result; - } - try (Connection connection = datasource.getConnection()) { + try (Connection connection = DatasourceUtil.getConnection(type, connectionParam)) { if (connection == null) { putMsg(result, Status.CONNECTION_TEST_FAILURE); return result; @@ -415,7 +316,7 @@ public class DataSourceServiceImpl extends BaseServiceImpl implements DataSource putMsg(result, Status.SUCCESS); return result; } catch (Exception e) { - logger.error("datasource test connection error, dbType:{}, jdbcUrl:{}, message:{}.", type, datasource.getJdbcUrl(), e.getMessage()); + logger.error("datasource test connection error, dbType:{}, connectionParam:{}, message:{}.", type, connectionParam, e.getMessage()); return new Result<>(Status.CONNECTION_TEST_FAILURE.getCode(), e.getMessage()); } } @@ -434,135 +335,7 @@ public class DataSourceServiceImpl extends BaseServiceImpl implements DataSource putMsg(result, Status.RESOURCE_NOT_EXIST); return result; } - return checkConnection(dataSource.getType(), dataSource.getConnectionParams()); - } - - /** - * build paramters - * - * @param type data source type - * @param host data source host - * @param port data source port - * @param database data source database name - * @param userName user name - * @param password password - * @param other other parameters - * @param principal principal - * @return datasource parameter - */ - @Override - public String buildParameter(DbType type, String host, - String port, String database, String principal, String userName, - String password, DbConnectType connectType, String other, - String javaSecurityKrb5Conf, String loginUserKeytabUsername, String loginUserKeytabPath) { - checkParams(type, port, host, database, other); - String address = buildAddress(type, host, port, connectType); - Map parameterMap = new LinkedHashMap<>(); - String jdbcUrl; - if (DbType.SQLSERVER == type) { - jdbcUrl = address + ";databaseName=" + database; - } else { - jdbcUrl = address + "/" + database; - } - - if (Constants.ORACLE.equals(type.name())) { - parameterMap.put(Constants.ORACLE_DB_CONNECT_TYPE, connectType); - } - - if (CommonUtils.getKerberosStartupState() - && (type == DbType.HIVE || type == DbType.SPARK)) { - jdbcUrl += ";principal=" + principal; - } - - String separator = ""; - if (Constants.MYSQL.equals(type.name()) - || Constants.POSTGRESQL.equals(type.name()) - || Constants.CLICKHOUSE.equals(type.name()) - || Constants.ORACLE.equals(type.name()) - || Constants.PRESTO.equals(type.name())) { - separator = "&"; - } else if (Constants.HIVE.equals(type.name()) - || Constants.SPARK.equals(type.name()) - || Constants.DB2.equals(type.name()) - || Constants.SQLSERVER.equals(type.name())) { - separator = ";"; - } - - parameterMap.put(Constants.ADDRESS, address); - parameterMap.put(Constants.DATABASE, database); - parameterMap.put(Constants.JDBC_URL, jdbcUrl); - parameterMap.put(Constants.USER, userName); - parameterMap.put(Constants.PASSWORD, CommonUtils.encodePassword(password)); - if (CommonUtils.getKerberosStartupState() - && (type == DbType.HIVE || type == DbType.SPARK)) { - parameterMap.put(Constants.PRINCIPAL, principal); - parameterMap.put(Constants.KERBEROS_KRB5_CONF_PATH, javaSecurityKrb5Conf); - parameterMap.put(Constants.KERBEROS_KEY_TAB_USERNAME, loginUserKeytabUsername); - parameterMap.put(Constants.KERBEROS_KEY_TAB_PATH, loginUserKeytabPath); - } - - Map map = JSONUtils.toMap(other); - if (type == DbType.MYSQL) { - map = MySQLDataSource.buildOtherParams(other); - } - - if (MapUtils.isNotEmpty(map)) { - StringBuilder otherSb = new StringBuilder(); - for (Map.Entry entry : map.entrySet()) { - otherSb.append(String.format("%s=%s%s", entry.getKey(), entry.getValue(), separator)); - } - if (!Constants.DB2.equals(type.name())) { - otherSb.deleteCharAt(otherSb.length() - 1); - } - parameterMap.put(Constants.OTHER, otherSb); - } - - if (logger.isDebugEnabled()) { - logger.info("parameters map:{}", JSONUtils.toJsonString(parameterMap)); - } - return JSONUtils.toJsonString(parameterMap); - - } - - private String buildAddress(DbType type, String host, String port, DbConnectType connectType) { - StringBuilder sb = new StringBuilder(); - if (Constants.MYSQL.equals(type.name())) { - sb.append(Constants.JDBC_MYSQL); - sb.append(host).append(":").append(port); - } else if (Constants.POSTGRESQL.equals(type.name())) { - sb.append(Constants.JDBC_POSTGRESQL); - sb.append(host).append(":").append(port); - } else if (Constants.HIVE.equals(type.name()) || Constants.SPARK.equals(type.name())) { - sb.append(Constants.JDBC_HIVE_2); - String[] hostArray = host.split(","); - if (hostArray.length > 0) { - for (String zkHost : hostArray) { - sb.append(String.format("%s:%s,", zkHost, port)); - } - sb.deleteCharAt(sb.length() - 1); - } - } else if (Constants.CLICKHOUSE.equals(type.name())) { - sb.append(Constants.JDBC_CLICKHOUSE); - sb.append(host).append(":").append(port); - } else if (Constants.ORACLE.equals(type.name())) { - if (connectType == DbConnectType.ORACLE_SID) { - sb.append(Constants.JDBC_ORACLE_SID); - } else { - sb.append(Constants.JDBC_ORACLE_SERVICE_NAME); - } - sb.append(host).append(":").append(port); - } else if (Constants.SQLSERVER.equals(type.name())) { - sb.append(Constants.JDBC_SQLSERVER); - sb.append(host).append(":").append(port); - } else if (Constants.DB2.equals(type.name())) { - sb.append(Constants.JDBC_DB2); - sb.append(host).append(":").append(port); - } else if (Constants.PRESTO.equals(type.name())) { - sb.append(Constants.JDBC_PRESTO); - sb.append(host).append(":").append(port); - } - - return sb.toString(); + return checkConnection(dataSource.getType(), DatasourceUtil.buildConnectionParams(dataSource.getType(), dataSource.getConnectionParams())); } /** @@ -661,61 +434,4 @@ public class DataSourceServiceImpl extends BaseServiceImpl implements DataSource return result; } - /** - * get host and port by address - * - * @param address address - * @param separator separator - * @return sting array: [host,port] - */ - private String[] getHostsAndPort(String address, String separator) { - String[] result = new String[2]; - String[] tmpArray = address.split(separator); - String hostsAndPorts = tmpArray[tmpArray.length - 1]; - StringBuilder hosts = new StringBuilder(); - String[] hostPortArray = hostsAndPorts.split(Constants.COMMA); - String port = hostPortArray[0].split(Constants.COLON)[1]; - for (String hostPort : hostPortArray) { - hosts.append(hostPort.split(Constants.COLON)[0]).append(Constants.COMMA); - } - hosts.deleteCharAt(hosts.length() - 1); - result[0] = hosts.toString(); - result[1] = port; - return result; - } - - private void checkParams(DbType type, String port, String host, String database, String other) { - if (null == DbType.of(type.getCode())) { - throw new ServiceException(Status.DATASOURCE_DB_TYPE_ILLEGAL); - } - if (!isNumeric(port)) { - throw new ServiceException(Status.DATASOURCE_PORT_ILLEGAL); - } - if (!IPV4_PATTERN.matcher(host).matches() || !IPV6_PATTERN.matcher(host).matches()) { - throw new ServiceException(Status.DATASOURCE_HOST_ILLEGAL); - } - if (!DATABASE_PATTER.matcher(database).matches()) { - throw new ServiceException(Status.DATASOURCE_NAME_ILLEGAL); - } - if (StringUtils.isBlank(other)) { - return; - } - Map map = JSONUtils.toMap(other); - if (MapUtils.isEmpty(map)) { - return; - } - boolean paramsCheck = map.entrySet().stream().allMatch(p -> PARAMS_PATTER.matcher(p.getValue()).matches()); - if (!paramsCheck) { - throw new ServiceException(Status.DATASOURCE_OTHER_PARAMS_ILLEGAL); - } - } - - private static boolean isNumeric(String str) { - for (int i = str.length(); --i >= 0; ) { - if (!Character.isDigit(str.charAt(i))) { - return false; - } - } - return true; - } } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkerGroupServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkerGroupServiceImpl.java index 381d1e853b..5d65c38785 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkerGroupServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkerGroupServiceImpl.java @@ -81,10 +81,6 @@ public class WorkerGroupServiceImpl extends BaseServiceImpl implements WorkerGro if (isNotAdmin(loginUser, result)) { return result; } - if (Constants.DOCKER_MODE && !Constants.KUBERNETES_MODE) { - putMsg(result, Status.CREATE_WORKER_GROUP_FORBIDDEN_IN_DOCKER); - return result; - } if (StringUtils.isEmpty(name)) { putMsg(result, Status.NAME_NULL); return result; @@ -303,10 +299,6 @@ public class WorkerGroupServiceImpl extends BaseServiceImpl implements WorkerGro if (isNotAdmin(loginUser, result)) { return result; } - if (Constants.DOCKER_MODE && !Constants.KUBERNETES_MODE) { - putMsg(result, Status.DELETE_WORKER_GROUP_FORBIDDEN_IN_DOCKER); - return result; - } WorkerGroup workerGroup = workerGroupMapper.selectById(id); if (workerGroup == null) { putMsg(result, Status.DELETE_WORKER_GROUP_NOT_EXIST); @@ -323,4 +315,18 @@ public class WorkerGroupServiceImpl extends BaseServiceImpl implements WorkerGro return result; } + /** + * query all worker address list + * + * @return all worker address list + */ + @Override + public Map getWorkerAddressList() { + Map result = new HashMap<>(); + List serverNodeList = zookeeperMonitor.getServerNodeList(ZKNodeType.WORKER, true); + result.put(Constants.DATA_LIST, serverNodeList); + putMsg(result, Status.SUCCESS); + return result; + } + } diff --git a/dolphinscheduler-api/src/main/resources/application-api.properties b/dolphinscheduler-api/src/main/resources/application-api.properties index f42588112b..a635cccd6e 100644 --- a/dolphinscheduler-api/src/main/resources/application-api.properties +++ b/dolphinscheduler-api/src/main/resources/application-api.properties @@ -35,10 +35,10 @@ spring.servlet.multipart.max-request-size=1024MB server.compression.enabled=true server.compression.mime-types=text/html,text/xml,text/plain,text/css,text/javascript,application/javascript,application/json,application/xml -# post content +# max http post size server.jetty.max-http-form-post-size=5000000 -# i18n +# messages encoding spring.messages.encoding=UTF-8 # i18n classpath folder , file prefix messages， if have many files, use "," seperator @@ -47,6 +47,15 @@ spring.messages.basename=i18n/messages # Authentication types (supported types: PASSWORD) security.authentication.type=PASSWORD +# Traffic control, if you turn on this config, the maximum number of request/s will be limited. +# global max request number per second +# default tenant-level max request number +#traffic.control.global.switch=true +#traffic.control.max.global.qps.rate=500 +#traffic.control.tenant.switch=true +#traffic.control.default.tenant.qps.rate=10 +#traffic.control.customize.tenant.qps.rate={'tenant1':11,'tenant2':20} + #============================================================================ # LDAP Config # mock ldap server from https://www.forumsys.com/tutorials/integration-how-to/ldap/online-ldap-test-server/ diff --git a/dolphinscheduler-api/src/main/resources/i18n/messages_zh_CN.properties b/dolphinscheduler-api/src/main/resources/i18n/messages_zh_CN.properties index f1e8ebefc3..b3390feb1a 100644 --- a/dolphinscheduler-api/src/main/resources/i18n/messages_zh_CN.properties +++ b/dolphinscheduler-api/src/main/resources/i18n/messages_zh_CN.properties @@ -52,7 +52,7 @@ COUNT_PROCESS_INSTANCE_NOTES=统计流程实例状态 COUNT_PROCESS_DEFINITION_BY_USER_NOTES=统计用户创建的流程定义 COUNT_COMMAND_STATE_NOTES=统计命令状态 COUNT_QUEUE_STATE_NOTES=统计队列里任务状态 -ACCESS_TOKEN_TAG=access token相关操作，需要先登录 +ACCESS_TOKEN_TAG=访问token相关操作 MONITOR_TAG=监控相关操作 MASTER_LIST_NOTES=master服务列表 WORKER_LIST_NOTES=worker服务列表 diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/configuration/TrafficConfigurationTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/configuration/TrafficConfigurationTest.java new file mode 100644 index 0000000000..b5e9244186 --- /dev/null +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/configuration/TrafficConfigurationTest.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.configuration; + +import org.apache.commons.collections.MapUtils; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.junit4.SpringRunner; + +@RunWith(SpringRunner.class) +@SpringBootTest +public class TrafficConfigurationTest { + + @Autowired + private TrafficConfiguration trafficConfiguration; + + @Test + public void isTrafficGlobalControlSwitch() { + Assert.assertFalse(trafficConfiguration.isTrafficGlobalControlSwitch()); + } + + @Test + public void getMaxGlobalQpsLimit() { + Assert.assertEquals(300, (int) trafficConfiguration.getMaxGlobalQpsRate()); + } + + @Test + public void isTrafficTenantControlSwitch() { + Assert.assertFalse(trafficConfiguration.isTrafficTenantControlSwitch()); + } + + @Test + public void getDefaultTenantQpsLimit() { + Assert.assertEquals(10, (int) trafficConfiguration.getDefaultTenantQpsRate()); + } + + @Test + public void getCustomizeTenantQpsRate() { + Assert.assertTrue(MapUtils.isEmpty(trafficConfiguration.getCustomizeTenantQpsRate())); + } +} \ No newline at end of file diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/AbstractControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/AbstractControllerTest.java index 9c75e8638e..794b69b960 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/AbstractControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/AbstractControllerTest.java @@ -38,7 +38,6 @@ import org.springframework.web.context.WebApplicationContext; /** * abstract controller test */ -@Ignore @RunWith(SpringRunner.class) @SpringBootTest(classes = ApiApplicationServer.class) public class AbstractControllerTest { @@ -64,7 +63,7 @@ public class AbstractControllerTest { } @After - public void after(){ + public void after() throws Exception { sessionService.signOut("127.0.0.1", user); } diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/DataSourceControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/DataSourceControllerTest.java index dbd0e70f87..04b706cd40 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/DataSourceControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/DataSourceControllerTest.java @@ -24,8 +24,11 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers. import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.utils.Result; +import org.apache.dolphinscheduler.common.datasource.mysql.MysqlDatasourceParamDTO; import org.apache.dolphinscheduler.common.utils.JSONUtils; +import java.util.HashMap; + import org.junit.Assert; import org.junit.Ignore; import org.junit.Test; @@ -46,20 +49,19 @@ public class DataSourceControllerTest extends AbstractControllerTest{ @Ignore @Test public void testCreateDataSource() throws Exception { - MultiValueMap paramsMap = new LinkedMultiValueMap<>(); - paramsMap.add("name","mysql"); - paramsMap.add("node","mysql data source test"); - paramsMap.add("type","MYSQL"); - paramsMap.add("host","192.168.xxxx.xx"); - paramsMap.add("port","3306"); - paramsMap.add("principal",""); - paramsMap.add("database","dolphinscheduler"); - paramsMap.add("userName","root"); - paramsMap.add("password","root@123"); - paramsMap.add("other",""); + MysqlDatasourceParamDTO mysqlDatasourceParam = new MysqlDatasourceParamDTO(); + mysqlDatasourceParam.setName("mysql"); + mysqlDatasourceParam.setNote("mysql data source test"); + mysqlDatasourceParam.setHost("192.168.xxxx.xx"); + mysqlDatasourceParam.setPort(3306); + mysqlDatasourceParam.setDatabase("dolphinscheduler"); + mysqlDatasourceParam.setUserName("root"); + mysqlDatasourceParam.setPassword("root@123"); + mysqlDatasourceParam.setOther(new HashMap<>()); MvcResult mvcResult = mockMvc.perform(post("/datasources/create") .header("sessionId", sessionId) - .params(paramsMap)) + .contentType(MediaType.APPLICATION_JSON_UTF8) + .content(JSONUtils.toJsonString(mysqlDatasourceParam))) .andExpect(status().isCreated()) .andExpect(content().contentType(MediaType.APPLICATION_JSON_UTF8)) .andReturn(); @@ -91,7 +93,7 @@ public class DataSourceControllerTest extends AbstractControllerTest{ .andExpect(content().contentType(MediaType.APPLICATION_JSON_UTF8)) .andReturn(); Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); - Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); + Assert.assertEquals(Status.SUCCESS.getCode(), result.getCode().intValue()); logger.info(mvcResult.getResponse().getContentAsString()); } diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProcessDefinitionControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProcessDefinitionControllerTest.java index 4a9908c3ea..376f7e3945 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProcessDefinitionControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProcessDefinitionControllerTest.java @@ -383,6 +383,13 @@ public class ProcessDefinitionControllerTest { @Test public void testQueryProcessDefinitionVersions() { String projectName = "test"; + + Result result = processDefinitionController.queryProcessDefinitionVersions(user, projectName, 1, -10, 1); + Assert.assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR.getCode(), result.getCode().intValue()); + + result = processDefinitionController.queryProcessDefinitionVersions(user, projectName, -1, 10, 1); + Assert.assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR.getCode(), result.getCode().intValue()); + Map resultMap = new HashMap<>(); putMsg(resultMap, Status.SUCCESS); resultMap.put(Constants.DATA_LIST, new PageInfo(1, 10)); @@ -393,7 +400,7 @@ public class ProcessDefinitionControllerTest { , 10 , 1)) .thenReturn(resultMap); - Result result = processDefinitionController.queryProcessDefinitionVersions( + result = processDefinitionController.queryProcessDefinitionVersions( user , projectName , 1 @@ -408,17 +415,8 @@ public class ProcessDefinitionControllerTest { String projectName = "test"; Map resultMap = new HashMap<>(); putMsg(resultMap, Status.SUCCESS); - Mockito.when(processDefinitionService.switchProcessDefinitionVersion( - user - , projectName - , 1 - , 10)) - .thenReturn(resultMap); - Result result = processDefinitionController.switchProcessDefinitionVersion( - user - , projectName - , 1 - , 10); + Mockito.when(processDefinitionService.switchProcessDefinitionVersion(user, projectName, 1, 10)).thenReturn(resultMap); + Result result = processDefinitionController.switchProcessDefinitionVersion(user, projectName, 1, 10); Assert.assertEquals(Status.SUCCESS.getCode(), (int) result.getCode()); } @@ -439,7 +437,6 @@ public class ProcessDefinitionControllerTest { , projectName , 1 , 10); - Assert.assertEquals(Status.SUCCESS.getCode(), (int) result.getCode()); } diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProjectControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProjectControllerTest.java index 0f58476cc1..7a4a51a9b6 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProjectControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProjectControllerTest.java @@ -26,7 +26,9 @@ import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.junit.After; import org.junit.Assert; +import org.junit.Before; import org.junit.Ignore; import org.junit.Test; import org.slf4j.Logger; @@ -36,19 +38,32 @@ import org.springframework.test.web.servlet.MvcResult; import org.springframework.util.LinkedMultiValueMap; import org.springframework.util.MultiValueMap; +import com.fasterxml.jackson.core.type.TypeReference; + /** * project controller test */ -public class ProjectControllerTest extends AbstractControllerTest{ +public class ProjectControllerTest extends AbstractControllerTest { private static Logger logger = LoggerFactory.getLogger(ProjectControllerTest.class); - @Test - public void testCreateProject() throws Exception { + private String projectId; + + @Before + public void before() throws Exception { + projectId = testCreateProject("project_test1", "the test project"); + } + + @After + public void after() throws Exception { + testDeleteProject(projectId); + } + + private String testCreateProject(String projectName, String description) throws Exception { MultiValueMap paramsMap = new LinkedMultiValueMap<>(); - paramsMap.add("projectName","project_test1"); - paramsMap.add("description","the test project"); + paramsMap.add("projectName",projectName); + paramsMap.add("description",description); MvcResult mvcResult = mockMvc.perform(post("/projects/create") .header(SESSION_ID, sessionId) @@ -57,17 +72,19 @@ public class ProjectControllerTest extends AbstractControllerTest{ .andExpect(content().contentType(MediaType.APPLICATION_JSON_UTF8)) .andReturn(); - Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); + Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), new TypeReference>() {}); Assert.assertEquals(Status.SUCCESS.getCode(), result.getCode().intValue()); Assert.assertNotNull(result.getData()); - logger.info(mvcResult.getResponse().getContentAsString()); + logger.info("create project return result:{}", mvcResult.getResponse().getContentAsString()); + + return (String)result.getData(); } @Test public void testUpdateProject() throws Exception { MultiValueMap paramsMap = new LinkedMultiValueMap<>(); - paramsMap.add("projectId","18"); + paramsMap.add("projectId", projectId); paramsMap.add("projectName","project_test_update"); paramsMap.add("desc","the test project update"); @@ -80,16 +97,15 @@ public class ProjectControllerTest extends AbstractControllerTest{ Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); - logger.info(mvcResult.getResponse().getContentAsString()); + logger.info("update project return result:{}", mvcResult.getResponse().getContentAsString()); + } - - @Test public void testQueryProjectById() throws Exception { MultiValueMap paramsMap = new LinkedMultiValueMap<>(); - paramsMap.add("projectId","18"); + paramsMap.add("projectId", projectId); MvcResult mvcResult = mockMvc.perform(get("/projects/query-by-id") .header(SESSION_ID, sessionId) @@ -99,8 +115,10 @@ public class ProjectControllerTest extends AbstractControllerTest{ .andReturn(); Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); - Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); + Assert.assertEquals(Status.SUCCESS.getCode(), result.getCode().intValue()); logger.info(mvcResult.getResponse().getContentAsString()); + logger.info("query project by id :{}, return result:{}", projectId, mvcResult.getResponse().getContentAsString()); + } @Test @@ -111,7 +129,6 @@ public class ProjectControllerTest extends AbstractControllerTest{ paramsMap.add("pageSize","2"); paramsMap.add("pageNo","2"); - MvcResult mvcResult = mockMvc.perform(get("/projects/list-paging") .header(SESSION_ID, sessionId) .params(paramsMap)) @@ -121,10 +138,9 @@ public class ProjectControllerTest extends AbstractControllerTest{ Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); - logger.info(mvcResult.getResponse().getContentAsString()); + logger.info("query list-paging project return result:{}", mvcResult.getResponse().getContentAsString()); } - @Test public void testQueryUnauthorizedProject() throws Exception { @@ -140,16 +156,14 @@ public class ProjectControllerTest extends AbstractControllerTest{ Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); - logger.info(mvcResult.getResponse().getContentAsString()); + logger.info("query unauth project return result:{}", mvcResult.getResponse().getContentAsString()); } - - @Test public void testQueryAuthorizedProject() throws Exception { MultiValueMap paramsMap = new LinkedMultiValueMap<>(); - paramsMap.add("userId","2"); + paramsMap.add("userId",String.valueOf(user.getId())); MvcResult mvcResult = mockMvc.perform(get("/projects/authed-project") .header(SESSION_ID, sessionId) @@ -160,9 +174,9 @@ public class ProjectControllerTest extends AbstractControllerTest{ Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); - logger.info(mvcResult.getResponse().getContentAsString()); - } + logger.info("query authed project return result:{}", mvcResult.getResponse().getContentAsString()); + } @Test public void testQueryAllProjectList() throws Exception { @@ -178,7 +192,8 @@ public class ProjectControllerTest extends AbstractControllerTest{ Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); - logger.info(mvcResult.getResponse().getContentAsString()); + logger.info("query all project return result:{}", mvcResult.getResponse().getContentAsString()); + } @Ignore @@ -200,12 +215,10 @@ public class ProjectControllerTest extends AbstractControllerTest{ logger.info(mvcResult.getResponse().getContentAsString()); } - - @Test - public void testDeleteProject() throws Exception { + private void testDeleteProject(String projectId) throws Exception { MultiValueMap paramsMap = new LinkedMultiValueMap<>(); - paramsMap.add("projectId","18"); + paramsMap.add("projectId", projectId); MvcResult mvcResult = mockMvc.perform(get("/projects/delete") .header(SESSION_ID, sessionId) @@ -216,6 +229,7 @@ public class ProjectControllerTest extends AbstractControllerTest{ Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); - logger.info(mvcResult.getResponse().getContentAsString()); + logger.info("delete project return result:{}", mvcResult.getResponse().getContentAsString()); } + } diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/QueueControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/QueueControllerTest.java index cad97f5592..f53fa01586 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/QueueControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/QueueControllerTest.java @@ -38,10 +38,12 @@ import org.springframework.util.MultiValueMap; /** * queue controller test */ -public class QueueControllerTest extends AbstractControllerTest{ +public class QueueControllerTest extends AbstractControllerTest { private static Logger logger = LoggerFactory.getLogger(QueueControllerTest.class); + private static final String QUEUE_CREATE_STRING = "queue1"; + @Test public void testQueryList() throws Exception { @@ -53,14 +55,13 @@ public class QueueControllerTest extends AbstractControllerTest{ Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); - logger.info(mvcResult.getResponse().getContentAsString()); + logger.info("query list queue return result:{}", mvcResult.getResponse().getContentAsString()); } @Test public void testQueryQueueListPaging() throws Exception { MultiValueMap paramsMap = new LinkedMultiValueMap<>(); - //paramsMap.add("processInstanceId","1380"); paramsMap.add("searchVal",""); paramsMap.add("pageNo","1"); paramsMap.add("pageSize","20"); @@ -73,18 +74,16 @@ public class QueueControllerTest extends AbstractControllerTest{ .andReturn(); Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); - logger.info(mvcResult.getResponse().getContentAsString()); + logger.info("query list-paging queue return result:{}", mvcResult.getResponse().getContentAsString()); + } - - - @Test public void testCreateQueue() throws Exception { MultiValueMap paramsMap = new LinkedMultiValueMap<>(); - paramsMap.add("queue","ait"); - paramsMap.add("queueName","ait"); + paramsMap.add("queue", QUEUE_CREATE_STRING); + paramsMap.add("queueName","root.queue1"); MvcResult mvcResult = mockMvc.perform(post("/queue/create") .header(SESSION_ID, sessionId) @@ -93,17 +92,17 @@ public class QueueControllerTest extends AbstractControllerTest{ .andExpect(content().contentType(MediaType.APPLICATION_JSON_UTF8)) .andReturn(); Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); -// Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); - logger.info(mvcResult.getResponse().getContentAsString()); + Assert.assertEquals(Status.SUCCESS.getCode(), result.getCode().intValue()); + logger.info("create queue return result:{}", mvcResult.getResponse().getContentAsString()); } @Test public void testUpdateQueue() throws Exception { MultiValueMap paramsMap = new LinkedMultiValueMap<>(); - paramsMap.add("id","2"); - paramsMap.add("queue","ait12"); - paramsMap.add("queueName","aitName"); + paramsMap.add("id","1"); + paramsMap.add("queue","queue2"); + paramsMap.add("queueName","root.queue2"); MvcResult mvcResult = mockMvc.perform(post("/queue/update") .header(SESSION_ID, sessionId) @@ -112,16 +111,17 @@ public class QueueControllerTest extends AbstractControllerTest{ .andExpect(content().contentType(MediaType.APPLICATION_JSON_UTF8)) .andReturn(); Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); - //Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); - logger.info(mvcResult.getResponse().getContentAsString()); + Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); + logger.info("update queue return result:{}", mvcResult.getResponse().getContentAsString()); } @Test public void testVerifyQueue() throws Exception { + // queue value exist MultiValueMap paramsMap = new LinkedMultiValueMap<>(); - paramsMap.add("queue","ait123"); - paramsMap.add("queueName","aitName"); + paramsMap.add("queue",QUEUE_CREATE_STRING); + paramsMap.add("queueName","queue.name"); MvcResult mvcResult = mockMvc.perform(post("/queue/verify-queue") .header(SESSION_ID, sessionId) @@ -130,7 +130,22 @@ public class QueueControllerTest extends AbstractControllerTest{ .andExpect(content().contentType(MediaType.APPLICATION_JSON_UTF8)) .andReturn(); Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); - //Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); + Assert.assertEquals(Status.QUEUE_VALUE_EXIST.getCode(),result.getCode().intValue()); + + // success + paramsMap.clear(); + paramsMap.add("queue","ait123"); + paramsMap.add("queueName","aitName"); + + mvcResult = mockMvc.perform(post("/queue/verify-queue") + .header(SESSION_ID, sessionId) + .params(paramsMap)) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.APPLICATION_JSON_UTF8)) + .andReturn(); + result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); + Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); logger.info(mvcResult.getResponse().getContentAsString()); + logger.info("verify queue return result:{}", mvcResult.getResponse().getContentAsString()); } } diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/interceptor/RateLimitInterceptorTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/interceptor/RateLimitInterceptorTest.java new file mode 100644 index 0000000000..95ae3e3779 --- /dev/null +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/interceptor/RateLimitInterceptorTest.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.interceptor; + +import org.apache.dolphinscheduler.api.configuration.TrafficConfiguration; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.ExecutionException; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.modules.junit4.PowerMockRunner; + +@RunWith(PowerMockRunner.class) +public class RateLimitInterceptorTest { + + @Test + public void testPreHandleWithoutControl() throws ExecutionException { + HttpServletRequest request = Mockito.mock(HttpServletRequest.class); + HttpServletResponse response = Mockito.mock(HttpServletResponse.class); + RateLimitInterceptor rateLimitInterceptor = new RateLimitInterceptor(new TrafficConfiguration()); + Assert.assertTrue(rateLimitInterceptor.preHandle(request, response, null)); + Assert.assertTrue(rateLimitInterceptor.preHandle(request, response, null)); + } + + @Test + public void testPreHandleWithTenantLevenControl() throws ExecutionException { + TrafficConfiguration trafficConfiguration = new TrafficConfiguration(); + trafficConfiguration.setTrafficTenantControlSwitch(true); + Map map = new HashMap<>(); + map.put("tenant1", 2); + map.put("tenant2", 2); + trafficConfiguration.setCustomizeTenantQpsRate(map); + trafficConfiguration.setDefaultTenantQpsRate(4); + RateLimitInterceptor rateLimitInterceptor = new RateLimitInterceptor(trafficConfiguration); + + HttpServletRequest tenant1Request = Mockito.mock(HttpServletRequest.class); + HttpServletRequest tenant2Request = Mockito.mock(HttpServletRequest.class); + PowerMockito.when(tenant1Request.getHeader(Mockito.any())).thenReturn("tenant1"); + PowerMockito.when(tenant2Request.getHeader(Mockito.any())).thenReturn("tenant2"); + HttpServletResponse response = Mockito.mock(HttpServletResponse.class); + + for (int i = 0; i < 2; i++) { + rateLimitInterceptor.preHandle(tenant1Request, response, null); + } + Assert.assertFalse(rateLimitInterceptor.preHandle(tenant1Request, response, null)); + Assert.assertTrue(rateLimitInterceptor.preHandle(tenant2Request, response, null)); + } + + @Test + public void testPreHandleWithGlobalControl() throws ExecutionException { + TrafficConfiguration trafficConfiguration = new TrafficConfiguration(); + trafficConfiguration.setTrafficTenantControlSwitch(true); + trafficConfiguration.setTrafficGlobalControlSwitch(true); + trafficConfiguration.setMaxGlobalQpsRate(3); + + RateLimitInterceptor rateLimitInterceptor = new RateLimitInterceptor(trafficConfiguration); + HttpServletRequest request = Mockito.mock(HttpServletRequest.class); + HttpServletResponse response = Mockito.mock(HttpServletResponse.class); + + for (int i = 0; i < 2; i++) { + rateLimitInterceptor.preHandle(request, response, null); + } + Assert.assertFalse(rateLimitInterceptor.preHandle(request, response, null)); + } + +} \ No newline at end of file diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataSourceServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataSourceServiceTest.java index f127b9d98b..855ae833e8 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataSourceServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataSourceServiceTest.java @@ -21,15 +21,18 @@ import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.service.impl.DataSourceServiceImpl; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.datasource.ConnectionParam; +import org.apache.dolphinscheduler.common.datasource.DatasourceUtil; +import org.apache.dolphinscheduler.common.datasource.hive.HiveDataSourceParamDTO; +import org.apache.dolphinscheduler.common.datasource.mysql.MysqlDatasourceParamDTO; +import org.apache.dolphinscheduler.common.datasource.oracle.OracleDatasourceParamDTO; +import org.apache.dolphinscheduler.common.datasource.postgresql.PostgreSqlDatasourceParamDTO; import org.apache.dolphinscheduler.common.enums.DbConnectType; import org.apache.dolphinscheduler.common.enums.DbType; import org.apache.dolphinscheduler.common.enums.UserType; import org.apache.dolphinscheduler.common.utils.CommonUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.PropertyUtils; -import org.apache.dolphinscheduler.dao.datasource.BaseDataSource; -import org.apache.dolphinscheduler.dao.datasource.DataSourceFactory; -import org.apache.dolphinscheduler.dao.datasource.MySQLDataSource; import org.apache.dolphinscheduler.dao.entity.DataSource; import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.mapper.DataSourceMapper; @@ -37,6 +40,7 @@ import org.apache.dolphinscheduler.dao.mapper.DataSourceUserMapper; import java.sql.Connection; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; @@ -56,7 +60,7 @@ import org.powermock.modules.junit4.PowerMockRunner; */ @RunWith(PowerMockRunner.class) @PowerMockIgnore({"sun.security.*", "javax.net.*"}) -@PrepareForTest({DataSourceFactory.class, CommonUtils.class}) +@PrepareForTest({DatasourceUtil.class, CommonUtils.class}) public class DataSourceServiceTest { @InjectMocks @@ -70,11 +74,17 @@ public class DataSourceServiceTest { public void createDataSourceTest() { User loginUser = getAdminUser(); - String dataSourceName = "dataSource01"; String dataSourceDesc = "test dataSource"; - DbType dataSourceType = DbType.POSTGRESQL; - String parameter = dataSourceService.buildParameter(dataSourceType, "172.16.133.200", "5432", "dolphinscheduler", null, "postgres", "", null, null, null, null, null); + + PostgreSqlDatasourceParamDTO postgreSqlDatasourceParam = new PostgreSqlDatasourceParamDTO(); + postgreSqlDatasourceParam.setDatabase(dataSourceName); + postgreSqlDatasourceParam.setNote(dataSourceDesc); + postgreSqlDatasourceParam.setHost("172.16.133.200"); + postgreSqlDatasourceParam.setPort(5432); + postgreSqlDatasourceParam.setDatabase("dolphinscheduler"); + postgreSqlDatasourceParam.setUserName("postgres"); + postgreSqlDatasourceParam.setPassword(""); // data source exits List dataSourceList = new ArrayList<>(); @@ -82,30 +92,30 @@ public class DataSourceServiceTest { dataSource.setName(dataSourceName); dataSourceList.add(dataSource); PowerMockito.when(dataSourceMapper.queryDataSourceByName(dataSourceName.trim())).thenReturn(dataSourceList); - Result dataSourceExitsResult = dataSourceService.createDataSource(loginUser, dataSourceName, dataSourceDesc, dataSourceType, parameter); + Result dataSourceExitsResult = dataSourceService.createDataSource(loginUser, postgreSqlDatasourceParam); Assert.assertEquals(Status.DATASOURCE_EXIST.getCode(), dataSourceExitsResult.getCode().intValue()); + ConnectionParam connectionParam = DatasourceUtil.buildConnectionParams(postgreSqlDatasourceParam); + DbType dataSourceType = postgreSqlDatasourceParam.getType(); // data source exits PowerMockito.when(dataSourceMapper.queryDataSourceByName(dataSourceName.trim())).thenReturn(null); Result connectionResult = new Result(Status.DATASOURCE_CONNECT_FAILED.getCode(), Status.DATASOURCE_CONNECT_FAILED.getMsg()); //PowerMockito.when(dataSourceService.checkConnection(dataSourceType, parameter)).thenReturn(connectionResult); - PowerMockito.doReturn(connectionResult).when(dataSourceService).checkConnection(dataSourceType, parameter); - Result connectFailedResult = dataSourceService.createDataSource(loginUser, dataSourceName, dataSourceDesc, dataSourceType, parameter); + PowerMockito.doReturn(connectionResult).when(dataSourceService).checkConnection(dataSourceType, connectionParam); + Result connectFailedResult = dataSourceService.createDataSource(loginUser, postgreSqlDatasourceParam); Assert.assertEquals(Status.DATASOURCE_CONNECT_FAILED.getCode(), connectFailedResult.getCode().intValue()); // data source exits PowerMockito.when(dataSourceMapper.queryDataSourceByName(dataSourceName.trim())).thenReturn(null); connectionResult = new Result(Status.SUCCESS.getCode(), Status.SUCCESS.getMsg()); - PowerMockito.when(dataSourceService.checkConnection(dataSourceType, parameter)).thenReturn(connectionResult); - PowerMockito.when(DataSourceFactory.getDatasource(dataSourceType, parameter)).thenReturn(null); - Result notValidError = dataSourceService.createDataSource(loginUser, dataSourceName, dataSourceDesc, dataSourceType, parameter); + PowerMockito.when(dataSourceService.checkConnection(dataSourceType, connectionParam)).thenReturn(connectionResult); + Result notValidError = dataSourceService.createDataSource(loginUser, postgreSqlDatasourceParam); Assert.assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR.getCode(), notValidError.getCode().intValue()); // success PowerMockito.when(dataSourceMapper.queryDataSourceByName(dataSourceName.trim())).thenReturn(null); - PowerMockito.when(dataSourceService.checkConnection(dataSourceType, parameter)).thenReturn(connectionResult); - PowerMockito.when(DataSourceFactory.getDatasource(dataSourceType, parameter)).thenReturn(JSONUtils.parseObject(parameter, MySQLDataSource.class)); - Result success = dataSourceService.createDataSource(loginUser, dataSourceName, dataSourceDesc, dataSourceType, parameter); + PowerMockito.when(dataSourceService.checkConnection(dataSourceType, connectionParam)).thenReturn(connectionResult); + Result success = dataSourceService.createDataSource(loginUser, postgreSqlDatasourceParam); Assert.assertEquals(Status.SUCCESS.getCode(), success.getCode().intValue()); } @@ -115,18 +125,25 @@ public class DataSourceServiceTest { int dataSourceId = 12; String dataSourceName = "dataSource01"; String dataSourceDesc = "test dataSource"; - DbType dataSourceType = DbType.POSTGRESQL; - String parameter = dataSourceService.buildParameter(dataSourceType, "172.16.133.200", "5432", "dolphinscheduler", null, "postgres", "", null, null, null, null, null); + + PostgreSqlDatasourceParamDTO postgreSqlDatasourceParam = new PostgreSqlDatasourceParamDTO(); + postgreSqlDatasourceParam.setDatabase(dataSourceName); + postgreSqlDatasourceParam.setNote(dataSourceDesc); + postgreSqlDatasourceParam.setHost("172.16.133.200"); + postgreSqlDatasourceParam.setPort(5432); + postgreSqlDatasourceParam.setDatabase("dolphinscheduler"); + postgreSqlDatasourceParam.setUserName("postgres"); + postgreSqlDatasourceParam.setPassword(""); // data source not exits PowerMockito.when(dataSourceMapper.selectById(dataSourceId)).thenReturn(null); - Result resourceNotExits = dataSourceService.updateDataSource(dataSourceId, loginUser, dataSourceName, dataSourceDesc, dataSourceType, parameter); + Result resourceNotExits = dataSourceService.updateDataSource(dataSourceId, loginUser, postgreSqlDatasourceParam); Assert.assertEquals(Status.RESOURCE_NOT_EXIST.getCode(), resourceNotExits.getCode().intValue()); // user no operation perm DataSource dataSource = new DataSource(); dataSource.setUserId(0); PowerMockito.when(dataSourceMapper.selectById(dataSourceId)).thenReturn(dataSource); - Result userNoOperationPerm = dataSourceService.updateDataSource(dataSourceId, loginUser, dataSourceName, dataSourceDesc, dataSourceType, parameter); + Result userNoOperationPerm = dataSourceService.updateDataSource(dataSourceId, loginUser, postgreSqlDatasourceParam); Assert.assertEquals(Status.USER_NO_OPERATION_PERM.getCode(), userNoOperationPerm.getCode().intValue()); // data source name exits @@ -135,23 +152,25 @@ public class DataSourceServiceTest { dataSourceList.add(dataSource); PowerMockito.when(dataSourceMapper.selectById(dataSourceId)).thenReturn(dataSource); PowerMockito.when(dataSourceMapper.queryDataSourceByName(dataSourceName)).thenReturn(dataSourceList); - Result dataSourceNameExist = dataSourceService.updateDataSource(dataSourceId, loginUser, dataSourceName, dataSourceDesc, dataSourceType, parameter); + Result dataSourceNameExist = dataSourceService.updateDataSource(dataSourceId, loginUser, postgreSqlDatasourceParam); Assert.assertEquals(Status.DATASOURCE_EXIST.getCode(), dataSourceNameExist.getCode().intValue()); // data source connect failed + DbType dataSourceType = postgreSqlDatasourceParam.getType(); + ConnectionParam connectionParam = DatasourceUtil.buildConnectionParams(postgreSqlDatasourceParam); PowerMockito.when(dataSourceMapper.selectById(dataSourceId)).thenReturn(dataSource); PowerMockito.when(dataSourceMapper.queryDataSourceByName(dataSourceName)).thenReturn(null); Result connectionResult = new Result(Status.SUCCESS.getCode(), Status.SUCCESS.getMsg()); - PowerMockito.when(dataSourceService.checkConnection(dataSourceType, parameter)).thenReturn(connectionResult); - Result connectFailed = dataSourceService.updateDataSource(dataSourceId, loginUser, dataSourceName, dataSourceDesc, dataSourceType, parameter); + PowerMockito.when(dataSourceService.checkConnection(dataSourceType, connectionParam)).thenReturn(connectionResult); + Result connectFailed = dataSourceService.updateDataSource(dataSourceId, loginUser, postgreSqlDatasourceParam); Assert.assertEquals(Status.DATASOURCE_CONNECT_FAILED.getCode(), connectFailed.getCode().intValue()); //success PowerMockito.when(dataSourceMapper.selectById(dataSourceId)).thenReturn(dataSource); PowerMockito.when(dataSourceMapper.queryDataSourceByName(dataSourceName)).thenReturn(null); connectionResult = new Result(Status.DATASOURCE_CONNECT_FAILED.getCode(), Status.DATASOURCE_CONNECT_FAILED.getMsg()); - PowerMockito.when(dataSourceService.checkConnection(dataSourceType, parameter)).thenReturn(connectionResult); - Result success = dataSourceService.updateDataSource(dataSourceId, loginUser, dataSourceName, dataSourceDesc, dataSourceType, parameter); + PowerMockito.when(dataSourceService.checkConnection(dataSourceType, connectionParam)).thenReturn(connectionResult); + Result success = dataSourceService.updateDataSource(dataSourceId, loginUser, postgreSqlDatasourceParam); Assert.assertEquals(Status.SUCCESS.getCode(), success.getCode().intValue()); } @@ -279,40 +298,69 @@ public class DataSourceServiceTest { @Test public void buildParameter() { - String param = dataSourceService.buildParameter(DbType.ORACLE, "192.168.9.1", "1521", "im" - , "", "test", "test", DbConnectType.ORACLE_SERVICE_NAME, "", "", "", ""); - String expected = "{\"connectType\":\"ORACLE_SERVICE_NAME\",\"address\":\"jdbc:oracle:thin:@//192.168.9.1:1521\",\"database\":\"im\"," - + "\"jdbcUrl\":\"jdbc:oracle:thin:@//192.168.9.1:1521/im\",\"user\":\"test\",\"password\":\"test\"}"; - Assert.assertEquals(expected, param); + OracleDatasourceParamDTO oracleDatasourceParamDTO = new OracleDatasourceParamDTO(); + oracleDatasourceParamDTO.setHost("192.168.9.1"); + oracleDatasourceParamDTO.setPort(1521); + oracleDatasourceParamDTO.setDatabase("im"); + oracleDatasourceParamDTO.setUserName("test"); + oracleDatasourceParamDTO.setPassword("test"); + oracleDatasourceParamDTO.setConnectType(DbConnectType.ORACLE_SERVICE_NAME); + + ConnectionParam connectionParam = DatasourceUtil.buildConnectionParams(oracleDatasourceParamDTO); + String expected = "{\"user\":\"test\",\"password\":\"test\",\"address\":\"jdbc:oracle:thin:@//192.168.9.1:1521\"," + + "\"database\":\"im\",\"jdbcUrl\":\"jdbc:oracle:thin:@//192.168.9.1:1521/im\",\"connectType\":\"ORACLE_SERVICE_NAME\"}"; + Assert.assertEquals(expected, JSONUtils.toJsonString(connectionParam)); PowerMockito.mockStatic(CommonUtils.class); PowerMockito.when(CommonUtils.getKerberosStartupState()).thenReturn(true); PowerMockito.when(CommonUtils.encodePassword(Mockito.anyString())).thenReturn("test"); - param = dataSourceService.buildParameter(DbType.HIVE, "192.168.9.1", "10000", "im" - , "hive/hdfs-mycluster@ESZ.COM", "test", "test", null, "", "/opt/krb5.conf", "test2/hdfs-mycluster@ESZ.COM", "/opt/hdfs.headless.keytab"); - expected = "{\"address\":\"jdbc:hive2://192.168.9.1:10000\",\"database\":\"im\",\"jdbcUrl\":\"jdbc:hive2://192.168.9.1:10000/im;principal=hive/hdfs-mycluster@ESZ.COM\"," - + "\"user\":\"test\",\"password\":\"test\",\"principal\":\"hive/hdfs-mycluster@ESZ.COM\",\"javaSecurityKrb5Conf\":\"/opt/krb5.conf\"," - + "\"loginUserKeytabUsername\":\"test2/hdfs-mycluster@ESZ.COM\",\"loginUserKeytabPath\":\"/opt/hdfs.headless.keytab\"}"; - Assert.assertEquals(expected, param); + HiveDataSourceParamDTO hiveDataSourceParamDTO = new HiveDataSourceParamDTO(); + hiveDataSourceParamDTO.setHost("192.168.9.1"); + hiveDataSourceParamDTO.setPort(10000); + hiveDataSourceParamDTO.setDatabase("im"); + hiveDataSourceParamDTO.setPrincipal("hive/hdfs-mycluster@ESZ.COM"); + hiveDataSourceParamDTO.setUserName("test"); + hiveDataSourceParamDTO.setPassword("test"); + hiveDataSourceParamDTO.setJavaSecurityKrb5Conf("/opt/krb5.conf"); + hiveDataSourceParamDTO.setLoginUserKeytabPath("/opt/hdfs.headless.keytab"); + hiveDataSourceParamDTO.setLoginUserKeytabUsername("test2/hdfs-mycluster@ESZ.COM"); + connectionParam = DatasourceUtil.buildConnectionParams(hiveDataSourceParamDTO); + expected = "{\"user\":\"test\",\"password\":\"test\",\"address\":\"jdbc:hive2://192.168.9.1:10000\"," + + "\"database\":\"im\",\"jdbcUrl\":\"jdbc:hive2://192.168.9.1:10000/im;principal=hive/hdfs-mycluster@ESZ.COM\",\"principal\":\"hive/hdfs-mycluster@ESZ.COM\"," + + "\"javaSecurityKrb5Conf\":\"/opt/krb5.conf\",\"loginUserKeytabUsername\":\"test2/hdfs-mycluster@ESZ.COM\",\"loginUserKeytabPath\":\"/opt/hdfs.headless.keytab\"}"; + Assert.assertEquals(expected, JSONUtils.toJsonString(connectionParam)); } @Test public void buildParameterWithDecodePassword() { PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE, "true"); - String other = "{\"autoDeserialize\":\"yes\",\"allowUrlInLocalInfile\":\"true\"}"; - String param = dataSourceService.buildParameter(DbType.MYSQL, "192.168.9.1", "1521", "im" - , "", "test", "123456", null, other, "", "", ""); - String expected = "{\"address\":\"jdbc:mysql://192.168.9.1:1521\",\"database\":\"im\",\"jdbcUrl\":\"jdbc:mysql://192.168.9.1:1521/im\"," - + "\"user\":\"test\",\"password\":\"IUAjJCVeJipNVEl6TkRVMg==\"}"; - Assert.assertEquals(expected, param); + Map other = new HashMap<>(); + other.put("autoDeserialize", "yes"); + other.put("allowUrlInLocalInfile", "true"); + MysqlDatasourceParamDTO mysqlDatasourceParamDTO = new MysqlDatasourceParamDTO(); + mysqlDatasourceParamDTO.setHost("192.168.9.1"); + mysqlDatasourceParamDTO.setPort(1521); + mysqlDatasourceParamDTO.setDatabase("im"); + mysqlDatasourceParamDTO.setUserName("test"); + mysqlDatasourceParamDTO.setPassword("123456"); + mysqlDatasourceParamDTO.setOther(other); + ConnectionParam connectionParam = DatasourceUtil.buildConnectionParams(mysqlDatasourceParamDTO); + String expected = "{\"user\":\"test\",\"password\":\"IUAjJCVeJipNVEl6TkRVMg==\",\"address\":\"jdbc:mysql://192.168.9.1:1521\"," + + "\"database\":\"im\",\"jdbcUrl\":\"jdbc:mysql://192.168.9.1:1521/im\"}"; + Assert.assertEquals(expected, JSONUtils.toJsonString(connectionParam)); PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE, "false"); - param = dataSourceService.buildParameter(DbType.MYSQL, "192.168.9.1", "1521", "im" - , "", "test", "123456", null, "", "", "", ""); - expected = "{\"address\":\"jdbc:mysql://192.168.9.1:1521\",\"database\":\"im\",\"jdbcUrl\":\"jdbc:mysql://192.168.9.1:1521/im\"," - + "\"user\":\"test\",\"password\":\"123456\"}"; - Assert.assertEquals(expected, param); + mysqlDatasourceParamDTO = new MysqlDatasourceParamDTO(); + mysqlDatasourceParamDTO.setHost("192.168.9.1"); + mysqlDatasourceParamDTO.setPort(1521); + mysqlDatasourceParamDTO.setDatabase("im"); + mysqlDatasourceParamDTO.setUserName("test"); + mysqlDatasourceParamDTO.setPassword("123456"); + connectionParam = DatasourceUtil.buildConnectionParams(mysqlDatasourceParamDTO); + expected = "{\"user\":\"test\",\"password\":\"123456\",\"address\":\"jdbc:mysql://192.168.9.1:1521\"," + + "\"database\":\"im\",\"jdbcUrl\":\"jdbc:mysql://192.168.9.1:1521/im\"}"; + Assert.assertEquals(expected, JSONUtils.toJsonString(connectionParam)); } /** @@ -334,22 +382,27 @@ public class DataSourceServiceTest { @Test public void testCheckConnection() throws Exception { DbType dataSourceType = DbType.POSTGRESQL; - String parameter = dataSourceService.buildParameter(dataSourceType, "172.16.133.200", "5432", "dolphinscheduler", null, "postgres", "", null, null, null, null, null); + String dataSourceName = "dataSource01"; + String dataSourceDesc = "test dataSource"; - PowerMockito.mockStatic(DataSourceFactory.class); - PowerMockito.when(DataSourceFactory.getDatasource(Mockito.any(), Mockito.anyString())).thenReturn(null); - Result result = dataSourceService.checkConnection(dataSourceType, parameter); - Assert.assertEquals(Status.DATASOURCE_TYPE_NOT_EXIST.getCode(), result.getCode().intValue()); + PostgreSqlDatasourceParamDTO postgreSqlDatasourceParam = new PostgreSqlDatasourceParamDTO(); + postgreSqlDatasourceParam.setDatabase(dataSourceName); + postgreSqlDatasourceParam.setNote(dataSourceDesc); + postgreSqlDatasourceParam.setHost("172.16.133.200"); + postgreSqlDatasourceParam.setPort(5432); + postgreSqlDatasourceParam.setDatabase("dolphinscheduler"); + postgreSqlDatasourceParam.setUserName("postgres"); + postgreSqlDatasourceParam.setPassword(""); + ConnectionParam connectionParam = DatasourceUtil.buildConnectionParams(postgreSqlDatasourceParam); - BaseDataSource dataSource = PowerMockito.mock(BaseDataSource.class); - PowerMockito.when(DataSourceFactory.getDatasource(Mockito.any(), Mockito.anyString())).thenReturn(dataSource); - PowerMockito.when(dataSource.getConnection()).thenReturn(null); - result = dataSourceService.checkConnection(dataSourceType, parameter); + PowerMockito.mockStatic(DatasourceUtil.class); + + Result result = dataSourceService.checkConnection(dataSourceType, connectionParam); Assert.assertEquals(Status.CONNECTION_TEST_FAILURE.getCode(), result.getCode().intValue()); Connection connection = PowerMockito.mock(Connection.class); - PowerMockito.when(dataSource.getConnection()).thenReturn(connection); - result = dataSourceService.checkConnection(dataSourceType, parameter); + PowerMockito.when(DatasourceUtil.getConnection(Mockito.any(), Mockito.any())).thenReturn(connection); + result = dataSourceService.checkConnection(dataSourceType, connectionParam); Assert.assertEquals(Status.SUCCESS.getCode(), result.getCode().intValue()); } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java index 148df29cc0..1544f2b638 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java @@ -380,21 +380,11 @@ public final class Constants { */ public static final int DEFAULT_MASTER_CPU_LOAD = Runtime.getRuntime().availableProcessors() * 2; - /** - * master reserved memory - */ - public static final double DEFAULT_MASTER_RESERVED_MEMORY = OSUtils.totalMemorySize() / 10; - /** * worker cpu load */ public static final int DEFAULT_WORKER_CPU_LOAD = Runtime.getRuntime().availableProcessors() * 2; - /** - * worker reserved memory - */ - public static final double DEFAULT_WORKER_RESERVED_MEMORY = OSUtils.totalMemorySize() / 10; - /** * worker host weight */ diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/AbstractDatasourceProcessor.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/AbstractDatasourceProcessor.java new file mode 100644 index 0000000000..d03c13d864 --- /dev/null +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/AbstractDatasourceProcessor.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.common.datasource; + +import org.apache.commons.collections4.MapUtils; + +import java.util.Map; +import java.util.regex.Pattern; + +public abstract class AbstractDatasourceProcessor implements DatasourceProcessor { + + private static final Pattern IPV4_PATTERN = Pattern.compile("^[a-zA-Z0-9\\_\\-\\.]+$"); + + private static final Pattern IPV6_PATTERN = Pattern.compile("^[a-zA-Z0-9\\_\\-\\.\\:\\[\\]]+$"); + + private static final Pattern DATABASE_PATTER = Pattern.compile("^[a-zA-Z0-9\\_\\-\\.]+$"); + + private static final Pattern PARAMS_PATTER = Pattern.compile("^[a-zA-Z0-9]+$"); + + @Override + public void checkDatasourceParam(BaseDataSourceParamDTO baseDataSourceParamDTO) { + checkHost(baseDataSourceParamDTO.getHost()); + checkDatasourcePatter(baseDataSourceParamDTO.getDatabase()); + checkOther(baseDataSourceParamDTO.getOther()); + } + + /** + * Check the host is valid + * + * @param host datasource host + */ + protected void checkHost(String host) { + if (!IPV4_PATTERN.matcher(host).matches() || !IPV6_PATTERN.matcher(host).matches()) { + throw new IllegalArgumentException("datasource host illegal"); + } + } + + /** + * check database name is valid + * + * @param database database name + */ + protected void checkDatasourcePatter(String database) { + if (!DATABASE_PATTER.matcher(database).matches()) { + throw new IllegalArgumentException("datasource name illegal"); + } + } + + /** + * check other is valid + * + * @param other other + */ + protected void checkOther(Map other) { + if (MapUtils.isEmpty(other)) { + return; + } + boolean paramsCheck = other.entrySet().stream().allMatch(p -> PARAMS_PATTER.matcher(p.getValue()).matches()); + if (!paramsCheck) { + throw new IllegalArgumentException("datasource other params illegal"); + } + } + +} diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/BaseConnectionParam.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/BaseConnectionParam.java new file mode 100644 index 0000000000..ab6e4a84ab --- /dev/null +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/BaseConnectionParam.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.common.datasource; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonInclude.Include; + +/** + * The base model of connection param + *

+ * {@link org.apache.dolphinscheduler.common.datasource.clickhouse.ClickhouseConnectionParam} + * {@link org.apache.dolphinscheduler.common.datasource.db2.Db2ConnectionParam} + * {@link org.apache.dolphinscheduler.common.datasource.hive.HiveConnectionParam} + * {@link org.apache.dolphinscheduler.common.datasource.mysql.MysqlConnectionParam} + * {@link org.apache.dolphinscheduler.common.datasource.oracle.OracleConnectionParam} + * {@link org.apache.dolphinscheduler.common.datasource.postgresql.PostgreSqlConnectionParam} + * {@link org.apache.dolphinscheduler.common.datasource.presto.PrestoConnectionParam} + * {@link org.apache.dolphinscheduler.common.datasource.spark.SparkConnectionParam} + * {@link org.apache.dolphinscheduler.common.datasource.sqlserver.SqlServerConnectionParam} + */ +@JsonInclude(Include.NON_NULL) +public abstract class BaseConnectionParam implements ConnectionParam { + + protected String user; + + protected String password; + + protected String address; + + protected String database; + + protected String jdbcUrl; + + protected String other; + + public String getUser() { + return user; + } + + public void setUser(String user) { + this.user = user; + } + + public String getPassword() { + return password; + } + + public void setPassword(String password) { + this.password = password; + } + + public String getAddress() { + return address; + } + + public void setAddress(String address) { + this.address = address; + } + + public String getDatabase() { + return database; + } + + public void setDatabase(String database) { + this.database = database; + } + + public String getJdbcUrl() { + return jdbcUrl; + } + + public void setJdbcUrl(String jdbcUrl) { + this.jdbcUrl = jdbcUrl; + } + + public String getOther() { + return other; + } + + public void setOther(String other) { + this.other = other; + } +} diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/BaseDataSourceParamDTO.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/BaseDataSourceParamDTO.java new file mode 100644 index 0000000000..8bc3f94379 --- /dev/null +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/BaseDataSourceParamDTO.java @@ -0,0 +1,161 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.common.datasource; + +import org.apache.dolphinscheduler.common.datasource.clickhouse.ClickHouseDatasourceParamDTO; +import org.apache.dolphinscheduler.common.datasource.db2.Db2DatasourceParamDTO; +import org.apache.dolphinscheduler.common.datasource.hive.HiveDataSourceParamDTO; +import org.apache.dolphinscheduler.common.datasource.mysql.MysqlDatasourceParamDTO; +import org.apache.dolphinscheduler.common.datasource.oracle.OracleDatasourceParamDTO; +import org.apache.dolphinscheduler.common.datasource.postgresql.PostgreSqlDatasourceParamDTO; +import org.apache.dolphinscheduler.common.datasource.presto.PrestoDatasourceParamDTO; +import org.apache.dolphinscheduler.common.datasource.spark.SparkDatasourceParamDTO; +import org.apache.dolphinscheduler.common.datasource.sqlserver.SqlServerDatasourceParamDTO; +import org.apache.dolphinscheduler.common.enums.DbType; + +import java.io.Serializable; +import java.util.Map; + +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; + +/** + * Basic datasource params submitted to api. + *

+ * see {@link MysqlDatasourceParamDTO} + * see {@link PostgreSqlDatasourceParamDTO} + * see {@link HiveDataSourceParamDTO} + * see {@link SparkDatasourceParamDTO} + * see {@link ClickHouseDatasourceParamDTO} + * see {@link OracleDatasourceParamDTO} + * see {@link SqlServerDatasourceParamDTO} + * see {@link Db2DatasourceParamDTO} + * see {@link PrestoDatasourceParamDTO} + */ +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type") +@JsonSubTypes(value = { + @JsonSubTypes.Type(value = MysqlDatasourceParamDTO.class, name = "MYSQL"), + @JsonSubTypes.Type(value = PostgreSqlDatasourceParamDTO.class, name = "POSTGRESQL"), + @JsonSubTypes.Type(value = HiveDataSourceParamDTO.class, name = "HIVE"), + @JsonSubTypes.Type(value = SparkDatasourceParamDTO.class, name = "SPARK"), + @JsonSubTypes.Type(value = ClickHouseDatasourceParamDTO.class, name = "CLICKHOUSE"), + @JsonSubTypes.Type(value = OracleDatasourceParamDTO.class, name = "ORACLE"), + @JsonSubTypes.Type(value = SqlServerDatasourceParamDTO.class, name = "SQLSERVER"), + @JsonSubTypes.Type(value = Db2DatasourceParamDTO.class, name = "DB2"), + @JsonSubTypes.Type(value = PrestoDatasourceParamDTO.class, name = "PRESTO"), +}) +public abstract class BaseDataSourceParamDTO implements Serializable { + + protected Integer id; + + protected String name; + + protected String note; + + protected String host; + + protected Integer port; + + protected String database; + + protected String userName; + + protected String password; + + protected Map other; + + public Integer getId() { + return id; + } + + public void setId(Integer id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getNote() { + return note; + } + + public void setNote(String note) { + this.note = note; + } + + public String getHost() { + return host; + } + + public void setHost(String host) { + this.host = host; + } + + public Integer getPort() { + return port; + } + + public void setPort(Integer port) { + this.port = port; + } + + public String getDatabase() { + return database; + } + + public void setDatabase(String database) { + this.database = database; + } + + public String getUserName() { + return userName; + } + + public void setUserName(String userName) { + this.userName = userName; + } + + public String getPassword() { + return password; + } + + public void setPassword(String password) { + this.password = password; + } + + public Map getOther() { + return other; + } + + public void setOther(Map other) { + this.other = other; + } + + /** + * Get the datasource type + * see{@link DbType} + * + * @return datasource type code + */ + public abstract DbType getType(); +} diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/BaseHdfsConnectionParam.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/BaseHdfsConnectionParam.java new file mode 100644 index 0000000000..908b7c8cef --- /dev/null +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/BaseHdfsConnectionParam.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.common.datasource; + +public class BaseHdfsConnectionParam extends BaseConnectionParam { + protected String principal; + protected String javaSecurityKrb5Conf; + protected String loginUserKeytabUsername; + protected String loginUserKeytabPath; + + public String getPrincipal() { + return principal; + } + + public void setPrincipal(String principal) { + this.principal = principal; + } + + public String getJavaSecurityKrb5Conf() { + return javaSecurityKrb5Conf; + } + + public void setJavaSecurityKrb5Conf(String javaSecurityKrb5Conf) { + this.javaSecurityKrb5Conf = javaSecurityKrb5Conf; + } + + public String getLoginUserKeytabUsername() { + return loginUserKeytabUsername; + } + + public void setLoginUserKeytabUsername(String loginUserKeytabUsername) { + this.loginUserKeytabUsername = loginUserKeytabUsername; + } + + public String getLoginUserKeytabPath() { + return loginUserKeytabPath; + } + + public void setLoginUserKeytabPath(String loginUserKeytabPath) { + this.loginUserKeytabPath = loginUserKeytabPath; + } +} diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/BaseHdfsDatasourceParamDTO.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/BaseHdfsDatasourceParamDTO.java new file mode 100644 index 0000000000..747d3ea274 --- /dev/null +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/BaseHdfsDatasourceParamDTO.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.common.datasource; + +public abstract class BaseHdfsDatasourceParamDTO extends BaseDataSourceParamDTO { + + protected String principal; + + protected String javaSecurityKrb5Conf; + + protected String loginUserKeytabUsername; + + protected String loginUserKeytabPath; + + public String getPrincipal() { + return principal; + } + + public void setPrincipal(String principal) { + this.principal = principal; + } + + public String getLoginUserKeytabUsername() { + return loginUserKeytabUsername; + } + + public void setLoginUserKeytabUsername(String loginUserKeytabUsername) { + this.loginUserKeytabUsername = loginUserKeytabUsername; + } + + public String getLoginUserKeytabPath() { + return loginUserKeytabPath; + } + + public void setLoginUserKeytabPath(String loginUserKeytabPath) { + this.loginUserKeytabPath = loginUserKeytabPath; + } + + public String getJavaSecurityKrb5Conf() { + return javaSecurityKrb5Conf; + } + + public void setJavaSecurityKrb5Conf(String javaSecurityKrb5Conf) { + this.javaSecurityKrb5Conf = javaSecurityKrb5Conf; + } +} diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/PrestoDataSource.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/ConnectionParam.java similarity index 63% rename from dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/PrestoDataSource.java rename to dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/ConnectionParam.java index 93ed3d61a5..d4ec697751 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/PrestoDataSource.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/ConnectionParam.java @@ -14,26 +14,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.dolphinscheduler.dao.datasource; -import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.common.enums.DbType; +package org.apache.dolphinscheduler.common.datasource; -public class PrestoDataSource extends BaseDataSource { +import java.io.Serializable; - /** - * @return driver class - */ - @Override - public String driverClassSelector() { - return Constants.COM_PRESTO_JDBC_DRIVER; - } - - /** - * @return db type - */ - @Override - public DbType dbTypeSelector() { - return DbType.PRESTO; - } +/** + * The model of Datasource Connection param + */ +public interface ConnectionParam extends Serializable { } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/DatasourceProcessor.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/DatasourceProcessor.java new file mode 100644 index 0000000000..e76a088001 --- /dev/null +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/DatasourceProcessor.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.common.datasource; + +import org.apache.dolphinscheduler.common.enums.DbType; + +import java.io.IOException; +import java.sql.Connection; +import java.sql.SQLException; + +public interface DatasourceProcessor { + + /** + * check datasource param is valid + */ + void checkDatasourceParam(BaseDataSourceParamDTO datasourceParam); + + /** + * create BaseDataSourceParamDTO by connectionJson + * + * @param connectionJson see{@link org.apache.dolphinscheduler.dao.entity.Datasource} + * @return {@link BaseDataSourceParamDTO} + */ + BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson); + + /** + * create datasource connection parameter which will be stored at DataSource + *

+ * see {@code org.apache.dolphinscheduler.dao.entity.DataSource.connectionParams} + */ + ConnectionParam createConnectionParams(BaseDataSourceParamDTO datasourceParam); + + /** + * deserialize json to datasource connection param + * + * @param connectionJson {@code org.apache.dolphinscheduler.dao.entity.DataSource.connectionParams} + * @return {@link BaseConnectionParam} + */ + ConnectionParam createConnectionParams(String connectionJson); + + /** + * get datasource Driver + */ + String getDatasourceDriver(); + + /** + * get jdbcUrl by connection param, the jdbcUrl is different with ConnectionParam.jdbcUrl, this method will inject + * other to jdbcUrl + * + * @param connectionParam connection param + */ + String getJdbcUrl(ConnectionParam connectionParam); + + /** + * get connection by connectionParam + * + * @param connectionParam connectionParam + * @return {@link Connection} + */ + Connection getConnection(ConnectionParam connectionParam) throws ClassNotFoundException, SQLException, IOException; + + /** + * @return {@link DbType} + */ + DbType getDbType(); +} diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/DatasourceUtil.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/DatasourceUtil.java new file mode 100644 index 0000000000..45dcece4ac --- /dev/null +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/DatasourceUtil.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.common.datasource; + +import org.apache.dolphinscheduler.common.datasource.clickhouse.ClickHouseDatasourceProcessor; +import org.apache.dolphinscheduler.common.datasource.db2.Db2DatasourceProcessor; +import org.apache.dolphinscheduler.common.datasource.hive.HiveDatasourceProcessor; +import org.apache.dolphinscheduler.common.datasource.mysql.MysqlDatasourceProcessor; +import org.apache.dolphinscheduler.common.datasource.oracle.OracleDatasourceProcessor; +import org.apache.dolphinscheduler.common.datasource.postgresql.PostgreSqlDatasourceProcessor; +import org.apache.dolphinscheduler.common.datasource.presto.PrestoDatasourceProcessor; +import org.apache.dolphinscheduler.common.datasource.spark.SparkDatasourceProcessor; +import org.apache.dolphinscheduler.common.datasource.sqlserver.SqlServerDatasourceProcessor; +import org.apache.dolphinscheduler.common.enums.DbType; + +import java.sql.Connection; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class DatasourceUtil { + + private DatasourceUtil() { + } + + private static final Logger logger = LoggerFactory.getLogger(DatasourceUtil.class); + + private static final DatasourceProcessor mysqlProcessor = new MysqlDatasourceProcessor(); + private static final DatasourceProcessor postgreSqlProcessor = new PostgreSqlDatasourceProcessor(); + private static final DatasourceProcessor hiveProcessor = new HiveDatasourceProcessor(); + private static final DatasourceProcessor sparkProcessor = new SparkDatasourceProcessor(); + private static final DatasourceProcessor clickhouseProcessor = new ClickHouseDatasourceProcessor(); + private static final DatasourceProcessor oracleProcessor = new OracleDatasourceProcessor(); + private static final DatasourceProcessor sqlServerProcessor = new SqlServerDatasourceProcessor(); + private static final DatasourceProcessor db2PROCESSOR = new Db2DatasourceProcessor(); + private static final DatasourceProcessor prestoPROCESSOR = new PrestoDatasourceProcessor(); + + /** + * check datasource param + * + * @param baseDataSourceParamDTO datasource param + */ + public static void checkDatasourceParam(BaseDataSourceParamDTO baseDataSourceParamDTO) { + getDatasourceProcessor(baseDataSourceParamDTO.getType()).checkDatasourceParam(baseDataSourceParamDTO); + } + + /** + * build connection url + * + * @param baseDataSourceParamDTO datasourceParam + */ + public static ConnectionParam buildConnectionParams(BaseDataSourceParamDTO baseDataSourceParamDTO) { + ConnectionParam connectionParams = getDatasourceProcessor(baseDataSourceParamDTO.getType()) + .createConnectionParams(baseDataSourceParamDTO); + if (logger.isDebugEnabled()) { + logger.info("parameters map:{}", connectionParams); + } + return connectionParams; + } + + public static ConnectionParam buildConnectionParams(DbType dbType, String connectionJson) { + return getDatasourceProcessor(dbType).createConnectionParams(connectionJson); + } + + public static Connection getConnection(DbType dbType, ConnectionParam connectionParam) { + try { + return getDatasourceProcessor(dbType).getConnection(connectionParam); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + public static String getJdbcUrl(DbType dbType, ConnectionParam baseConnectionParam) { + return getDatasourceProcessor(dbType).getJdbcUrl(baseConnectionParam); + } + + public static BaseDataSourceParamDTO buildDatasourceParamDTO(DbType dbType, String connectionParams) { + return getDatasourceProcessor(dbType).createDatasourceParamDTO(connectionParams); + } + + public static DatasourceProcessor getDatasourceProcessor(DbType dbType) { + switch (dbType) { + case MYSQL: + return mysqlProcessor; + case POSTGRESQL: + return postgreSqlProcessor; + case HIVE: + return hiveProcessor; + case SPARK: + return sparkProcessor; + case CLICKHOUSE: + return clickhouseProcessor; + case ORACLE: + return oracleProcessor; + case SQLSERVER: + return sqlServerProcessor; + case DB2: + return db2PROCESSOR; + case PRESTO: + return prestoPROCESSOR; + default: + throw new IllegalArgumentException("datasource type illegal:" + dbType); + } + } + +} diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/ClickHouseDataSource.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/clickhouse/ClickHouseDatasourceParamDTO.java similarity index 60% rename from dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/ClickHouseDataSource.java rename to dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/clickhouse/ClickHouseDatasourceParamDTO.java index ba34ff82d6..f4168fce5f 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/ClickHouseDataSource.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/clickhouse/ClickHouseDatasourceParamDTO.java @@ -14,29 +14,28 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.dolphinscheduler.dao.datasource; -import org.apache.dolphinscheduler.common.Constants; +package org.apache.dolphinscheduler.common.datasource.clickhouse; + +import org.apache.dolphinscheduler.common.datasource.BaseDataSourceParamDTO; import org.apache.dolphinscheduler.common.enums.DbType; -/** - * data source of ClickHouse - */ -public class ClickHouseDataSource extends BaseDataSource { +public class ClickHouseDatasourceParamDTO extends BaseDataSourceParamDTO { - /** - * @return driver class - */ @Override - public String driverClassSelector() { - return Constants.COM_CLICKHOUSE_JDBC_DRIVER; + public String toString() { + return "ClickHouseDatasourceParamDTO{" + + "host='" + host + '\'' + + ", port=" + port + + ", database='" + database + '\'' + + ", userName='" + userName + '\'' + + ", password='" + password + '\'' + + ", other='" + other + '\'' + + '}'; } - /** - * @return db type - */ @Override - public DbType dbTypeSelector() { + public DbType getType() { return DbType.CLICKHOUSE; } } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/clickhouse/ClickHouseDatasourceProcessor.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/clickhouse/ClickHouseDatasourceProcessor.java new file mode 100644 index 0000000000..4dbdb07fe6 --- /dev/null +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/clickhouse/ClickHouseDatasourceProcessor.java @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.common.datasource.clickhouse; + +import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.datasource.AbstractDatasourceProcessor; +import org.apache.dolphinscheduler.common.datasource.BaseDataSourceParamDTO; +import org.apache.dolphinscheduler.common.datasource.ConnectionParam; +import org.apache.dolphinscheduler.common.enums.DbType; +import org.apache.dolphinscheduler.common.utils.CommonUtils; +import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.StringUtils; + +import org.apache.commons.collections4.MapUtils; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.util.LinkedHashMap; +import java.util.Map; + +public class ClickHouseDatasourceProcessor extends AbstractDatasourceProcessor { + + @Override + public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) { + ClickhouseConnectionParam connectionParams = (ClickhouseConnectionParam) createConnectionParams(connectionJson); + + ClickHouseDatasourceParamDTO clickHouseDatasourceParamDTO = new ClickHouseDatasourceParamDTO(); + clickHouseDatasourceParamDTO.setDatabase(connectionParams.getDatabase()); + clickHouseDatasourceParamDTO.setUserName(connectionParams.getUser()); + clickHouseDatasourceParamDTO.setOther(parseOther(connectionParams.getOther())); + + String[] hostSeperator = connectionParams.getAddress().split(Constants.DOUBLE_SLASH); + String[] hostPortArray = hostSeperator[hostSeperator.length - 1].split(Constants.COMMA); + clickHouseDatasourceParamDTO.setPort(Integer.parseInt(hostPortArray[0].split(Constants.COLON)[1])); + clickHouseDatasourceParamDTO.setHost(hostPortArray[0].split(Constants.COLON)[0]); + + return clickHouseDatasourceParamDTO; + } + + @Override + public ConnectionParam createConnectionParams(BaseDataSourceParamDTO datasourceParam) { + ClickHouseDatasourceParamDTO clickHouseParam = (ClickHouseDatasourceParamDTO) datasourceParam; + String address = String.format("%s%s:%s", Constants.JDBC_CLICKHOUSE, clickHouseParam.getHost(), clickHouseParam.getPort()); + String jdbcUrl = address + "/" + clickHouseParam.getDatabase(); + + ClickhouseConnectionParam clickhouseConnectionParam = new ClickhouseConnectionParam(); + clickhouseConnectionParam.setDatabase(clickHouseParam.getDatabase()); + clickhouseConnectionParam.setAddress(address); + clickhouseConnectionParam.setJdbcUrl(jdbcUrl); + clickhouseConnectionParam.setUser(clickHouseParam.getUserName()); + clickhouseConnectionParam.setPassword(CommonUtils.encodePassword(clickHouseParam.getPassword())); + clickhouseConnectionParam.setOther(transformOther(clickHouseParam.getOther())); + return clickhouseConnectionParam; + } + + @Override + public ConnectionParam createConnectionParams(String connectionJson) { + return JSONUtils.parseObject(connectionJson, ClickhouseConnectionParam.class); + } + + @Override + public String getDatasourceDriver() { + return Constants.COM_CLICKHOUSE_JDBC_DRIVER; + } + + @Override + public String getJdbcUrl(ConnectionParam connectionParam) { + ClickhouseConnectionParam clickhouseConnectionParam = (ClickhouseConnectionParam) connectionParam; + String jdbcUrl = clickhouseConnectionParam.getJdbcUrl(); + if (StringUtils.isNotEmpty(clickhouseConnectionParam.getOther())) { + jdbcUrl = String.format("%s?%s", jdbcUrl, clickhouseConnectionParam.getOther()); + } + return jdbcUrl; + } + + @Override + public Connection getConnection(ConnectionParam connectionParam) throws ClassNotFoundException, SQLException { + ClickhouseConnectionParam clickhouseConnectionParam = (ClickhouseConnectionParam) connectionParam; + Class.forName(getDatasourceDriver()); + return DriverManager.getConnection(getJdbcUrl(clickhouseConnectionParam), + clickhouseConnectionParam.getUser(), CommonUtils.decodePassword(clickhouseConnectionParam.getPassword())); + } + + @Override + public DbType getDbType() { + return DbType.CLICKHOUSE; + } + + private String transformOther(Map otherMap) { + if (MapUtils.isEmpty(otherMap)) { + return null; + } + StringBuilder stringBuilder = new StringBuilder(); + otherMap.forEach((key, value) -> stringBuilder.append(String.format("%s=%s%s", key, value, "&"))); + return stringBuilder.toString(); + } + + private Map parseOther(String other) { + if (other == null) { + return null; + } + Map otherMap = new LinkedHashMap<>(); + String[] configs = other.split("&"); + for (String config : configs) { + otherMap.put(config.split("=")[0], config.split("=")[1]); + } + return otherMap; + } +} diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/clickhouse/ClickhouseConnectionParam.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/clickhouse/ClickhouseConnectionParam.java new file mode 100644 index 0000000000..d2fdd0a333 --- /dev/null +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/clickhouse/ClickhouseConnectionParam.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.common.datasource.clickhouse; + +import org.apache.dolphinscheduler.common.datasource.BaseConnectionParam; + +public class ClickhouseConnectionParam extends BaseConnectionParam { + @Override + public String toString() { + return "ClickhouseConnectionParam{" + + "user='" + user + '\'' + + ", password='" + password + '\'' + + ", address='" + address + '\'' + + ", database='" + database + '\'' + + ", jdbcUrl='" + jdbcUrl + '\'' + + ", other='" + other + '\'' + + '}'; + } +} diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/DB2ServerDataSource.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/db2/Db2ConnectionParam.java similarity index 59% rename from dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/DB2ServerDataSource.java rename to dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/db2/Db2ConnectionParam.java index 29448a0fdd..bb47ec2aea 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/DB2ServerDataSource.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/db2/Db2ConnectionParam.java @@ -14,30 +14,21 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.dolphinscheduler.dao.datasource; -import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.common.enums.DbType; +package org.apache.dolphinscheduler.common.datasource.db2; -/** - * data source of DB2 Server - */ -public class DB2ServerDataSource extends BaseDataSource { +import org.apache.dolphinscheduler.common.datasource.BaseConnectionParam; - /** - * gets the JDBC url for the data source connection - * @return jdbc url - */ +public class Db2ConnectionParam extends BaseConnectionParam { @Override - public String driverClassSelector() { - return Constants.COM_DB2_JDBC_DRIVER; - } - - /** - * @return db type - */ - @Override - public DbType dbTypeSelector() { - return DbType.DB2; + public String toString() { + return "Db2ConnectionParam{" + + "user='" + user + '\'' + + ", password='" + password + '\'' + + ", address='" + address + '\'' + + ", database='" + database + '\'' + + ", jdbcUrl='" + jdbcUrl + '\'' + + ", other='" + other + '\'' + + '}'; } } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/db2/Db2DatasourceParamDTO.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/db2/Db2DatasourceParamDTO.java new file mode 100644 index 0000000000..c989472832 --- /dev/null +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/db2/Db2DatasourceParamDTO.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.common.datasource.db2; + +import org.apache.dolphinscheduler.common.datasource.BaseDataSourceParamDTO; +import org.apache.dolphinscheduler.common.enums.DbType; + +public class Db2DatasourceParamDTO extends BaseDataSourceParamDTO { + + @Override + public String toString() { + return "Db2DatasourceParamDTO{" + + "name='" + name + '\'' + + ", note='" + note + '\'' + + ", host='" + host + '\'' + + ", port=" + port + + ", database='" + database + '\'' + + ", userName='" + userName + '\'' + + ", password='" + password + '\'' + + ", other='" + other + '\'' + + '}'; + } + + @Override + public DbType getType() { + return DbType.DB2; + } +} diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/db2/Db2DatasourceProcessor.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/db2/Db2DatasourceProcessor.java new file mode 100644 index 0000000000..f24ef7e5e7 --- /dev/null +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/db2/Db2DatasourceProcessor.java @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.common.datasource.db2; + +import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.datasource.AbstractDatasourceProcessor; +import org.apache.dolphinscheduler.common.datasource.BaseConnectionParam; +import org.apache.dolphinscheduler.common.datasource.BaseDataSourceParamDTO; +import org.apache.dolphinscheduler.common.datasource.ConnectionParam; +import org.apache.dolphinscheduler.common.enums.DbType; +import org.apache.dolphinscheduler.common.utils.CommonUtils; +import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.StringUtils; + +import org.apache.commons.collections4.MapUtils; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.util.LinkedHashMap; +import java.util.Map; + +public class Db2DatasourceProcessor extends AbstractDatasourceProcessor { + + @Override + public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) { + Db2ConnectionParam connectionParams = (Db2ConnectionParam) createConnectionParams(connectionJson); + + Db2DatasourceParamDTO db2DatasourceParamDTO = new Db2DatasourceParamDTO(); + db2DatasourceParamDTO.setDatabase(connectionParams.getDatabase()); + db2DatasourceParamDTO.setOther(parseOther(connectionParams.getOther())); + db2DatasourceParamDTO.setUserName(db2DatasourceParamDTO.getUserName()); + + String[] hostSeperator = connectionParams.getAddress().split(Constants.DOUBLE_SLASH); + String[] hostPortArray = hostSeperator[hostSeperator.length - 1].split(Constants.COMMA); + db2DatasourceParamDTO.setHost(hostPortArray[0].split(Constants.COLON)[0]); + db2DatasourceParamDTO.setPort(Integer.parseInt(hostPortArray[0].split(Constants.COLON)[1])); + + return db2DatasourceParamDTO; + } + + @Override + public BaseConnectionParam createConnectionParams(BaseDataSourceParamDTO datasourceParam) { + Db2DatasourceParamDTO db2Param = (Db2DatasourceParamDTO) datasourceParam; + String address = String.format("%s%s:%s", Constants.JDBC_DB2, db2Param.getHost(), db2Param.getPort()); + String jdbcUrl = String.format("%s/%s", address, db2Param.getDatabase()); + + Db2ConnectionParam db2ConnectionParam = new Db2ConnectionParam(); + db2ConnectionParam.setAddress(address); + db2ConnectionParam.setDatabase(db2Param.getDatabase()); + db2ConnectionParam.setJdbcUrl(jdbcUrl); + db2ConnectionParam.setUser(db2Param.getUserName()); + db2ConnectionParam.setPassword(CommonUtils.encodePassword(db2Param.getPassword())); + db2ConnectionParam.setOther(transformOther(db2Param.getOther())); + + return db2ConnectionParam; + } + + @Override + public ConnectionParam createConnectionParams(String connectionJson) { + return JSONUtils.parseObject(connectionJson, Db2ConnectionParam.class); + } + + @Override + public String getDatasourceDriver() { + return Constants.COM_DB2_JDBC_DRIVER; + } + + @Override + public String getJdbcUrl(ConnectionParam connectionParam) { + Db2ConnectionParam db2ConnectionParam = (Db2ConnectionParam) connectionParam; + if (StringUtils.isNotEmpty(db2ConnectionParam.getOther())) { + return String.format("%s;%s", db2ConnectionParam.getJdbcUrl(), db2ConnectionParam.getOther()); + } + return db2ConnectionParam.getJdbcUrl(); + } + + @Override + public Connection getConnection(ConnectionParam connectionParam) throws ClassNotFoundException, SQLException { + Db2ConnectionParam db2ConnectionParam = (Db2ConnectionParam) connectionParam; + Class.forName(getDatasourceDriver()); + return DriverManager.getConnection(getJdbcUrl(db2ConnectionParam), + db2ConnectionParam.getUser(), CommonUtils.decodePassword(db2ConnectionParam.getPassword())); + } + + @Override + public DbType getDbType() { + return DbType.DB2; + } + + private String transformOther(Map otherMap) { + if (MapUtils.isEmpty(otherMap)) { + return null; + } + StringBuilder stringBuilder = new StringBuilder(); + otherMap.forEach((key, value) -> stringBuilder.append(String.format("%s=%s%s", key, value, ";"))); + stringBuilder.deleteCharAt(stringBuilder.length() - 1); + return stringBuilder.toString(); + } + + private Map parseOther(String other) { + if (other == null) { + return null; + } + Map otherMap = new LinkedHashMap<>(); + for (String config : other.split("&")) { + otherMap.put(config.split("=")[0], config.split("=")[1]); + } + return otherMap; + } +} diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/hive/HiveConnectionParam.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/hive/HiveConnectionParam.java new file mode 100644 index 0000000000..e982211df7 --- /dev/null +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/hive/HiveConnectionParam.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.common.datasource.hive; + +import org.apache.dolphinscheduler.common.datasource.BaseHdfsConnectionParam; + +public class HiveConnectionParam extends BaseHdfsConnectionParam { + @Override + public String toString() { + return "HiveConnectionParam{" + + "user='" + user + '\'' + + ", password='" + password + '\'' + + ", address='" + address + '\'' + + ", database='" + database + '\'' + + ", jdbcUrl='" + jdbcUrl + '\'' + + ", other='" + other + '\'' + + ", principal='" + principal + '\'' + + ", javaSecurityKrb5Conf='" + javaSecurityKrb5Conf + '\'' + + ", loginUserKeytabUsername='" + loginUserKeytabUsername + '\'' + + ", loginUserKeytabPath='" + loginUserKeytabPath + '\'' + + '}'; + } +} diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/hive/HiveDataSourceParamDTO.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/hive/HiveDataSourceParamDTO.java new file mode 100644 index 0000000000..816d08889a --- /dev/null +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/hive/HiveDataSourceParamDTO.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.common.datasource.hive; + +import org.apache.dolphinscheduler.common.datasource.BaseHdfsDatasourceParamDTO; +import org.apache.dolphinscheduler.common.enums.DbType; + +public class HiveDataSourceParamDTO extends BaseHdfsDatasourceParamDTO { + + @Override + public String toString() { + return "HiveDataSourceParamDTO{" + + "host='" + host + '\'' + + ", port=" + port + + ", database='" + database + '\'' + + ", principal='" + principal + '\'' + + ", userName='" + userName + '\'' + + ", password='" + password + '\'' + + ", other='" + other + '\'' + + ", javaSecurityKrb5Conf='" + javaSecurityKrb5Conf + '\'' + + ", loginUserKeytabUsername='" + loginUserKeytabUsername + '\'' + + ", loginUserKeytabPath='" + loginUserKeytabPath + '\'' + + '}'; + } + + @Override + public DbType getType() { + return DbType.HIVE; + } +} diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/hive/HiveDatasourceProcessor.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/hive/HiveDatasourceProcessor.java new file mode 100644 index 0000000000..4baebb80bb --- /dev/null +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/hive/HiveDatasourceProcessor.java @@ -0,0 +1,185 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.common.datasource.hive; + +import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.datasource.AbstractDatasourceProcessor; +import org.apache.dolphinscheduler.common.datasource.BaseConnectionParam; +import org.apache.dolphinscheduler.common.datasource.BaseDataSourceParamDTO; +import org.apache.dolphinscheduler.common.datasource.ConnectionParam; +import org.apache.dolphinscheduler.common.enums.DbType; +import org.apache.dolphinscheduler.common.utils.CommonUtils; +import org.apache.dolphinscheduler.common.utils.HiveConfUtils; +import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.StringUtils; + +import org.apache.commons.collections4.MapUtils; + +import java.io.IOException; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.util.LinkedHashMap; +import java.util.Map; + +public class HiveDatasourceProcessor extends AbstractDatasourceProcessor { + + @Override + public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) { + HiveDataSourceParamDTO hiveDataSourceParamDTO = new HiveDataSourceParamDTO(); + HiveConnectionParam hiveConnectionParam = (HiveConnectionParam) createConnectionParams(connectionJson); + + hiveDataSourceParamDTO.setDatabase(hiveConnectionParam.getDatabase()); + hiveDataSourceParamDTO.setUserName(hiveConnectionParam.getUser()); + hiveDataSourceParamDTO.setOther(parseOther(hiveConnectionParam.getOther())); + hiveDataSourceParamDTO.setLoginUserKeytabUsername(hiveConnectionParam.getLoginUserKeytabUsername()); + hiveDataSourceParamDTO.setLoginUserKeytabPath(hiveConnectionParam.getLoginUserKeytabPath()); + hiveDataSourceParamDTO.setJavaSecurityKrb5Conf(hiveConnectionParam.getJavaSecurityKrb5Conf()); + + String[] tmpArray = hiveConnectionParam.getAddress().split(Constants.DOUBLE_SLASH); + StringBuilder hosts = new StringBuilder(); + String[] hostPortArray = tmpArray[tmpArray.length - 1].split(Constants.COMMA); + for (String hostPort : hostPortArray) { + hosts.append(hostPort.split(Constants.COLON)[0]).append(Constants.COMMA); + } + hosts.deleteCharAt(hosts.length() - 1); + hiveDataSourceParamDTO.setHost(hosts.toString()); + hiveDataSourceParamDTO.setPort(Integer.parseInt(hostPortArray[0].split(Constants.COLON)[1])); + + return hiveDataSourceParamDTO; + } + + @Override + public BaseConnectionParam createConnectionParams(BaseDataSourceParamDTO datasourceParam) { + HiveDataSourceParamDTO hiveParam = (HiveDataSourceParamDTO) datasourceParam; + StringBuilder address = new StringBuilder(); + address.append(Constants.JDBC_HIVE_2); + for (String zkHost : hiveParam.getHost().split(",")) { + address.append(String.format("%s:%s,", zkHost, hiveParam.getPort())); + } + address.deleteCharAt(address.length() - 1); + String jdbcUrl = address.toString() + "/" + hiveParam.getDatabase(); + if (CommonUtils.getKerberosStartupState()) { + jdbcUrl += ";principal=" + hiveParam.getPrincipal(); + } + + HiveConnectionParam hiveConnectionParam = new HiveConnectionParam(); + hiveConnectionParam.setDatabase(hiveParam.getDatabase()); + hiveConnectionParam.setAddress(address.toString()); + hiveConnectionParam.setJdbcUrl(jdbcUrl); + hiveConnectionParam.setUser(hiveParam.getUserName()); + hiveConnectionParam.setPassword(CommonUtils.encodePassword(hiveParam.getPassword())); + + if (CommonUtils.getKerberosStartupState()) { + hiveConnectionParam.setPrincipal(hiveParam.getPrincipal()); + hiveConnectionParam.setJavaSecurityKrb5Conf(hiveParam.getJavaSecurityKrb5Conf()); + hiveConnectionParam.setLoginUserKeytabPath(hiveParam.getLoginUserKeytabPath()); + hiveConnectionParam.setLoginUserKeytabUsername(hiveParam.getLoginUserKeytabUsername()); + } + hiveConnectionParam.setOther(transformOther(hiveParam.getOther())); + return hiveConnectionParam; + } + + @Override + public ConnectionParam createConnectionParams(String connectionJson) { + return JSONUtils.parseObject(connectionJson, HiveConnectionParam.class); + } + + @Override + public String getDatasourceDriver() { + return Constants.ORG_APACHE_HIVE_JDBC_HIVE_DRIVER; + } + + @Override + public String getJdbcUrl(ConnectionParam connectionParam) { + HiveConnectionParam hiveConnectionParam = (HiveConnectionParam) connectionParam; + String jdbcUrl = hiveConnectionParam.getJdbcUrl(); + String otherParams = filterOther(hiveConnectionParam.getOther()); + if (StringUtils.isNotEmpty(otherParams) && !"?".equals(otherParams.substring(0, 1))) { + jdbcUrl += ";"; + } + return jdbcUrl + otherParams; + } + + @Override + public Connection getConnection(ConnectionParam connectionParam) throws IOException, ClassNotFoundException, SQLException { + HiveConnectionParam hiveConnectionParam = (HiveConnectionParam) connectionParam; + CommonUtils.loadKerberosConf(hiveConnectionParam.getJavaSecurityKrb5Conf(), + hiveConnectionParam.getLoginUserKeytabUsername(), hiveConnectionParam.getLoginUserKeytabPath()); + Class.forName(getDatasourceDriver()); + return DriverManager.getConnection(getJdbcUrl(connectionParam), + hiveConnectionParam.getUser(), CommonUtils.decodePassword(hiveConnectionParam.getPassword())); + } + + @Override + public DbType getDbType() { + return DbType.HIVE; + } + + private String transformOther(Map otherMap) { + if (MapUtils.isEmpty(otherMap)) { + return null; + } + StringBuilder stringBuilder = new StringBuilder(); + otherMap.forEach((key, value) -> stringBuilder.append(String.format("%s=%s;", key, value))); + return stringBuilder.toString(); + } + + private String filterOther(String otherParams) { + if (StringUtils.isBlank(otherParams)) { + return ""; + } + + StringBuilder hiveConfListSb = new StringBuilder(); + hiveConfListSb.append("?"); + StringBuilder sessionVarListSb = new StringBuilder(); + + String[] otherArray = otherParams.split(";", -1); + + for (String conf : otherArray) { + if (HiveConfUtils.isHiveConfVar(conf)) { + hiveConfListSb.append(conf).append(";"); + } else { + sessionVarListSb.append(conf).append(";"); + } + } + + // remove the last ";" + if (sessionVarListSb.length() > 0) { + sessionVarListSb.deleteCharAt(sessionVarListSb.length() - 1); + } + + if (hiveConfListSb.length() > 0) { + hiveConfListSb.deleteCharAt(hiveConfListSb.length() - 1); + } + + return sessionVarListSb.toString() + hiveConfListSb.toString(); + } + + private Map parseOther(String other) { + if (other == null) { + return null; + } + Map otherMap = new LinkedHashMap<>(); + String[] configs = other.split(";"); + for (String config : configs) { + otherMap.put(config.split("=")[0], config.split("=")[1]); + } + return otherMap; + } +} diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/PostgreDataSource.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/mysql/MysqlConnectionParam.java similarity index 57% rename from dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/PostgreDataSource.java rename to dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/mysql/MysqlConnectionParam.java index 5a71976c53..3c5117c2e2 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/PostgreDataSource.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/mysql/MysqlConnectionParam.java @@ -14,30 +14,21 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.dolphinscheduler.dao.datasource; -import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.common.enums.DbType; +package org.apache.dolphinscheduler.common.datasource.mysql; -/** - * data source of postgreSQL - */ -public class PostgreDataSource extends BaseDataSource { +import org.apache.dolphinscheduler.common.datasource.BaseConnectionParam; - /** - * gets the JDBC url for the data source connection - * @return jdbc url - */ - @Override - public String driverClassSelector() { - return Constants.ORG_POSTGRESQL_DRIVER; - } - - /** - * @return db type - */ - @Override - public DbType dbTypeSelector() { - return DbType.POSTGRESQL; - } +public class MysqlConnectionParam extends BaseConnectionParam { + @Override + public String toString() { + return "MysqlConnectionParam{" + + "user='" + user + '\'' + + ", password='" + password + '\'' + + ", address='" + address + '\'' + + ", database='" + database + '\'' + + ", jdbcUrl='" + jdbcUrl + '\'' + + ", other='" + other + '\'' + + '}'; + } } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/mysql/MysqlDatasourceParamDTO.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/mysql/MysqlDatasourceParamDTO.java new file mode 100644 index 0000000000..3e8f0ed918 --- /dev/null +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/mysql/MysqlDatasourceParamDTO.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.common.datasource.mysql; + +import org.apache.dolphinscheduler.common.datasource.BaseDataSourceParamDTO; +import org.apache.dolphinscheduler.common.enums.DbType; + +public class MysqlDatasourceParamDTO extends BaseDataSourceParamDTO { + + @Override + public String toString() { + return "MysqlDatasourceParamDTO{" + + "name='" + name + '\'' + + ", note='" + note + '\'' + + ", host='" + host + '\'' + + ", port=" + port + + ", database='" + database + '\'' + + ", userName='" + userName + '\'' + + ", password='" + password + '\'' + + ", other='" + other + '\'' + + '}'; + } + + @Override + public DbType getType() { + return DbType.MYSQL; + } +} diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/mysql/MysqlDatasourceProcessor.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/mysql/MysqlDatasourceProcessor.java new file mode 100644 index 0000000000..f684378ecc --- /dev/null +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/mysql/MysqlDatasourceProcessor.java @@ -0,0 +1,170 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.common.datasource.mysql; + +import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.datasource.AbstractDatasourceProcessor; +import org.apache.dolphinscheduler.common.datasource.BaseConnectionParam; +import org.apache.dolphinscheduler.common.datasource.BaseDataSourceParamDTO; +import org.apache.dolphinscheduler.common.datasource.ConnectionParam; +import org.apache.dolphinscheduler.common.enums.DbType; +import org.apache.dolphinscheduler.common.utils.CommonUtils; +import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.StringUtils; + +import org.apache.commons.collections4.MapUtils; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.Map; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class MysqlDatasourceProcessor extends AbstractDatasourceProcessor { + + private final Logger logger = LoggerFactory.getLogger(MysqlDatasourceProcessor.class); + + private static final String ALLOW_LOAD_LOCAL_IN_FILE_NAME = "allowLoadLocalInfile"; + + private static final String AUTO_DESERIALIZE = "autoDeserialize"; + + private static final String ALLOW_LOCAL_IN_FILE_NAME = "allowLocalInfile"; + + private static final String ALLOW_URL_IN_LOCAL_IN_FILE_NAME = "allowUrlInLocalInfile"; + + private static final String APPEND_PARAMS = "allowLoadLocalInfile=false&autoDeserialize=false&allowLocalInfile=false&allowUrlInLocalInfile=false"; + + @Override + public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) { + MysqlConnectionParam connectionParams = (MysqlConnectionParam) createConnectionParams(connectionJson); + MysqlDatasourceParamDTO mysqlDatasourceParamDTO = new MysqlDatasourceParamDTO(); + + mysqlDatasourceParamDTO.setUserName(connectionParams.getUser()); + mysqlDatasourceParamDTO.setDatabase(connectionParams.getDatabase()); + mysqlDatasourceParamDTO.setOther(parseOther(connectionParams.getOther())); + + String address = connectionParams.getAddress(); + String[] hostSeperator = address.split(Constants.DOUBLE_SLASH); + String[] hostPortArray = hostSeperator[hostSeperator.length - 1].split(Constants.COMMA); + mysqlDatasourceParamDTO.setPort(Integer.parseInt(hostPortArray[0].split(Constants.COLON)[1])); + mysqlDatasourceParamDTO.setHost(hostPortArray[0].split(Constants.COLON)[0]); + + return mysqlDatasourceParamDTO; + } + + @Override + public BaseConnectionParam createConnectionParams(BaseDataSourceParamDTO dataSourceParam) { + MysqlDatasourceParamDTO mysqlDatasourceParam = (MysqlDatasourceParamDTO) dataSourceParam; + String address = String.format("%s%s:%s", Constants.JDBC_MYSQL, mysqlDatasourceParam.getHost(), mysqlDatasourceParam.getPort()); + String jdbcUrl = String.format("%s/%s", address, mysqlDatasourceParam.getDatabase()); + + MysqlConnectionParam mysqlConnectionParam = new MysqlConnectionParam(); + mysqlConnectionParam.setJdbcUrl(jdbcUrl); + mysqlConnectionParam.setDatabase(mysqlDatasourceParam.getDatabase()); + mysqlConnectionParam.setAddress(address); + mysqlConnectionParam.setUser(mysqlDatasourceParam.getUserName()); + mysqlConnectionParam.setPassword(CommonUtils.encodePassword(mysqlDatasourceParam.getPassword())); + mysqlConnectionParam.setOther(transformOther(mysqlDatasourceParam.getOther())); + + return mysqlConnectionParam; + } + + @Override + public ConnectionParam createConnectionParams(String connectionJson) { + return JSONUtils.parseObject(connectionJson, MysqlConnectionParam.class); + } + + @Override + public String getDatasourceDriver() { + return Constants.COM_MYSQL_JDBC_DRIVER; + } + + @Override + public String getJdbcUrl(ConnectionParam connectionParam) { + MysqlConnectionParam mysqlConnectionParam = (MysqlConnectionParam) connectionParam; + String jdbcUrl = mysqlConnectionParam.getJdbcUrl(); + if (StringUtils.isNotEmpty(mysqlConnectionParam.getOther())) { + return String.format("%s?%s&%s", jdbcUrl, mysqlConnectionParam.getOther(), APPEND_PARAMS); + } + return String.format("%s?%s", jdbcUrl, APPEND_PARAMS); + } + + @Override + public Connection getConnection(ConnectionParam connectionParam) throws ClassNotFoundException, SQLException { + MysqlConnectionParam mysqlConnectionParam = (MysqlConnectionParam) connectionParam; + Class.forName(getDatasourceDriver()); + String user = mysqlConnectionParam.getUser(); + if (user.contains(AUTO_DESERIALIZE)) { + logger.warn("sensitive param : {} in username field is filtered", AUTO_DESERIALIZE); + user = user.replace(AUTO_DESERIALIZE, ""); + } + String password = CommonUtils.decodePassword(mysqlConnectionParam.getPassword()); + if (password.contains(AUTO_DESERIALIZE)) { + logger.warn("sensitive param : {} in password field is filtered", AUTO_DESERIALIZE); + password = password.replace(AUTO_DESERIALIZE, ""); + } + return DriverManager.getConnection(getJdbcUrl(connectionParam), user, password); + } + + @Override + public DbType getDbType() { + return DbType.MYSQL; + } + + private String transformOther(Map paramMap) { + if (MapUtils.isEmpty(paramMap)) { + return null; + } + Map otherMap = new HashMap<>(); + paramMap.forEach((k, v) -> { + if (!checkKeyIsLegitimate(k)) { + return; + } + otherMap.put(k, v); + }); + if (MapUtils.isEmpty(otherMap)) { + return null; + } + StringBuilder stringBuilder = new StringBuilder(); + otherMap.forEach((key, value) -> stringBuilder.append(String.format("%s=%s&", key, value))); + return stringBuilder.toString(); + } + + private static boolean checkKeyIsLegitimate(String key) { + return !key.contains(ALLOW_LOAD_LOCAL_IN_FILE_NAME) + && !key.contains(AUTO_DESERIALIZE) + && !key.contains(ALLOW_LOCAL_IN_FILE_NAME) + && !key.contains(ALLOW_URL_IN_LOCAL_IN_FILE_NAME); + } + + private Map parseOther(String other) { + if (StringUtils.isEmpty(other)) { + return null; + } + Map otherMap = new LinkedHashMap<>(); + for (String config : other.split("&")) { + otherMap.put(config.split("=")[0], config.split("=")[1]); + } + return otherMap; + } + +} diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/oracle/OracleConnectionParam.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/oracle/OracleConnectionParam.java new file mode 100644 index 0000000000..a59f50badb --- /dev/null +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/oracle/OracleConnectionParam.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.common.datasource.oracle; + +import org.apache.dolphinscheduler.common.datasource.BaseConnectionParam; +import org.apache.dolphinscheduler.common.enums.DbConnectType; + +public class OracleConnectionParam extends BaseConnectionParam { + protected DbConnectType connectType; + + public DbConnectType getConnectType() { + return connectType; + } + + public void setConnectType(DbConnectType connectType) { + this.connectType = connectType; + } + + @Override + public String toString() { + return "OracleConnectionParam{" + + "user='" + user + '\'' + + ", password='" + password + '\'' + + ", address='" + address + '\'' + + ", database='" + database + '\'' + + ", jdbcUrl='" + jdbcUrl + '\'' + + ", other='" + other + '\'' + + ", connectType=" + connectType + + '}'; + } +} diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/OracleDataSource.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/oracle/OracleDatasourceParamDTO.java similarity index 62% rename from dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/OracleDataSource.java rename to dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/oracle/OracleDatasourceParamDTO.java index 37c451ac86..615c514074 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/OracleDataSource.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/oracle/OracleDatasourceParamDTO.java @@ -14,16 +14,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.dolphinscheduler.dao.datasource; -import org.apache.dolphinscheduler.common.Constants; +package org.apache.dolphinscheduler.common.datasource.oracle; + +import org.apache.dolphinscheduler.common.datasource.BaseDataSourceParamDTO; import org.apache.dolphinscheduler.common.enums.DbConnectType; import org.apache.dolphinscheduler.common.enums.DbType; -/** - * data source of Oracle - */ -public class OracleDataSource extends BaseDataSource { +public class OracleDatasourceParamDTO extends BaseDataSourceParamDTO { private DbConnectType connectType; @@ -35,33 +33,23 @@ public class OracleDataSource extends BaseDataSource { this.connectType = connectType; } - /** - * @return driver class - */ @Override - public String driverClassSelector() { - return Constants.COM_ORACLE_JDBC_DRIVER; + public String toString() { + return "OracleDatasourceParamDTO{" + + "name='" + name + '\'' + + ", note='" + note + '\'' + + ", host='" + host + '\'' + + ", port=" + port + + ", database='" + database + '\'' + + ", userName='" + userName + '\'' + + ", password='" + password + '\'' + + ", connectType=" + connectType + + ", other='" + other + '\'' + + '}'; } - /** - * append service name or SID - */ @Override - protected void appendDatabase(StringBuilder jdbcUrl) { - if (getConnectType() == DbConnectType.ORACLE_SID) { - jdbcUrl.append(":"); - } else { - jdbcUrl.append("/"); - } - jdbcUrl.append(getDatabase()); - } - - /** - * @return db type - */ - @Override - public DbType dbTypeSelector() { + public DbType getType() { return DbType.ORACLE; } - } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/oracle/OracleDatasourceProcessor.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/oracle/OracleDatasourceProcessor.java new file mode 100644 index 0000000000..e18a963993 --- /dev/null +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/oracle/OracleDatasourceProcessor.java @@ -0,0 +1,141 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.common.datasource.oracle; + +import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.datasource.AbstractDatasourceProcessor; +import org.apache.dolphinscheduler.common.datasource.BaseConnectionParam; +import org.apache.dolphinscheduler.common.datasource.BaseDataSourceParamDTO; +import org.apache.dolphinscheduler.common.datasource.ConnectionParam; +import org.apache.dolphinscheduler.common.enums.DbConnectType; +import org.apache.dolphinscheduler.common.enums.DbType; +import org.apache.dolphinscheduler.common.utils.CommonUtils; +import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.StringUtils; + +import org.apache.commons.collections4.MapUtils; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +public class OracleDatasourceProcessor extends AbstractDatasourceProcessor { + + @Override + public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) { + OracleConnectionParam connectionParams = (OracleConnectionParam) createConnectionParams(connectionJson); + OracleDatasourceParamDTO oracleDatasourceParamDTO = new OracleDatasourceParamDTO(); + + oracleDatasourceParamDTO.setDatabase(connectionParams.getDatabase()); + oracleDatasourceParamDTO.setUserName(connectionParams.getUser()); + oracleDatasourceParamDTO.setOther(parseOther(connectionParams.getOther())); + + String hostSeperator = Constants.DOUBLE_SLASH; + if (DbConnectType.ORACLE_SID.equals(connectionParams.connectType)) { + hostSeperator = Constants.AT_SIGN; + } + String[] hostPort = connectionParams.getAddress().split(hostSeperator); + String[] hostPortArray = hostPort[hostPort.length - 1].split(Constants.COMMA); + oracleDatasourceParamDTO.setPort(Integer.parseInt(hostPortArray[0].split(Constants.COLON)[1])); + oracleDatasourceParamDTO.setHost(hostPortArray[0].split(Constants.COLON)[0]); + + return oracleDatasourceParamDTO; + } + + @Override + public BaseConnectionParam createConnectionParams(BaseDataSourceParamDTO datasourceParam) { + OracleDatasourceParamDTO oracleParam = (OracleDatasourceParamDTO) datasourceParam; + String address; + if (DbConnectType.ORACLE_SID.equals(oracleParam.getConnectType())) { + address = String.format("%s%s:%s", + Constants.JDBC_ORACLE_SID, oracleParam.getHost(), oracleParam.getPort()); + } else { + address = String.format("%s%s:%s", + Constants.JDBC_ORACLE_SERVICE_NAME, oracleParam.getHost(), oracleParam.getPort()); + } + String jdbcUrl = address + "/" + oracleParam.getDatabase(); + + OracleConnectionParam oracleConnectionParam = new OracleConnectionParam(); + oracleConnectionParam.setUser(oracleParam.getUserName()); + oracleConnectionParam.setPassword(CommonUtils.encodePassword(oracleParam.getPassword())); + oracleConnectionParam.setAddress(address); + oracleConnectionParam.setJdbcUrl(jdbcUrl); + oracleConnectionParam.setDatabase(oracleParam.getDatabase()); + oracleConnectionParam.setConnectType(oracleParam.getConnectType()); + oracleConnectionParam.setOther(transformOther(oracleParam.getOther())); + + return oracleConnectionParam; + } + + @Override + public ConnectionParam createConnectionParams(String connectionJson) { + return JSONUtils.parseObject(connectionJson, OracleConnectionParam.class); + } + + @Override + public String getDatasourceDriver() { + return Constants.COM_ORACLE_JDBC_DRIVER; + } + + @Override + public String getJdbcUrl(ConnectionParam connectionParam) { + OracleConnectionParam oracleConnectionParam = (OracleConnectionParam) connectionParam; + if (StringUtils.isNotEmpty(oracleConnectionParam.getOther())) { + return String.format("%s?%s", oracleConnectionParam.getJdbcUrl(), oracleConnectionParam.getOther()); + } + return oracleConnectionParam.getJdbcUrl(); + } + + @Override + public Connection getConnection(ConnectionParam connectionParam) throws ClassNotFoundException, SQLException { + OracleConnectionParam oracleConnectionParam = (OracleConnectionParam) connectionParam; + Class.forName(getDatasourceDriver()); + return DriverManager.getConnection(getJdbcUrl(connectionParam), + oracleConnectionParam.getUser(), CommonUtils.decodePassword(oracleConnectionParam.getPassword())); + } + + @Override + public DbType getDbType() { + return DbType.ORACLE; + } + + private String transformOther(Map otherMap) { + if (MapUtils.isEmpty(otherMap)) { + return null; + } + List list = new ArrayList<>(); + otherMap.forEach((key, value) -> list.add(String.format("%s=%s", key, value))); + return String.join("&", list); + } + + private Map parseOther(String other) { + if (StringUtils.isEmpty(other)) { + return null; + } + Map otherMap = new LinkedHashMap<>(); + String[] configs = other.split("&"); + for (String config : configs) { + otherMap.put(config.split("=")[0], config.split("=")[1]); + } + return otherMap; + } +} diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/postgresql/PostgreSqlConnectionParam.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/postgresql/PostgreSqlConnectionParam.java new file mode 100644 index 0000000000..7090581f65 --- /dev/null +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/postgresql/PostgreSqlConnectionParam.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.common.datasource.postgresql; + +import org.apache.dolphinscheduler.common.datasource.BaseConnectionParam; + +public class PostgreSqlConnectionParam extends BaseConnectionParam { + @Override + public String toString() { + return "PostgreSqlConnectionParam{" + + "user='" + user + '\'' + + ", password='" + password + '\'' + + ", address='" + address + '\'' + + ", database='" + database + '\'' + + ", jdbcUrl='" + jdbcUrl + '\'' + + ", other='" + other + '\'' + + '}'; + } +} diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/postgresql/PostgreSqlDatasourceParamDTO.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/postgresql/PostgreSqlDatasourceParamDTO.java new file mode 100644 index 0000000000..5d81233a7a --- /dev/null +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/postgresql/PostgreSqlDatasourceParamDTO.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.common.datasource.postgresql; + +import org.apache.dolphinscheduler.common.datasource.BaseDataSourceParamDTO; +import org.apache.dolphinscheduler.common.enums.DbType; + +public class PostgreSqlDatasourceParamDTO extends BaseDataSourceParamDTO { + + @Override + public String toString() { + return "PostgreSqlDatasourceParamDTO{" + + "host='" + host + '\'' + + ", port=" + port + + ", database='" + database + '\'' + + ", userName='" + userName + '\'' + + ", password='" + password + '\'' + + ", other='" + other + '\'' + + '}'; + } + + @Override + public DbType getType() { + return DbType.POSTGRESQL; + } +} diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/postgresql/PostgreSqlDatasourceProcessor.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/postgresql/PostgreSqlDatasourceProcessor.java new file mode 100644 index 0000000000..284aa41a7a --- /dev/null +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/postgresql/PostgreSqlDatasourceProcessor.java @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.common.datasource.postgresql; + +import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.datasource.AbstractDatasourceProcessor; +import org.apache.dolphinscheduler.common.datasource.BaseConnectionParam; +import org.apache.dolphinscheduler.common.datasource.BaseDataSourceParamDTO; +import org.apache.dolphinscheduler.common.datasource.ConnectionParam; +import org.apache.dolphinscheduler.common.enums.DbType; +import org.apache.dolphinscheduler.common.utils.CommonUtils; +import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.StringUtils; + +import org.apache.commons.collections4.MapUtils; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.util.LinkedHashMap; +import java.util.Map; + +public class PostgreSqlDatasourceProcessor extends AbstractDatasourceProcessor { + + @Override + public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) { + PostgreSqlConnectionParam connectionParams = (PostgreSqlConnectionParam) createConnectionParams(connectionJson); + PostgreSqlDatasourceParamDTO postgreSqlDatasourceParamDTO = new PostgreSqlDatasourceParamDTO(); + postgreSqlDatasourceParamDTO.setDatabase(connectionParams.getDatabase()); + postgreSqlDatasourceParamDTO.setUserName(connectionParams.getUser()); + postgreSqlDatasourceParamDTO.setOther(parseOther(connectionParams.getOther())); + + String address = connectionParams.getAddress(); + String[] hostSeperator = address.split(Constants.DOUBLE_SLASH); + String[] hostPortArray = hostSeperator[hostSeperator.length - 1].split(Constants.COMMA); + postgreSqlDatasourceParamDTO.setHost(hostPortArray[0].split(Constants.COLON)[0]); + postgreSqlDatasourceParamDTO.setPort(Integer.parseInt(hostPortArray[0].split(Constants.COLON)[1])); + + return postgreSqlDatasourceParamDTO; + } + + @Override + public BaseConnectionParam createConnectionParams(BaseDataSourceParamDTO datasourceParam) { + PostgreSqlDatasourceParamDTO postgreSqlParam = (PostgreSqlDatasourceParamDTO) datasourceParam; + String address = String.format("%s%s:%s", Constants.JDBC_POSTGRESQL, postgreSqlParam.getHost(), postgreSqlParam.getPort()); + String jdbcUrl = String.format("%s/%s", address, postgreSqlParam.getDatabase()); + + PostgreSqlConnectionParam postgreSqlConnectionParam = new PostgreSqlConnectionParam(); + postgreSqlConnectionParam.setJdbcUrl(jdbcUrl); + postgreSqlConnectionParam.setAddress(address); + postgreSqlConnectionParam.setDatabase(postgreSqlParam.getDatabase()); + postgreSqlConnectionParam.setUser(postgreSqlParam.getUserName()); + postgreSqlConnectionParam.setPassword(CommonUtils.encodePassword(postgreSqlParam.getPassword())); + postgreSqlConnectionParam.setOther(transformOther(postgreSqlParam.getOther())); + + return postgreSqlConnectionParam; + } + + @Override + public ConnectionParam createConnectionParams(String connectionJson) { + return JSONUtils.parseObject(connectionJson, PostgreSqlConnectionParam.class); + } + + @Override + public String getDatasourceDriver() { + return Constants.ORG_POSTGRESQL_DRIVER; + } + + @Override + public String getJdbcUrl(ConnectionParam connectionParam) { + PostgreSqlConnectionParam postgreSqlConnectionParam = (PostgreSqlConnectionParam) connectionParam; + if (StringUtils.isNotEmpty(postgreSqlConnectionParam.getOther())) { + return String.format("%s?%s", postgreSqlConnectionParam.getJdbcUrl(), postgreSqlConnectionParam.getOther()); + } + return postgreSqlConnectionParam.getJdbcUrl(); + } + + @Override + public Connection getConnection(ConnectionParam connectionParam) throws ClassNotFoundException, SQLException { + PostgreSqlConnectionParam postgreSqlConnectionParam = (PostgreSqlConnectionParam) connectionParam; + Class.forName(getDatasourceDriver()); + return DriverManager.getConnection(getJdbcUrl(postgreSqlConnectionParam), + postgreSqlConnectionParam.getUser(), CommonUtils.decodePassword(postgreSqlConnectionParam.getPassword())); + } + + @Override + public DbType getDbType() { + return DbType.POSTGRESQL; + } + + private String transformOther(Map otherMap) { + if (MapUtils.isEmpty(otherMap)) { + return null; + } + StringBuilder stringBuilder = new StringBuilder(); + otherMap.forEach((key, value) -> stringBuilder.append(String.format("%s=%s&", key, value))); + return stringBuilder.toString(); + } + + private Map parseOther(String other) { + if (StringUtils.isEmpty(other)) { + return null; + } + Map otherMap = new LinkedHashMap<>(); + for (String config : other.split("&")) { + String[] split = config.split("="); + otherMap.put(split[0], split[1]); + } + return otherMap; + } +} diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/presto/PrestoConnectionParam.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/presto/PrestoConnectionParam.java new file mode 100644 index 0000000000..436bc6dd1e --- /dev/null +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/presto/PrestoConnectionParam.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.common.datasource.presto; + +import org.apache.dolphinscheduler.common.datasource.BaseConnectionParam; + +public class PrestoConnectionParam extends BaseConnectionParam { + @Override + public String toString() { + return "PrestoConnectionParam{" + + "user='" + user + '\'' + + ", password='" + password + '\'' + + ", address='" + address + '\'' + + ", database='" + database + '\'' + + ", jdbcUrl='" + jdbcUrl + '\'' + + ", other='" + other + '\'' + + '}'; + } +} diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/presto/PrestoDatasourceParamDTO.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/presto/PrestoDatasourceParamDTO.java new file mode 100644 index 0000000000..9477362b03 --- /dev/null +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/presto/PrestoDatasourceParamDTO.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.common.datasource.presto; + +import org.apache.dolphinscheduler.common.datasource.BaseDataSourceParamDTO; +import org.apache.dolphinscheduler.common.enums.DbType; + +public class PrestoDatasourceParamDTO extends BaseDataSourceParamDTO { + + @Override + public String toString() { + return "PrestoDatasourceParamDTO{" + + "name='" + name + '\'' + + ", note='" + note + '\'' + + ", host='" + host + '\'' + + ", port=" + port + + ", database='" + database + '\'' + + ", userName='" + userName + '\'' + + ", password='" + password + '\'' + + ", other='" + other + '\'' + + '}'; + } + + @Override + public DbType getType() { + return DbType.PRESTO; + } +} diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/presto/PrestoDatasourceProcessor.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/presto/PrestoDatasourceProcessor.java new file mode 100644 index 0000000000..e9be62b494 --- /dev/null +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/presto/PrestoDatasourceProcessor.java @@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.common.datasource.presto; + +import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.datasource.AbstractDatasourceProcessor; +import org.apache.dolphinscheduler.common.datasource.BaseConnectionParam; +import org.apache.dolphinscheduler.common.datasource.BaseDataSourceParamDTO; +import org.apache.dolphinscheduler.common.datasource.ConnectionParam; +import org.apache.dolphinscheduler.common.enums.DbType; +import org.apache.dolphinscheduler.common.utils.CommonUtils; +import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.StringUtils; + +import org.apache.commons.collections4.MapUtils; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +public class PrestoDatasourceProcessor extends AbstractDatasourceProcessor { + + @Override + public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) { + PrestoConnectionParam connectionParams = (PrestoConnectionParam) createConnectionParams(connectionJson); + + String[] hostSeperator = connectionParams.getAddress().split(Constants.DOUBLE_SLASH); + String[] hostPortArray = hostSeperator[hostSeperator.length - 1].split(Constants.COMMA); + + PrestoDatasourceParamDTO prestoDatasourceParamDTO = new PrestoDatasourceParamDTO(); + prestoDatasourceParamDTO.setPort(Integer.parseInt(hostPortArray[0].split(Constants.COLON)[1])); + prestoDatasourceParamDTO.setHost(hostPortArray[0].split(Constants.COLON)[0]); + prestoDatasourceParamDTO.setDatabase(connectionParams.getDatabase()); + prestoDatasourceParamDTO.setUserName(connectionParams.getUser()); + prestoDatasourceParamDTO.setOther(parseOther(connectionParams.getOther())); + + return prestoDatasourceParamDTO; + } + + @Override + public BaseConnectionParam createConnectionParams(BaseDataSourceParamDTO datasourceParam) { + PrestoDatasourceParamDTO prestoParam = (PrestoDatasourceParamDTO) datasourceParam; + String address = String.format("%s%s:%s", Constants.JDBC_PRESTO, prestoParam.getHost(), prestoParam.getPort()); + String jdbcUrl = address + "/" + prestoParam.getDatabase(); + + PrestoConnectionParam prestoConnectionParam = new PrestoConnectionParam(); + prestoConnectionParam.setUser(prestoParam.getUserName()); + prestoConnectionParam.setPassword(CommonUtils.encodePassword(prestoParam.getPassword())); + prestoConnectionParam.setOther(transformOther(prestoParam.getOther())); + prestoConnectionParam.setAddress(address); + prestoConnectionParam.setJdbcUrl(jdbcUrl); + prestoConnectionParam.setDatabase(prestoParam.getDatabase()); + + return prestoConnectionParam; + } + + @Override + public ConnectionParam createConnectionParams(String connectionJson) { + return JSONUtils.parseObject(connectionJson, PrestoConnectionParam.class); + } + + @Override + public String getDatasourceDriver() { + return Constants.COM_PRESTO_JDBC_DRIVER; + } + + @Override + public String getJdbcUrl(ConnectionParam connectionParam) { + PrestoConnectionParam prestoConnectionParam = (PrestoConnectionParam) connectionParam; + if (StringUtils.isNotEmpty(prestoConnectionParam.getOther())) { + return String.format("%s?%s", prestoConnectionParam.getJdbcUrl(), prestoConnectionParam.getOther()); + } + return prestoConnectionParam.getJdbcUrl(); + } + + @Override + public Connection getConnection(ConnectionParam connectionParam) throws ClassNotFoundException, SQLException { + PrestoConnectionParam prestoConnectionParam = (PrestoConnectionParam) connectionParam; + Class.forName(getDatasourceDriver()); + return DriverManager.getConnection(getJdbcUrl(connectionParam), + prestoConnectionParam.getUser(), CommonUtils.decodePassword(prestoConnectionParam.getPassword())); + } + + @Override + public DbType getDbType() { + return DbType.PRESTO; + } + + private String transformOther(Map otherMap) { + if (MapUtils.isNotEmpty(otherMap)) { + List list = new ArrayList<>(); + otherMap.forEach((key, value) -> list.add(String.format("%s=%s", key, value))); + return String.join("&", list); + } + return null; + } + + private Map parseOther(String other) { + if (StringUtils.isEmpty(other)) { + return null; + } + Map otherMap = new LinkedHashMap<>(); + String[] configs = other.split("&"); + for (String config : configs) { + otherMap.put(config.split("=")[0], config.split("=")[1]); + } + return otherMap; + } +} diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/spark/SparkConnectionParam.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/spark/SparkConnectionParam.java new file mode 100644 index 0000000000..725d7b8e2b --- /dev/null +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/spark/SparkConnectionParam.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.common.datasource.spark; + +import org.apache.dolphinscheduler.common.datasource.BaseHdfsConnectionParam; + +public class SparkConnectionParam extends BaseHdfsConnectionParam { + @Override + public String toString() { + return "SparkConnectionParam{" + + "user='" + user + '\'' + + ", password='" + password + '\'' + + ", address='" + address + '\'' + + ", database='" + database + '\'' + + ", jdbcUrl='" + jdbcUrl + '\'' + + ", other='" + other + '\'' + + ", principal='" + principal + '\'' + + ", javaSecurityKrb5Conf='" + javaSecurityKrb5Conf + '\'' + + ", loginUserKeytabUsername='" + loginUserKeytabUsername + '\'' + + ", loginUserKeytabPath='" + loginUserKeytabPath + '\'' + + '}'; + } +} diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/SparkDataSource.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/spark/SparkDatasourceParamDTO.java similarity index 50% rename from dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/SparkDataSource.java rename to dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/spark/SparkDatasourceParamDTO.java index df17bb564c..784f7d01d9 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/SparkDataSource.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/spark/SparkDatasourceParamDTO.java @@ -15,44 +15,31 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.dao.datasource; +package org.apache.dolphinscheduler.common.datasource.spark; -import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.datasource.BaseHdfsDatasourceParamDTO; import org.apache.dolphinscheduler.common.enums.DbType; -import org.apache.dolphinscheduler.common.utils.CommonUtils; -import java.sql.Connection; +public class SparkDatasourceParamDTO extends BaseHdfsDatasourceParamDTO { -/** - * data source of spark - */ -public class SparkDataSource extends BaseDataSource { - - /** - * gets the JDBC url for the data source connection - * @return jdbc url - */ @Override - public String driverClassSelector() { - return Constants.ORG_APACHE_HIVE_JDBC_HIVE_DRIVER; + public String toString() { + return "SparkDatasourceParamDTO{" + + "host='" + host + '\'' + + ", port=" + port + + ", database='" + database + '\'' + + ", principal='" + principal + '\'' + + ", userName='" + userName + '\'' + + ", password='" + password + '\'' + + ", other='" + other + '\'' + + ", javaSecurityKrb5Conf='" + javaSecurityKrb5Conf + '\'' + + ", loginUserKeytabUsername='" + loginUserKeytabUsername + '\'' + + ", loginUserKeytabPath='" + loginUserKeytabPath + '\'' + + '}'; } - /** - * @return db type - */ @Override - public DbType dbTypeSelector() { + public DbType getType() { return DbType.SPARK; } - - /** - * the data source test connection - * @return Connection Connection - * @throws Exception Exception - */ - @Override - public Connection getConnection() throws Exception { - CommonUtils.loadKerberosConf(getJavaSecurityKrb5Conf(), getLoginUserKeytabUsername(), getLoginUserKeytabPath()); - return super.getConnection(); - } } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/spark/SparkDatasourceProcessor.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/spark/SparkDatasourceProcessor.java new file mode 100644 index 0000000000..83365ead22 --- /dev/null +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/spark/SparkDatasourceProcessor.java @@ -0,0 +1,154 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.common.datasource.spark; + +import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.datasource.AbstractDatasourceProcessor; +import org.apache.dolphinscheduler.common.datasource.BaseConnectionParam; +import org.apache.dolphinscheduler.common.datasource.BaseDataSourceParamDTO; +import org.apache.dolphinscheduler.common.datasource.ConnectionParam; +import org.apache.dolphinscheduler.common.enums.DbType; +import org.apache.dolphinscheduler.common.utils.CommonUtils; +import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.StringUtils; + +import org.apache.commons.collections4.MapUtils; + +import java.io.IOException; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +public class SparkDatasourceProcessor extends AbstractDatasourceProcessor { + + @Override + public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) { + SparkConnectionParam connectionParams = (SparkConnectionParam) createConnectionParams(connectionJson); + + SparkDatasourceParamDTO sparkDatasourceParamDTO = new SparkDatasourceParamDTO(); + sparkDatasourceParamDTO.setDatabase(connectionParams.getDatabase()); + sparkDatasourceParamDTO.setUserName(connectionParams.getUser()); + sparkDatasourceParamDTO.setOther(parseOther(connectionParams.getOther())); + sparkDatasourceParamDTO.setJavaSecurityKrb5Conf(connectionParams.getJavaSecurityKrb5Conf()); + sparkDatasourceParamDTO.setLoginUserKeytabPath(connectionParams.getLoginUserKeytabPath()); + sparkDatasourceParamDTO.setLoginUserKeytabUsername(connectionParams.getLoginUserKeytabUsername()); + + StringBuilder hosts = new StringBuilder(); + String[] tmpArray = connectionParams.getAddress().split(Constants.DOUBLE_SLASH); + String[] hostPortArray = tmpArray[tmpArray.length - 1].split(Constants.COMMA); + Arrays.stream(hostPortArray).forEach(hostPort -> hosts.append(hostPort.split(Constants.COLON)[0]).append(Constants.COMMA)); + hosts.deleteCharAt(hosts.length() - 1); + + sparkDatasourceParamDTO.setHost(hosts.toString()); + sparkDatasourceParamDTO.setPort(Integer.parseInt(hostPortArray[0].split(Constants.COLON)[1])); + + return sparkDatasourceParamDTO; + } + + @Override + public BaseConnectionParam createConnectionParams(BaseDataSourceParamDTO dataSourceParam) { + StringBuilder address = new StringBuilder(); + SparkDatasourceParamDTO sparkDatasourceParam = (SparkDatasourceParamDTO) dataSourceParam; + address.append(Constants.JDBC_HIVE_2); + for (String zkHost : sparkDatasourceParam.getHost().split(",")) { + address.append(String.format("%s:%s,", zkHost, sparkDatasourceParam.getPort())); + } + address.deleteCharAt(address.length() - 1); + + String jdbcUrl = address + "/" + sparkDatasourceParam.getDatabase(); + if (CommonUtils.getKerberosStartupState()) { + jdbcUrl += ";principal=" + sparkDatasourceParam.getPrincipal(); + } + + SparkConnectionParam sparkConnectionParam = new SparkConnectionParam(); + sparkConnectionParam.setPassword(CommonUtils.encodePassword(sparkDatasourceParam.getPassword())); + sparkConnectionParam.setUser(sparkDatasourceParam.getUserName()); + sparkConnectionParam.setOther(transformOther(sparkDatasourceParam.getOther())); + sparkConnectionParam.setDatabase(sparkDatasourceParam.getDatabase()); + sparkConnectionParam.setAddress(address.toString()); + sparkConnectionParam.setJdbcUrl(jdbcUrl); + if (CommonUtils.getKerberosStartupState()) { + sparkConnectionParam.setPrincipal(sparkDatasourceParam.getPrincipal()); + sparkConnectionParam.setJavaSecurityKrb5Conf(sparkDatasourceParam.getJavaSecurityKrb5Conf()); + sparkConnectionParam.setLoginUserKeytabPath(sparkDatasourceParam.getLoginUserKeytabPath()); + sparkConnectionParam.setLoginUserKeytabUsername(sparkDatasourceParam.getLoginUserKeytabUsername()); + } + + return sparkConnectionParam; + } + + @Override + public ConnectionParam createConnectionParams(String connectionJson) { + return JSONUtils.parseObject(connectionJson, SparkConnectionParam.class); + } + + @Override + public String getDatasourceDriver() { + return Constants.ORG_APACHE_HIVE_JDBC_HIVE_DRIVER; + } + + @Override + public String getJdbcUrl(ConnectionParam connectionParam) { + SparkConnectionParam sparkConnectionParam = (SparkConnectionParam) connectionParam; + if (StringUtils.isNotEmpty(sparkConnectionParam.getOther())) { + return String.format("%s;%s", sparkConnectionParam.getJdbcUrl(), sparkConnectionParam.getOther()); + } + return sparkConnectionParam.getJdbcUrl(); + } + + @Override + public Connection getConnection(ConnectionParam connectionParam) throws IOException, ClassNotFoundException, SQLException { + SparkConnectionParam sparkConnectionParam = (SparkConnectionParam) connectionParam; + CommonUtils.loadKerberosConf(sparkConnectionParam.getJavaSecurityKrb5Conf(), + sparkConnectionParam.getLoginUserKeytabUsername(), sparkConnectionParam.getLoginUserKeytabPath()); + Class.forName(getDatasourceDriver()); + return DriverManager.getConnection(getJdbcUrl(sparkConnectionParam), + sparkConnectionParam.getUser(), CommonUtils.decodePassword(sparkConnectionParam.getPassword())); + } + + @Override + public DbType getDbType() { + return DbType.SPARK; + } + + private String transformOther(Map otherMap) { + if (MapUtils.isEmpty(otherMap)) { + return null; + } + List stringBuilder = otherMap.entrySet().stream() + .map(entry -> String.format("%s=%s", entry.getKey(), entry.getValue())).collect(Collectors.toList()); + return String.join(";", stringBuilder); + } + + private Map parseOther(String other) { + if (StringUtils.isEmpty(other)) { + return null; + } + Map otherMap = new LinkedHashMap<>(); + String[] configs = other.split(";"); + for (String config : configs) { + otherMap.put(config.split("=")[0], config.split("=")[1]); + } + return otherMap; + } +} diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/sqlserver/SqlServerConnectionParam.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/sqlserver/SqlServerConnectionParam.java new file mode 100644 index 0000000000..ff5225771a --- /dev/null +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/sqlserver/SqlServerConnectionParam.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.common.datasource.sqlserver; + +import org.apache.dolphinscheduler.common.datasource.BaseConnectionParam; + +public class SqlServerConnectionParam extends BaseConnectionParam { + @Override + public String toString() { + return "SqlServerConnectionParam{" + + "user='" + user + '\'' + + ", password='" + password + '\'' + + ", address='" + address + '\'' + + ", database='" + database + '\'' + + ", jdbcUrl='" + jdbcUrl + '\'' + + ", other='" + other + '\'' + + '}'; + } +} diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/SQLServerDataSource.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/sqlserver/SqlServerDatasourceParamDTO.java similarity index 52% rename from dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/SQLServerDataSource.java rename to dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/sqlserver/SqlServerDatasourceParamDTO.java index 78062357e2..62c757638f 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/SQLServerDataSource.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/sqlserver/SqlServerDatasourceParamDTO.java @@ -15,46 +15,29 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.dao.datasource; +package org.apache.dolphinscheduler.common.datasource.sqlserver; -import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.datasource.BaseDataSourceParamDTO; import org.apache.dolphinscheduler.common.enums.DbType; -import org.apache.dolphinscheduler.common.utils.StringUtils; -/** - * data source of SQL Server - */ -public class SQLServerDataSource extends BaseDataSource { +public class SqlServerDatasourceParamDTO extends BaseDataSourceParamDTO { - /** - * gets the JDBC url for the data source connection - * @return jdbc url - */ @Override - public String getJdbcUrl() { - String jdbcUrl = getAddress(); - jdbcUrl += ";databaseName=" + getDatabase(); - - if (StringUtils.isNotEmpty(getOther())) { - jdbcUrl += ";" + getOther(); - } - - return jdbcUrl; + public String toString() { + return "SqlServerDatasourceParamDTO{" + + "name='" + name + '\'' + + ", note='" + note + '\'' + + ", host='" + host + '\'' + + ", port=" + port + + ", database='" + database + '\'' + + ", userName='" + userName + '\'' + + ", password='" + password + '\'' + + ", other='" + other + '\'' + + '}'; } - /** - * @return driver class - */ @Override - public String driverClassSelector() { - return Constants.COM_SQLSERVER_JDBC_DRIVER; - } - - /** - * @return db type - */ - @Override - public DbType dbTypeSelector() { + public DbType getType() { return DbType.SQLSERVER; } } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/sqlserver/SqlServerDatasourceProcessor.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/sqlserver/SqlServerDatasourceProcessor.java new file mode 100644 index 0000000000..d4afa6e312 --- /dev/null +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/datasource/sqlserver/SqlServerDatasourceProcessor.java @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.common.datasource.sqlserver; + +import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.datasource.AbstractDatasourceProcessor; +import org.apache.dolphinscheduler.common.datasource.BaseConnectionParam; +import org.apache.dolphinscheduler.common.datasource.BaseDataSourceParamDTO; +import org.apache.dolphinscheduler.common.datasource.ConnectionParam; +import org.apache.dolphinscheduler.common.enums.DbType; +import org.apache.dolphinscheduler.common.utils.CommonUtils; +import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.StringUtils; + +import org.apache.commons.collections4.MapUtils; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.util.LinkedHashMap; +import java.util.Map; + +public class SqlServerDatasourceProcessor extends AbstractDatasourceProcessor { + + @Override + public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) { + SqlServerConnectionParam connectionParams = (SqlServerConnectionParam) createConnectionParams(connectionJson); + String[] hostSeperator = connectionParams.getAddress().split(Constants.DOUBLE_SLASH); + String[] hostPortArray = hostSeperator[hostSeperator.length - 1].split(Constants.COMMA); + + SqlServerDatasourceParamDTO sqlServerDatasourceParamDTO = new SqlServerDatasourceParamDTO(); + sqlServerDatasourceParamDTO.setDatabase(connectionParams.getDatabase()); + sqlServerDatasourceParamDTO.setUserName(connectionParams.getUser()); + sqlServerDatasourceParamDTO.setOther(parseOther(connectionParams.getOther())); + sqlServerDatasourceParamDTO.setPort(Integer.parseInt(hostPortArray[0].split(Constants.COLON)[1])); + sqlServerDatasourceParamDTO.setHost(hostPortArray[0].split(Constants.COLON)[0]); + return sqlServerDatasourceParamDTO; + } + + @Override + public BaseConnectionParam createConnectionParams(BaseDataSourceParamDTO datasourceParam) { + SqlServerDatasourceParamDTO sqlServerParam = (SqlServerDatasourceParamDTO) datasourceParam; + String address = String.format("%s%s:%s", Constants.JDBC_SQLSERVER, sqlServerParam.getHost(), sqlServerParam.getPort()); + String jdbcUrl = address + ";databaseName=" + sqlServerParam.getDatabase(); + + SqlServerConnectionParam sqlServerConnectionParam = new SqlServerConnectionParam(); + sqlServerConnectionParam.setAddress(address); + sqlServerConnectionParam.setDatabase(sqlServerParam.getDatabase()); + sqlServerConnectionParam.setJdbcUrl(jdbcUrl); + sqlServerConnectionParam.setOther(transformOther(sqlServerParam.getOther())); + sqlServerConnectionParam.setUser(sqlServerParam.getUserName()); + sqlServerConnectionParam.setPassword(CommonUtils.encodePassword(sqlServerParam.getPassword())); + return sqlServerConnectionParam; + } + + @Override + public BaseConnectionParam createConnectionParams(String connectionJson) { + return JSONUtils.parseObject(connectionJson, SqlServerConnectionParam.class); + } + + @Override + public String getDatasourceDriver() { + return Constants.COM_SQLSERVER_JDBC_DRIVER; + } + + @Override + public String getJdbcUrl(ConnectionParam connectionParam) { + SqlServerConnectionParam sqlServerConnectionParam = (SqlServerConnectionParam) connectionParam; + + if (StringUtils.isNotEmpty(sqlServerConnectionParam.getOther())) { + return String.format("%s;%s", sqlServerConnectionParam.getJdbcUrl(), sqlServerConnectionParam.getOther()); + } + return sqlServerConnectionParam.getJdbcUrl(); + } + + @Override + public Connection getConnection(ConnectionParam connectionParam) throws ClassNotFoundException, SQLException { + SqlServerConnectionParam sqlServerConnectionParam = (SqlServerConnectionParam) connectionParam; + Class.forName(getDatasourceDriver()); + return DriverManager.getConnection(getJdbcUrl(connectionParam), sqlServerConnectionParam.getUser(), + CommonUtils.decodePassword(sqlServerConnectionParam.getPassword())); + } + + @Override + public DbType getDbType() { + return DbType.SQLSERVER; + } + + private String transformOther(Map otherMap) { + if (MapUtils.isEmpty(otherMap)) { + return null; + } + StringBuilder stringBuilder = new StringBuilder(); + otherMap.forEach((key, value) -> stringBuilder.append(String.format("%s=%s;", key, value))); + return stringBuilder.toString(); + } + + private Map parseOther(String other) { + if (StringUtils.isEmpty(other)) { + return null; + } + Map otherMap = new LinkedHashMap<>(); + for (String config : other.split(";")) { + otherMap.put(config.split("=")[0], config.split("=")[1]); + } + return otherMap; + } +} diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/DbConnectType.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/DbConnectType.java index ef0f454ff6..0088bf5dbd 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/DbConnectType.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/DbConnectType.java @@ -14,6 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.common.enums; import com.baomidou.mybatisplus.annotation.EnumValue; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/DependResult.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/DependResult.java index 3e305f3920..f906c866aa 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/DependResult.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/DependResult.java @@ -26,6 +26,7 @@ public enum DependResult { * 0 success * 1 waiting * 2 failed + * 3 non execution */ - SUCCESS, WAITING, FAILED + SUCCESS, WAITING, FAILED, NON_EXEC } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CommonUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CommonUtils.java index e68c342ea4..58e8ac7780 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CommonUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CommonUtils.java @@ -24,6 +24,7 @@ import org.apache.commons.codec.binary.Base64; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; +import java.io.IOException; import java.net.URL; import java.nio.charset.StandardCharsets; @@ -103,9 +104,9 @@ public class CommonUtils { * @param javaSecurityKrb5Conf javaSecurityKrb5Conf * @param loginUserKeytabUsername loginUserKeytabUsername * @param loginUserKeytabPath loginUserKeytabPath - * @throws Exception errors + * @throws IOException errors */ - public static void loadKerberosConf(String javaSecurityKrb5Conf, String loginUserKeytabUsername, String loginUserKeytabPath) throws Exception { + public static void loadKerberosConf(String javaSecurityKrb5Conf, String loginUserKeytabUsername, String loginUserKeytabPath) throws IOException { if (CommonUtils.getKerberosStartupState()) { System.setProperty(Constants.JAVA_SECURITY_KRB5_CONF, StringUtils.defaultIfBlank(javaSecurityKrb5Conf, PropertyUtils.getString(Constants.JAVA_SECURITY_KRB5_CONF_PATH))); Configuration configuration = new Configuration(); diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/IpUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/IpUtils.java deleted file mode 100644 index 63d43e7b69..0000000000 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/IpUtils.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.common.utils; - -/** - * http utils - */ -public class IpUtils { - - private IpUtils() { - throw new UnsupportedOperationException("Construct IpUtils"); - } - - public static final String DOT = "."; - - /** - * ip str to long

- * - * @param ipStr ip string - * @return ip to long - */ - public static Long ipToLong(String ipStr) { - String[] ipSet = ipStr.split("\\" + DOT); - - return Long.parseLong(ipSet[0]) << 24 | Long.parseLong(ipSet[1]) << 16 | Long.parseLong(ipSet[2]) << 8 | Long.parseLong(ipSet[3]); - } - - /** - * long to ip - * - * @param ipLong the long number converted from IP - * @return String - */ - public static String longToIp(long ipLong) { - long[] ipNumbers = new long[4]; - long tmp = 0xFF; - ipNumbers[0] = ipLong >> 24 & tmp; - ipNumbers[1] = ipLong >> 16 & tmp; - ipNumbers[2] = ipLong >> 8 & tmp; - ipNumbers[3] = ipLong & tmp; - - return ipNumbers[0] + DOT - + ipNumbers[1] + DOT - + ipNumbers[2] + DOT - + ipNumbers[3]; - } - -} diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/NetUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/NetUtils.java index 8a283c7a74..4314089ef4 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/NetUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/NetUtils.java @@ -86,6 +86,9 @@ public class NetUtils { if (STS_PATTERN.matcher(host).find()) { return String.format("%s.%s", host, host.replaceFirst("\\d+$", "headless")); } + } else if (canonicalHost.contains(".")) { + String[] items = canonicalHost.split("\\."); + return String.format("%s.%s", items[0], items[1]); } return canonicalHost; } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java index 09b4e06d9f..3851f77e94 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java @@ -17,11 +17,8 @@ package org.apache.dolphinscheduler.common.utils; -import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.shell.ShellExecutor; -import org.apache.commons.configuration.Configuration; - import java.io.BufferedReader; import java.io.FileInputStream; import java.io.IOException; @@ -85,7 +82,7 @@ public class OSUtils { */ public static double memoryUsage() { GlobalMemory memory = hal.getMemory(); - double memoryUsage = (memory.getTotal() - memory.getAvailable() - memory.getSwapUsed()) * 0.1 / memory.getTotal() * 10; + double memoryUsage = (memory.getTotal() - memory.getAvailable()) * 1.0 / memory.getTotal(); DecimalFormat df = new DecimalFormat(TWO_DECIMAL); df.setRoundingMode(RoundingMode.HALF_UP); @@ -101,12 +98,11 @@ public class OSUtils { */ public static double availablePhysicalMemorySize() { GlobalMemory memory = hal.getMemory(); - double availablePhysicalMemorySize = (memory.getAvailable() + memory.getSwapUsed()) / 1024.0 / 1024 / 1024; + double availablePhysicalMemorySize = memory.getAvailable() / 1024.0 / 1024 / 1024; DecimalFormat df = new DecimalFormat(TWO_DECIMAL); df.setRoundingMode(RoundingMode.HALF_UP); return Double.parseDouble(df.format(availablePhysicalMemorySize)); - } /** @@ -116,13 +112,13 @@ public class OSUtils { * * @return available Physical Memory Size, unit: G */ - public static double totalMemorySize() { + public static double totalPhysicalMemorySize() { GlobalMemory memory = hal.getMemory(); - double availablePhysicalMemorySize = memory.getTotal() / 1024.0 / 1024 / 1024; + double totalPhysicalMemorySize = memory.getTotal() / 1024.0 / 1024 / 1024; DecimalFormat df = new DecimalFormat(TWO_DECIMAL); df.setRoundingMode(RoundingMode.HALF_UP); - return Double.parseDouble(df.format(availablePhysicalMemorySize)); + return Double.parseDouble(df.format(totalPhysicalMemorySize)); } /** @@ -486,43 +482,22 @@ public class OSUtils { /** * check memory and cpu usage * - * @param systemCpuLoad systemCpuLoad - * @param systemReservedMemory systemReservedMemory + * @param maxCpuloadAvg maxCpuloadAvg + * @param reservedMemory reservedMemory * @return check memory and cpu usage */ - public static Boolean checkResource(double systemCpuLoad, double systemReservedMemory) { + public static Boolean checkResource(double maxCpuloadAvg, double reservedMemory) { // system load average double loadAverage = loadAverage(); // system available physical memory double availablePhysicalMemorySize = availablePhysicalMemorySize(); - - if (loadAverage > systemCpuLoad || availablePhysicalMemorySize < systemReservedMemory) { - logger.warn("load is too high or availablePhysicalMemorySize(G) is too low, it's availablePhysicalMemorySize(G):{},loadAvg:{}", availablePhysicalMemorySize, loadAverage); + if (loadAverage > maxCpuloadAvg || availablePhysicalMemorySize < reservedMemory) { + logger.warn("current cpu load average {} is too high or available memory {}G is too low, under max.cpuload.avg={} and reserved.memory={}G", + loadAverage, availablePhysicalMemorySize, maxCpuloadAvg, reservedMemory); return false; } else { return true; } } - /** - * check memory and cpu usage - * - * @param conf conf - * @param isMaster is master - * @return check memory and cpu usage - */ - public static Boolean checkResource(Configuration conf, Boolean isMaster) { - double systemCpuLoad; - double systemReservedMemory; - - if (Boolean.TRUE.equals(isMaster)) { - systemCpuLoad = conf.getDouble(Constants.MASTER_MAX_CPULOAD_AVG, Constants.DEFAULT_MASTER_CPU_LOAD); - systemReservedMemory = conf.getDouble(Constants.MASTER_RESERVED_MEMORY, Constants.DEFAULT_MASTER_RESERVED_MEMORY); - } else { - systemCpuLoad = conf.getDouble(Constants.WORKER_MAX_CPULOAD_AVG, Constants.DEFAULT_WORKER_CPU_LOAD); - systemReservedMemory = conf.getDouble(Constants.WORKER_RESERVED_MEMORY, Constants.DEFAULT_WORKER_RESERVED_MEMORY); - } - return checkResource(systemCpuLoad, systemReservedMemory); - } - } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ParameterUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ParameterUtils.java index 034217ef82..e010e062b3 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ParameterUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ParameterUtils.java @@ -45,7 +45,7 @@ public class ParameterUtils { private static final Logger logger = LoggerFactory.getLogger(ParameterUtils.class); - private static final String DATE_PARSE_PATTERN = "\\$\\[([^\\]]+)]"; + private static final String DATE_PARSE_PATTERN = "\\$\\[([^\\$\\]]+)]"; private static final String DATE_START_PATTERN = "^[0-9]"; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java index 199655e254..0b417a4dd3 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java @@ -57,19 +57,12 @@ public class PropertyUtils { */ public static synchronized void loadPropertyFile(String... propertyFiles) { for (String fileName : propertyFiles) { - InputStream fis = null; - try { - fis = PropertyUtils.class.getResourceAsStream(fileName); + try (InputStream fis = PropertyUtils.class.getResourceAsStream(fileName);) { properties.load(fis); } catch (IOException e) { logger.error(e.getMessage(), e); - if (fis != null) { - IOUtils.closeQuietly(fis); - } System.exit(1); - } finally { - IOUtils.closeQuietly(fis); } } } diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/datasource/DatasourceUtilTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/datasource/DatasourceUtilTest.java new file mode 100644 index 0000000000..8ebc5b17d4 --- /dev/null +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/datasource/DatasourceUtilTest.java @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.common.datasource; + +import org.apache.dolphinscheduler.common.datasource.mysql.MysqlConnectionParam; +import org.apache.dolphinscheduler.common.datasource.mysql.MysqlDatasourceParamDTO; +import org.apache.dolphinscheduler.common.datasource.mysql.MysqlDatasourceProcessor; +import org.apache.dolphinscheduler.common.enums.DbType; +import org.apache.dolphinscheduler.common.utils.JSONUtils; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; + +@RunWith(PowerMockRunner.class) +@PrepareForTest({Class.class, DriverManager.class, MysqlDatasourceProcessor.class}) +public class DatasourceUtilTest { + + @Test + public void testCheckDatasourceParam() { + MysqlDatasourceParamDTO mysqlDatasourceParamDTO = new MysqlDatasourceParamDTO(); + mysqlDatasourceParamDTO.setHost("localhost"); + mysqlDatasourceParamDTO.setDatabase("default"); + mysqlDatasourceParamDTO.setOther(null); + DatasourceUtil.checkDatasourceParam(mysqlDatasourceParamDTO); + Assert.assertTrue(true); + } + + @Test + public void testBuildConnectionParams() { + MysqlDatasourceParamDTO mysqlDatasourceParamDTO = new MysqlDatasourceParamDTO(); + mysqlDatasourceParamDTO.setHost("localhost"); + mysqlDatasourceParamDTO.setDatabase("default"); + mysqlDatasourceParamDTO.setUserName("root"); + mysqlDatasourceParamDTO.setPort(3306); + mysqlDatasourceParamDTO.setPassword("123456"); + ConnectionParam connectionParam = DatasourceUtil.buildConnectionParams(mysqlDatasourceParamDTO); + Assert.assertNotNull(connectionParam); + } + + @Test + public void testBuildConnectionParams2() { + MysqlDatasourceParamDTO mysqlDatasourceParamDTO = new MysqlDatasourceParamDTO(); + mysqlDatasourceParamDTO.setHost("localhost"); + mysqlDatasourceParamDTO.setDatabase("default"); + mysqlDatasourceParamDTO.setUserName("root"); + mysqlDatasourceParamDTO.setPort(3306); + mysqlDatasourceParamDTO.setPassword("123456"); + ConnectionParam connectionParam = DatasourceUtil.buildConnectionParams(DbType.MYSQL, JSONUtils.toJsonString(mysqlDatasourceParamDTO)); + Assert.assertNotNull(connectionParam); + } + + @Test + public void testGetConnection() throws ClassNotFoundException, SQLException { + PowerMockito.mockStatic(Class.class); + PowerMockito.when(Class.forName(Mockito.any())).thenReturn(null); + PowerMockito.mockStatic(DriverManager.class); + PowerMockito.when(DriverManager.getConnection(Mockito.any(), Mockito.any(), Mockito.any())).thenReturn(null); + + MysqlConnectionParam connectionParam = new MysqlConnectionParam(); + connectionParam.setUser("root"); + connectionParam.setPassword("123456"); + Connection connection = DatasourceUtil.getConnection(DbType.MYSQL, connectionParam); + + Assert.assertNull(connection); + + } + + @Test + public void testGetJdbcUrl() { + MysqlConnectionParam mysqlConnectionParam = new MysqlConnectionParam(); + mysqlConnectionParam.setJdbcUrl("jdbc:mysql://localhost:3308"); + String jdbcUrl = DatasourceUtil.getJdbcUrl(DbType.MYSQL, mysqlConnectionParam); + Assert.assertEquals("jdbc:mysql://localhost:3308?allowLoadLocalInfile=false&autoDeserialize=false&allowLocalInfile=false&allowUrlInLocalInfile=false", + jdbcUrl); + } + + @Test + public void testBuildDatasourceParamDTO() { + MysqlConnectionParam connectionParam = new MysqlConnectionParam(); + connectionParam.setJdbcUrl("jdbc:mysql://localhost:3308?allowLoadLocalInfile=false&autoDeserialize=false&allowLocalInfile=false&allowUrlInLocalInfile=false"); + connectionParam.setAddress("jdbc:mysql://localhost:3308"); + connectionParam.setUser("root"); + connectionParam.setPassword("123456"); + + Assert.assertNotNull(DatasourceUtil.buildDatasourceParamDTO(DbType.MYSQL, JSONUtils.toJsonString(connectionParam))); + + } + + @Test + public void testGetDatasourceProcessor() { + Assert.assertNotNull(DatasourceUtil.getDatasourceProcessor(DbType.MYSQL)); + Assert.assertNotNull(DatasourceUtil.getDatasourceProcessor(DbType.POSTGRESQL)); + Assert.assertNotNull(DatasourceUtil.getDatasourceProcessor(DbType.HIVE)); + Assert.assertNotNull(DatasourceUtil.getDatasourceProcessor(DbType.SPARK)); + Assert.assertNotNull(DatasourceUtil.getDatasourceProcessor(DbType.CLICKHOUSE)); + Assert.assertNotNull(DatasourceUtil.getDatasourceProcessor(DbType.ORACLE)); + Assert.assertNotNull(DatasourceUtil.getDatasourceProcessor(DbType.SQLSERVER)); + Assert.assertNotNull(DatasourceUtil.getDatasourceProcessor(DbType.DB2)); + Assert.assertNotNull(DatasourceUtil.getDatasourceProcessor(DbType.PRESTO)); + } + + @Test(expected = Exception.class) + public void testGetDatasourceProcessorError() { + DatasourceUtil.getDatasourceProcessor(null); + } +} \ No newline at end of file diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/datasource/clickhouse/ClickHouseDatasourceProcessorTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/datasource/clickhouse/ClickHouseDatasourceProcessorTest.java new file mode 100644 index 0000000000..75c1f3c17d --- /dev/null +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/datasource/clickhouse/ClickHouseDatasourceProcessorTest.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.common.datasource.clickhouse; + +import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.enums.DbType; + +import java.sql.DriverManager; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; + +@RunWith(PowerMockRunner.class) +@PrepareForTest({Class.class, DriverManager.class}) +public class ClickHouseDatasourceProcessorTest { + + private ClickHouseDatasourceProcessor clickHouseDatasourceProcessor = new ClickHouseDatasourceProcessor(); + + @Test + public void testCreateConnectionParams() { + ClickHouseDatasourceParamDTO clickhouseConnectionParam = new ClickHouseDatasourceParamDTO(); + clickhouseConnectionParam.setUserName("user"); + clickhouseConnectionParam.setPassword("password"); + clickhouseConnectionParam.setHost("localhost"); + clickhouseConnectionParam.setPort(8123); + clickhouseConnectionParam.setDatabase("default"); + ClickhouseConnectionParam connectionParams = (ClickhouseConnectionParam) clickHouseDatasourceProcessor + .createConnectionParams(clickhouseConnectionParam); + Assert.assertNotNull(connectionParams); + Assert.assertEquals("jdbc:clickhouse://localhost:8123", connectionParams.getAddress()); + Assert.assertEquals("jdbc:clickhouse://localhost:8123/default", connectionParams.getJdbcUrl()); + } + + @Test + public void testCreateConnectionParams2() { + String connectionParamJson = "{\"address\":\"jdbc:clickhouse://localhost:8123\",\"database\":\"default\"," + + "\"jdbcUrl\":\"jdbc:clickhouse://localhost:8123/default\",\"user\":\"default\",\"password\":\"123456\"}"; + ClickhouseConnectionParam clickhouseConnectionParam = (ClickhouseConnectionParam) clickHouseDatasourceProcessor + .createConnectionParams(connectionParamJson); + Assert.assertNotNull(clickhouseConnectionParam); + Assert.assertEquals("default", clickhouseConnectionParam.getUser()); + Assert.assertEquals("123456", clickhouseConnectionParam.getPassword()); + } + + @Test + public void testGetDatasourceDriver() { + Assert.assertNotNull(clickHouseDatasourceProcessor.getDatasourceDriver()); + Assert.assertEquals(Constants.COM_CLICKHOUSE_JDBC_DRIVER, clickHouseDatasourceProcessor.getDatasourceDriver()); + } + + @Test + public void testGetJdbcUrl() { + ClickhouseConnectionParam connectionParam = new ClickhouseConnectionParam(); + connectionParam.setUser("default"); + connectionParam.setJdbcUrl("jdbc:clickhouse://localhost:8123/default"); + connectionParam.setOther("other=other1"); + String jdbcUrl = clickHouseDatasourceProcessor.getJdbcUrl(connectionParam); + Assert.assertEquals("jdbc:clickhouse://localhost:8123/default?other=other1", jdbcUrl); + } + + @Test + public void testGetDbType() { + Assert.assertEquals(DbType.CLICKHOUSE, clickHouseDatasourceProcessor.getDbType()); + } +} \ No newline at end of file diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/datasource/db2/Db2DatasourceProcessorTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/datasource/db2/Db2DatasourceProcessorTest.java new file mode 100644 index 0000000000..21b1bc4663 --- /dev/null +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/datasource/db2/Db2DatasourceProcessorTest.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.common.datasource.db2; + +import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.enums.DbType; + +import java.sql.DriverManager; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; + +@RunWith(PowerMockRunner.class) +@PrepareForTest({Class.class, DriverManager.class}) +public class Db2DatasourceProcessorTest { + + private Db2DatasourceProcessor db2DatasourceProcessor = new Db2DatasourceProcessor(); + + @Test + public void testCreateConnectionParams() { + Db2DatasourceParamDTO db2DatasourceParamDTO = new Db2DatasourceParamDTO(); + db2DatasourceParamDTO.setUserName("root"); + db2DatasourceParamDTO.setPassword("123456"); + db2DatasourceParamDTO.setHost("localhost"); + db2DatasourceParamDTO.setPort(5142); + db2DatasourceParamDTO.setDatabase("default"); + + Db2ConnectionParam connectionParams = (Db2ConnectionParam) db2DatasourceProcessor + .createConnectionParams(db2DatasourceParamDTO); + Assert.assertNotNull(connectionParams); + Assert.assertEquals("jdbc:db2://localhost:5142", connectionParams.getAddress()); + Assert.assertEquals("jdbc:db2://localhost:5142/default", connectionParams.getJdbcUrl()); + } + + @Test + public void testCreateConnectionParams2() { + String connectionJson = "{\"user\":\"root\",\"password\":\"123456\",\"address\":\"jdbc:db2://localhost:5142\"" + + ",\"database\":\"default\",\"jdbcUrl\":\"jdbc:db2://localhost:5142/default\"}"; + Db2ConnectionParam connectionParams = (Db2ConnectionParam) db2DatasourceProcessor + .createConnectionParams(connectionJson); + Assert.assertNotNull(connectionJson); + Assert.assertEquals("root", connectionParams.getUser()); + + } + + @Test + public void testGetDatasourceDriver() { + Assert.assertEquals(Constants.COM_DB2_JDBC_DRIVER, db2DatasourceProcessor.getDatasourceDriver()); + } + + @Test + public void testGetJdbcUrl() { + Db2ConnectionParam db2ConnectionParam = new Db2ConnectionParam(); + db2ConnectionParam.setJdbcUrl("jdbc:db2://localhost:5142/default"); + db2ConnectionParam.setOther("other=other"); + String jdbcUrl = db2DatasourceProcessor.getJdbcUrl(db2ConnectionParam); + Assert.assertEquals("jdbc:db2://localhost:5142/default;other=other", jdbcUrl); + } + + @Test + public void testGetDbType() { + Assert.assertEquals(DbType.DB2, db2DatasourceProcessor.getDbType()); + } + +} \ No newline at end of file diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/datasource/hive/HiveDatasourceProcessorTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/datasource/hive/HiveDatasourceProcessorTest.java new file mode 100644 index 0000000000..230b0855e5 --- /dev/null +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/datasource/hive/HiveDatasourceProcessorTest.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.common.datasource.hive; + +import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.enums.DbType; +import org.apache.dolphinscheduler.common.utils.JSONUtils; + +import java.sql.DriverManager; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; + +@RunWith(PowerMockRunner.class) +@PrepareForTest({Class.class, DriverManager.class}) +public class HiveDatasourceProcessorTest { + + private HiveDatasourceProcessor hiveDatasourceProcessor = new HiveDatasourceProcessor(); + + @Test + public void testCreateConnectionParams() { + HiveDataSourceParamDTO hiveDataSourceParamDTO = new HiveDataSourceParamDTO(); + hiveDataSourceParamDTO.setHost("localhost1,localhost2"); + hiveDataSourceParamDTO.setPort(5142); + hiveDataSourceParamDTO.setUserName("default"); + hiveDataSourceParamDTO.setDatabase("default"); + HiveConnectionParam connectionParams = (HiveConnectionParam) hiveDatasourceProcessor + .createConnectionParams(hiveDataSourceParamDTO); + System.out.println(JSONUtils.toJsonString(connectionParams)); + Assert.assertNotNull(connectionParams); + Assert.assertEquals("jdbc:hive2://localhost1:5142,localhost2:5142", connectionParams.getAddress()); + } + + @Test + public void testCreateConnectionParams2() { + String connectionParam = "{\"user\":\"default\",\"address\":\"jdbc:hive2://localhost1:5142,localhost2:5142\"" + + ",\"jdbcUrl\":\"jdbc:hive2://localhost1:5142,localhost2:5142/default\"}"; + HiveConnectionParam connectionParams = (HiveConnectionParam) hiveDatasourceProcessor + .createConnectionParams(connectionParam); + Assert.assertNotNull(connectionParam); + Assert.assertEquals("default", connectionParams.getUser()); + } + + @Test + public void testGetDatasourceDriver() { + Assert.assertEquals(Constants.ORG_APACHE_HIVE_JDBC_HIVE_DRIVER, hiveDatasourceProcessor.getDatasourceDriver()); + } + + @Test + public void testGetJdbcUrl() { + HiveConnectionParam connectionParam = new HiveConnectionParam(); + connectionParam.setJdbcUrl("jdbc:hive2://localhost1:5142,localhost2:5142/default"); + Assert.assertEquals("jdbc:hive2://localhost1:5142,localhost2:5142/default", + hiveDatasourceProcessor.getJdbcUrl(connectionParam)); + } + + @Test + public void testGetDbType() { + Assert.assertEquals(DbType.HIVE, hiveDatasourceProcessor.getDbType()); + } +} \ No newline at end of file diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/datasource/mysql/MysqlDatasourceProcessorTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/datasource/mysql/MysqlDatasourceProcessorTest.java new file mode 100644 index 0000000000..89bf4b15f1 --- /dev/null +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/datasource/mysql/MysqlDatasourceProcessorTest.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.common.datasource.mysql; + +import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.enums.DbType; +import org.apache.dolphinscheduler.common.utils.JSONUtils; + +import java.sql.DriverManager; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; + +@RunWith(PowerMockRunner.class) +@PrepareForTest({Class.class, DriverManager.class}) +public class MysqlDatasourceProcessorTest { + + private MysqlDatasourceProcessor mysqlDatasourceProcessor = new MysqlDatasourceProcessor(); + + @Test + public void testCreateConnectionParams() { + MysqlDatasourceParamDTO mysqlDatasourceParamDTO = new MysqlDatasourceParamDTO(); + mysqlDatasourceParamDTO.setUserName("root"); + mysqlDatasourceParamDTO.setPassword("123456"); + mysqlDatasourceParamDTO.setHost("localhost"); + mysqlDatasourceParamDTO.setPort(3306); + mysqlDatasourceParamDTO.setDatabase("default"); + + MysqlConnectionParam connectionParams = (MysqlConnectionParam) mysqlDatasourceProcessor + .createConnectionParams(mysqlDatasourceParamDTO); + System.out.println(JSONUtils.toJsonString(connectionParams)); + Assert.assertEquals("jdbc:mysql://localhost:3306", connectionParams.getAddress()); + Assert.assertEquals("jdbc:mysql://localhost:3306/default", connectionParams.getJdbcUrl()); + } + + @Test + public void testCreateConnectionParams2() { + String connectionJson = "{\"user\":\"root\",\"password\":\"123456\",\"address\":\"jdbc:mysql://localhost:3306\"" + + ",\"database\":\"default\",\"jdbcUrl\":\"jdbc:mysql://localhost:3306/default\"}"; + MysqlConnectionParam connectionParams = (MysqlConnectionParam) mysqlDatasourceProcessor + .createConnectionParams(connectionJson); + Assert.assertNotNull(connectionJson); + Assert.assertEquals("root", connectionParams.getUser()); + } + + @Test + public void testGetDatasourceDriver() { + Assert.assertEquals(Constants.COM_MYSQL_JDBC_DRIVER, mysqlDatasourceProcessor.getDatasourceDriver()); + } + + @Test + public void testGetJdbcUrl() { + MysqlConnectionParam mysqlConnectionParam = new MysqlConnectionParam(); + mysqlConnectionParam.setJdbcUrl("jdbc:mysql://localhost:3306/default"); + Assert.assertEquals("jdbc:mysql://localhost:3306/default?allowLoadLocalInfile=false&autoDeserialize=false&allowLocalInfile=false&allowUrlInLocalInfile=false", + mysqlDatasourceProcessor.getJdbcUrl(mysqlConnectionParam)); + } + + @Test + public void testGetDbType() { + Assert.assertEquals(DbType.MYSQL, mysqlDatasourceProcessor.getDbType()); + } +} \ No newline at end of file diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/datasource/oracle/OracleDatasourceProcessorTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/datasource/oracle/OracleDatasourceProcessorTest.java new file mode 100644 index 0000000000..4f0fce2d9b --- /dev/null +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/datasource/oracle/OracleDatasourceProcessorTest.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.common.datasource.oracle; + +import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.enums.DbConnectType; +import org.apache.dolphinscheduler.common.enums.DbType; + +import java.sql.DriverManager; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; + +@RunWith(PowerMockRunner.class) +@PrepareForTest({Class.class, DriverManager.class}) +public class OracleDatasourceProcessorTest { + + private OracleDatasourceProcessor oracleDatasourceProcessor = new OracleDatasourceProcessor(); + + @Test + public void testCreateConnectionParams() { + OracleDatasourceParamDTO oracleDatasourceParamDTO = new OracleDatasourceParamDTO(); + oracleDatasourceParamDTO.setConnectType(DbConnectType.ORACLE_SID); + oracleDatasourceParamDTO.setHost("localhost"); + oracleDatasourceParamDTO.setPort(3308); + oracleDatasourceParamDTO.setUserName("root"); + oracleDatasourceParamDTO.setPassword("123456"); + oracleDatasourceParamDTO.setDatabase("default"); + + OracleConnectionParam connectionParams = (OracleConnectionParam) oracleDatasourceProcessor + .createConnectionParams(oracleDatasourceParamDTO); + Assert.assertNotNull(connectionParams); + Assert.assertEquals("jdbc:oracle:thin:@localhost:3308", connectionParams.getAddress()); + Assert.assertEquals("jdbc:oracle:thin:@localhost:3308/default", connectionParams.getJdbcUrl()); + } + + @Test + public void testCreateConnectionParams2() { + String connectionJson = "{\"user\":\"root\",\"password\":\"123456\",\"address\":\"jdbc:oracle:thin:@localhost:3308\"" + + ",\"database\":\"default\",\"jdbcUrl\":\"jdbc:oracle:thin:@localhost:3308/default\",\"connectType\":\"ORACLE_SID\"}"; + OracleConnectionParam connectionParams = (OracleConnectionParam) oracleDatasourceProcessor + .createConnectionParams(connectionJson); + Assert.assertNotNull(connectionParams); + Assert.assertEquals("root", connectionParams.getUser()); + } + + @Test + public void testGetDatasourceDriver() { + Assert.assertEquals(Constants.COM_ORACLE_JDBC_DRIVER, oracleDatasourceProcessor.getDatasourceDriver()); + } + + @Test + public void testGetJdbcUrl() { + OracleConnectionParam oracleConnectionParam = new OracleConnectionParam(); + oracleConnectionParam.setJdbcUrl("jdbc:oracle:thin:@localhost:3308/default"); + oracleConnectionParam.setOther("other=other"); + Assert.assertEquals("jdbc:oracle:thin:@localhost:3308/default?other=other", + oracleDatasourceProcessor.getJdbcUrl(oracleConnectionParam)); + } + + @Test + public void getDbType() { + Assert.assertEquals(DbType.ORACLE, oracleDatasourceProcessor.getDbType()); + } +} \ No newline at end of file diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/datasource/postgresql/PostgreSqlDatasourceProcessorTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/datasource/postgresql/PostgreSqlDatasourceProcessorTest.java new file mode 100644 index 0000000000..178c30ce7b --- /dev/null +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/datasource/postgresql/PostgreSqlDatasourceProcessorTest.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.common.datasource.postgresql; + +import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.enums.DbType; + +import java.sql.DriverManager; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; + +@RunWith(PowerMockRunner.class) +@PrepareForTest({Class.class, DriverManager.class}) +public class PostgreSqlDatasourceProcessorTest { + + private PostgreSqlDatasourceProcessor postgreSqlDatasourceProcessor = new PostgreSqlDatasourceProcessor(); + + @Test + public void testCreateConnectionParams() { + PostgreSqlDatasourceParamDTO postgreSqlDatasourceParamDTO = new PostgreSqlDatasourceParamDTO(); + postgreSqlDatasourceParamDTO.setUserName("root"); + postgreSqlDatasourceParamDTO.setPassword("123456"); + postgreSqlDatasourceParamDTO.setHost("localhost"); + postgreSqlDatasourceParamDTO.setPort(3308); + postgreSqlDatasourceParamDTO.setDatabase("default"); + + PostgreSqlConnectionParam connectionParams = (PostgreSqlConnectionParam) postgreSqlDatasourceProcessor + .createConnectionParams(postgreSqlDatasourceParamDTO); + Assert.assertEquals("jdbc:postgresql://localhost:3308", connectionParams.getAddress()); + Assert.assertEquals("jdbc:postgresql://localhost:3308/default", connectionParams.getJdbcUrl()); + Assert.assertEquals("root", connectionParams.getUser()); + } + + @Test + public void testCreateConnectionParams2() { + String connectionJson = "{\"user\":\"root\",\"password\":\"123456\",\"address\":\"jdbc:postgresql://localhost:3308\"" + + ",\"database\":\"default\",\"jdbcUrl\":\"jdbc:postgresql://localhost:3308/default\"}"; + PostgreSqlConnectionParam connectionParams = (PostgreSqlConnectionParam) postgreSqlDatasourceProcessor + .createConnectionParams(connectionJson); + Assert.assertNotNull(connectionParams); + Assert.assertEquals("root", connectionParams.getUser()); + } + + @Test + public void testGetDatasourceDriver() { + Assert.assertEquals(Constants.ORG_POSTGRESQL_DRIVER, postgreSqlDatasourceProcessor.getDatasourceDriver()); + } + + @Test + public void testGetJdbcUrl() { + PostgreSqlConnectionParam postgreSqlConnectionParam = new PostgreSqlConnectionParam(); + postgreSqlConnectionParam.setJdbcUrl("jdbc:postgresql://localhost:3308/default"); + postgreSqlConnectionParam.setOther("other"); + + String jdbcUrl = postgreSqlDatasourceProcessor.getJdbcUrl(postgreSqlConnectionParam); + Assert.assertEquals("jdbc:postgresql://localhost:3308/default?other", jdbcUrl); + + } + + @Test + public void testGetDbType() { + Assert.assertEquals(DbType.POSTGRESQL, postgreSqlDatasourceProcessor.getDbType()); + } +} \ No newline at end of file diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/datasource/presto/PrestoDatasourceProcessorTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/datasource/presto/PrestoDatasourceProcessorTest.java new file mode 100644 index 0000000000..a2bfe997bd --- /dev/null +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/datasource/presto/PrestoDatasourceProcessorTest.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.common.datasource.presto; + +import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.enums.DbType; + +import java.sql.DriverManager; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; + +@RunWith(PowerMockRunner.class) +@PrepareForTest({Class.class, DriverManager.class}) +public class PrestoDatasourceProcessorTest { + + private PrestoDatasourceProcessor prestoDatasourceProcessor = new PrestoDatasourceProcessor(); + + @Test + public void testCreateConnectionParams() { + PrestoDatasourceParamDTO prestoDatasourceParamDTO = new PrestoDatasourceParamDTO(); + prestoDatasourceParamDTO.setHost("localhost"); + prestoDatasourceParamDTO.setPort(1234); + prestoDatasourceParamDTO.setDatabase("default"); + prestoDatasourceParamDTO.setUserName("root"); + prestoDatasourceParamDTO.setPassword("123456"); + + PrestoConnectionParam connectionParams = (PrestoConnectionParam) prestoDatasourceProcessor + .createConnectionParams(prestoDatasourceParamDTO); + Assert.assertEquals("jdbc:presto://localhost:1234", connectionParams.getAddress()); + Assert.assertEquals("jdbc:presto://localhost:1234/default", connectionParams.getJdbcUrl()); + } + + @Test + public void testCreateConnectionParams2() { + String connectionJson = "{\"user\":\"root\",\"password\":\"123456\",\"address\":\"jdbc:presto://localhost:1234\"" + + ",\"database\":\"default\",\"jdbcUrl\":\"jdbc:presto://localhost:1234/default\"}"; + PrestoConnectionParam connectionParams = (PrestoConnectionParam) prestoDatasourceProcessor + .createConnectionParams(connectionJson); + Assert.assertNotNull(connectionParams); + Assert.assertEquals("root", connectionParams.getUser()); + } + + @Test + public void testGetDatasourceDriver() { + Assert.assertEquals(Constants.COM_PRESTO_JDBC_DRIVER, prestoDatasourceProcessor.getDatasourceDriver()); + } + + @Test + public void testGetJdbcUrl() { + PrestoConnectionParam prestoConnectionParam = new PrestoConnectionParam(); + prestoConnectionParam.setJdbcUrl("jdbc:postgresql://localhost:1234/default"); + prestoConnectionParam.setOther("other"); + Assert.assertEquals("jdbc:postgresql://localhost:1234/default?other", + prestoDatasourceProcessor.getJdbcUrl(prestoConnectionParam)); + + } + + @Test + public void testGetDbType() { + Assert.assertEquals(DbType.PRESTO, prestoDatasourceProcessor.getDbType()); + } +} \ No newline at end of file diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/datasource/spark/SparkDatasourceProcessorTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/datasource/spark/SparkDatasourceProcessorTest.java new file mode 100644 index 0000000000..706c0aef0f --- /dev/null +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/datasource/spark/SparkDatasourceProcessorTest.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.common.datasource.spark; + +import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.enums.DbType; + +import java.sql.DriverManager; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; + +@RunWith(PowerMockRunner.class) +@PrepareForTest({Class.class, DriverManager.class}) +public class SparkDatasourceProcessorTest { + + private SparkDatasourceProcessor sparkDatasourceProcessor = new SparkDatasourceProcessor(); + + @Test + public void testCreateConnectionParams() { + SparkDatasourceParamDTO sparkDatasourceParamDTO = new SparkDatasourceParamDTO(); + sparkDatasourceParamDTO.setUserName("root"); + sparkDatasourceParamDTO.setPassword("12345"); + sparkDatasourceParamDTO.setHost("localhost1,localhost2"); + sparkDatasourceParamDTO.setPort(1234); + sparkDatasourceParamDTO.setDatabase("default"); + + SparkConnectionParam connectionParams = (SparkConnectionParam) sparkDatasourceProcessor + .createConnectionParams(sparkDatasourceParamDTO); + Assert.assertEquals("jdbc:hive2://localhost1:1234,localhost2:1234", connectionParams.getAddress()); + Assert.assertEquals("jdbc:hive2://localhost1:1234,localhost2:1234/default", connectionParams.getJdbcUrl()); + } + + @Test + public void testCreateConnectionParams2() { + String connectionJson = "{\"user\":\"root\",\"password\":\"12345\",\"address\":\"jdbc:hive2://localhost1:1234,localhost2:1234\"" + + ",\"database\":\"default\",\"jdbcUrl\":\"jdbc:hive2://localhost1:1234,localhost2:1234/default\"}"; + SparkConnectionParam connectionParams = (SparkConnectionParam) sparkDatasourceProcessor + .createConnectionParams(connectionJson); + Assert.assertNotNull(connectionParams); + Assert.assertEquals("root", connectionParams.getUser()); + } + + @Test + public void testGetDatasourceDriver() { + Assert.assertEquals(Constants.ORG_APACHE_HIVE_JDBC_HIVE_DRIVER, sparkDatasourceProcessor.getDatasourceDriver()); + } + + @Test + public void testGetJdbcUrl() { + SparkConnectionParam sparkConnectionParam = new SparkConnectionParam(); + sparkConnectionParam.setJdbcUrl("jdbc:hive2://localhost1:1234,localhost2:1234/default"); + sparkConnectionParam.setOther("other"); + Assert.assertEquals("jdbc:hive2://localhost1:1234,localhost2:1234/default;other", + sparkDatasourceProcessor.getJdbcUrl(sparkConnectionParam)); + } + + @Test + public void testGetDbType() { + Assert.assertEquals(DbType.SPARK, sparkDatasourceProcessor.getDbType()); + } +} \ No newline at end of file diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/datasource/sqlserver/SqlServerDatasourceProcessorTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/datasource/sqlserver/SqlServerDatasourceProcessorTest.java new file mode 100644 index 0000000000..39a23f6e25 --- /dev/null +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/datasource/sqlserver/SqlServerDatasourceProcessorTest.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.common.datasource.sqlserver; + +import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.enums.DbType; +import org.apache.dolphinscheduler.common.utils.JSONUtils; + +import java.sql.DriverManager; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; + +@RunWith(PowerMockRunner.class) +@PrepareForTest({Class.class, DriverManager.class}) +public class SqlServerDatasourceProcessorTest { + + private SqlServerDatasourceProcessor sqlServerDatasourceProcessor = new SqlServerDatasourceProcessor(); + + @Test + public void testCreateConnectionParams() { + SqlServerDatasourceParamDTO sqlServerDatasourceParamDTO = new SqlServerDatasourceParamDTO(); + sqlServerDatasourceParamDTO.setUserName("root"); + sqlServerDatasourceParamDTO.setPassword("123456"); + sqlServerDatasourceParamDTO.setDatabase("default"); + sqlServerDatasourceParamDTO.setHost("localhost"); + sqlServerDatasourceParamDTO.setPort(1234); + + SqlServerConnectionParam connectionParams = (SqlServerConnectionParam) sqlServerDatasourceProcessor + .createConnectionParams(sqlServerDatasourceParamDTO); + Assert.assertEquals("jdbc:sqlserver://localhost:1234", connectionParams.getAddress()); + Assert.assertEquals("jdbc:sqlserver://localhost:1234;databaseName=default", connectionParams.getJdbcUrl()); + Assert.assertEquals("root", connectionParams.getUser()); + } + + @Test + public void testCreateConnectionParams2() { + String connectionJson = "{\"user\":\"root\",\"password\":\"123456\",\"address\":\"jdbc:sqlserver://localhost:1234\"" + + ",\"database\":\"default\",\"jdbcUrl\":\"jdbc:sqlserver://localhost:1234;databaseName=default\"}"; + SqlServerConnectionParam sqlServerConnectionParam = JSONUtils.parseObject(connectionJson, SqlServerConnectionParam.class); + Assert.assertNotNull(sqlServerConnectionParam); + Assert.assertEquals("root", sqlServerConnectionParam.getUser()); + } + + @Test + public void testGetDatasourceDriver() { + Assert.assertEquals(Constants.COM_SQLSERVER_JDBC_DRIVER, sqlServerDatasourceProcessor.getDatasourceDriver()); + } + + @Test + public void testGetJdbcUrl() { + SqlServerConnectionParam sqlServerConnectionParam = new SqlServerConnectionParam(); + sqlServerConnectionParam.setJdbcUrl("jdbc:sqlserver://localhost:1234;databaseName=default"); + sqlServerConnectionParam.setOther("other"); + Assert.assertEquals("jdbc:sqlserver://localhost:1234;databaseName=default;other", + sqlServerDatasourceProcessor.getJdbcUrl(sqlServerConnectionParam)); + } + + @Test + public void testGetDbType() { + Assert.assertEquals(DbType.SQLSERVER, sqlServerDatasourceProcessor.getDbType()); + } +} \ No newline at end of file diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/os/OSUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/os/OSUtilsTest.java index b2fd887d3d..d6f3ffd024 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/os/OSUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/os/OSUtilsTest.java @@ -18,13 +18,11 @@ package org.apache.dolphinscheduler.common.os; import org.apache.dolphinscheduler.common.utils.OSUtils; + +import org.junit.Assert; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import oshi.hardware.GlobalMemory; - -import java.math.RoundingMode; -import java.text.DecimalFormat; /** @@ -34,42 +32,34 @@ public class OSUtilsTest { private static Logger logger = LoggerFactory.getLogger(OSUtilsTest.class); - @Test public void memoryUsage() { - logger.info("memoryUsage : {}", OSUtils.memoryUsage());// 0.3361799418926239 + double memoryUsage = OSUtils.memoryUsage(); + logger.info("memoryUsage : {}", memoryUsage); + Assert.assertTrue(memoryUsage >= 0.0); } @Test - public void availablePhysicalMemorySize() { - logger.info("availablePhysicalMemorySize : {}", OSUtils.availablePhysicalMemorySize()); - logger.info("availablePhysicalMemorySize : {}", OSUtils.totalMemorySize() / 10); + public void physicalMemorySize() { + double availablePhysicalMemorySize = OSUtils.availablePhysicalMemorySize(); + double totalPhysicalMemorySize = OSUtils.totalPhysicalMemorySize(); + logger.info("availablePhysicalMemorySize : {}", availablePhysicalMemorySize); + logger.info("totalPhysicalMemorySize : {}", totalPhysicalMemorySize); + Assert.assertTrue(availablePhysicalMemorySize >= 0.0); + Assert.assertTrue(totalPhysicalMemorySize >= 0.0); } - @Test public void loadAverage() { - logger.info("memoryUsage : {}", OSUtils.loadAverage()); + double loadAverage = OSUtils.loadAverage(); + logger.info("loadAverage : {}", loadAverage); + Assert.assertTrue(loadAverage >= 0.0); } - - private void printMemory(GlobalMemory memory) { - logger.info("memoryUsage : {} %" , (memory.getTotal() - memory.getAvailable()) * 100 / memory.getTotal() ); - } - - @Test - public void cpuUsage() throws Exception { - logger.info("cpuUsage : {}", OSUtils.cpuUsage()); - Thread.sleep(1000L); - logger.info("cpuUsage : {}", OSUtils.cpuUsage()); - + public void cpuUsage() { double cpuUsage = OSUtils.cpuUsage(); - - DecimalFormat df = new DecimalFormat("0.00"); - - df.setRoundingMode(RoundingMode.HALF_UP); - - logger.info("cpuUsage1 : {}", df.format(cpuUsage)); + logger.info("cpuUsage : {}", cpuUsage); + Assert.assertTrue(cpuUsage >= 0.0); } } diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/IpUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/IpUtilsTest.java deleted file mode 100644 index ec6ffa35a7..0000000000 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/IpUtilsTest.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.dolphinscheduler.common.utils; - -import org.junit.Assert; -import org.junit.Test; - -public class IpUtilsTest { - - @Test - public void ipToLong() { - - String ip = "192.168.110.1"; - String ip2 = "0.0.0.0"; - long longNumber = IpUtils.ipToLong(ip); - long longNumber2 = IpUtils.ipToLong(ip2); - System.out.println(longNumber); - Assert.assertEquals(3232263681L, longNumber); - Assert.assertEquals(0L, longNumber2); - - String ip3 = "255.255.255.255"; - long longNumber3 = IpUtils.ipToLong(ip3); - System.out.println(longNumber3); - Assert.assertEquals(4294967295L, longNumber3); - - } - - @Test - public void longToIp() { - - String ip = "192.168.110.1"; - String ip2 = "0.0.0.0"; - long longNum = 3232263681L; - String i1 = IpUtils.longToIp(longNum); - - String i2 = IpUtils.longToIp(0); - - Assert.assertEquals(ip, i1); - Assert.assertEquals(ip2, i2); - } -} \ No newline at end of file diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/NetUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/NetUtilsTest.java index 1202b58e20..f912933d84 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/NetUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/NetUtilsTest.java @@ -50,7 +50,7 @@ public class NetUtilsTest { when(address.getHostAddress()).thenReturn("172.17.0.15"); assertEquals("172.17.0.15", NetUtils.getHost(address)); CommonTest.setFinalStatic(Constants.class.getDeclaredField("KUBERNETES_MODE"), true); - assertEquals("dolphinscheduler-worker-0.dolphinscheduler-worker-headless.default.svc.cluster.local", NetUtils.getHost(address)); + assertEquals("dolphinscheduler-worker-0.dolphinscheduler-worker-headless", NetUtils.getHost(address)); address = mock(InetAddress.class); when(address.getCanonicalHostName()).thenReturn("dolphinscheduler-worker-0"); when(address.getHostName()).thenReturn("dolphinscheduler-worker-0"); diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/OSUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/OSUtilsTest.java index 86da2684f7..8467cec9b4 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/OSUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/OSUtilsTest.java @@ -16,11 +16,6 @@ */ package org.apache.dolphinscheduler.common.utils; -import org.apache.dolphinscheduler.common.Constants; - -import org.apache.commons.configuration.Configuration; -import org.apache.commons.configuration.PropertiesConfiguration; - import java.io.IOException; import java.util.List; @@ -40,12 +35,12 @@ public class OSUtilsTest { } @Test - public void testOSMetric(){ + public void testOSMetric() { if (!OSUtils.isWindows()) { double availablePhysicalMemorySize = OSUtils.availablePhysicalMemorySize(); Assert.assertTrue(availablePhysicalMemorySize >= 0.0d); - double totalMemorySize = OSUtils.totalMemorySize(); - Assert.assertTrue(totalMemorySize >= 0.0d); + double totalPhysicalMemorySize = OSUtils.totalPhysicalMemorySize(); + Assert.assertTrue(totalPhysicalMemorySize >= 0.0d); double loadAverage = OSUtils.loadAverage(); logger.info("loadAverage {}", loadAverage); double memoryUsage = OSUtils.memoryUsage(); @@ -92,7 +87,7 @@ public class OSUtilsTest { @Test public void exeCmd() { - if(OSUtils.isMacOS() || !OSUtils.isWindows()){ + if (OSUtils.isMacOS() || !OSUtils.isWindows()) { try { String result = OSUtils.exeCmd("echo helloWorld"); Assert.assertEquals("helloWorld\n",result); @@ -102,39 +97,16 @@ public class OSUtilsTest { } } @Test - public void getProcessID(){ + public void getProcessID() { int processId = OSUtils.getProcessID(); Assert.assertNotEquals(0, processId); } @Test - public void checkResource(){ + public void checkResource() { boolean resource = OSUtils.checkResource(100,0); Assert.assertTrue(resource); resource = OSUtils.checkResource(0,Double.MAX_VALUE); Assert.assertFalse(resource); - - Configuration configuration = new PropertiesConfiguration(); - - configuration.setProperty(Constants.MASTER_MAX_CPULOAD_AVG,100); - configuration.setProperty(Constants.MASTER_RESERVED_MEMORY,0); - resource = OSUtils.checkResource(configuration,true); - Assert.assertTrue(resource); - - configuration.setProperty(Constants.MASTER_MAX_CPULOAD_AVG,0); - configuration.setProperty(Constants.MASTER_RESERVED_MEMORY,Double.MAX_VALUE); - resource = OSUtils.checkResource(configuration,true); - Assert.assertFalse(resource); - - configuration.setProperty(Constants.WORKER_MAX_CPULOAD_AVG,100); - configuration.setProperty(Constants.WORKER_RESERVED_MEMORY,0); - resource = OSUtils.checkResource(configuration,false); - Assert.assertTrue(resource); - - configuration.setProperty(Constants.WORKER_MAX_CPULOAD_AVG,0); - configuration.setProperty(Constants.WORKER_RESERVED_MEMORY,Double.MAX_VALUE); - resource = OSUtils.checkResource(configuration,false); - Assert.assertFalse(resource); - } } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/BaseDataSource.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/BaseDataSource.java deleted file mode 100644 index e26a3751fb..0000000000 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/BaseDataSource.java +++ /dev/null @@ -1,278 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.dao.datasource; - -import static org.apache.dolphinscheduler.common.Constants.PASSWORD; -import static org.apache.dolphinscheduler.common.Constants.USER; - -import org.apache.dolphinscheduler.common.enums.DbType; -import org.apache.dolphinscheduler.common.utils.CommonUtils; -import org.apache.dolphinscheduler.common.utils.StringUtils; - -import java.sql.Connection; -import java.sql.DriverManager; -import java.util.Properties; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * data source base class - */ -public abstract class BaseDataSource { - - private static final Logger logger = LoggerFactory.getLogger(BaseDataSource.class); - - /** - * user name - */ - protected String user; - - /** - * user password - */ - protected String password; - - /** - * data source address - */ - private String address; - - /** - * database name - */ - private String database; - - /** - * other connection parameters for the data source - */ - private String other; - - /** - * principal - */ - private String principal; - - /** - * java.security.krb5.conf - */ - private String javaSecurityKrb5Conf; - - /** - * login.user.keytab.username - */ - private String loginUserKeytabUsername; - - /** - * login.user.keytab.path - */ - private String loginUserKeytabPath; - - public String getPrincipal() { - return principal; - } - - public void setPrincipal(String principal) { - this.principal = principal; - } - - /** - * @return driver class - */ - public abstract String driverClassSelector(); - - /** - * @return db type - */ - public abstract DbType dbTypeSelector(); - - /** - * gets the JDBC url for the data source connection - * @return getJdbcUrl - */ - public String getJdbcUrl() { - StringBuilder jdbcUrl = new StringBuilder(getAddress()); - - appendDatabase(jdbcUrl); - appendPrincipal(jdbcUrl); - appendOther(jdbcUrl); - - return jdbcUrl.toString(); - } - - /** - * append database - * @param jdbcUrl jdbc url - */ - protected void appendDatabase(StringBuilder jdbcUrl) { - if (dbTypeSelector() == DbType.SQLSERVER) { - jdbcUrl.append(";databaseName=").append(getDatabase()); - } else { - if (getAddress().lastIndexOf('/') != (jdbcUrl.length() - 1)) { - jdbcUrl.append("/"); - } - jdbcUrl.append(getDatabase()); - } - } - - /** - * append principal - * @param jdbcUrl jdbc url - */ - private void appendPrincipal(StringBuilder jdbcUrl) { - boolean tag = dbTypeSelector() == DbType.HIVE || dbTypeSelector() == DbType.SPARK; - if (tag && StringUtils.isNotEmpty(getPrincipal())) { - jdbcUrl.append(";principal=").append(getPrincipal()); - } - } - - /** - * append other - * @param jdbcUrl jdbc url - */ - private void appendOther(StringBuilder jdbcUrl) { - String otherParams = filterOther(getOther()); - if (StringUtils.isNotEmpty(otherParams)) { - String separator = ""; - switch (dbTypeSelector()) { - case CLICKHOUSE: - case MYSQL: - case ORACLE: - case POSTGRESQL: - case PRESTO: - separator = "?"; - break; - case DB2: - separator = ":"; - break; - case HIVE: - if ("?".equals(otherParams.substring(0, 1))) { - break; - } - separator = ";"; - break; - case SPARK: - case SQLSERVER: - separator = ";"; - break; - default: - logger.error("Db type mismatch!"); - } - jdbcUrl.append(separator).append(otherParams); - } - } - - /** - * the data source test connection - * @return Connection Connection - * @throws Exception Exception - */ - public Connection getConnection() throws Exception { - Class.forName(driverClassSelector()); - return DriverManager.getConnection(getJdbcUrl(), getUser(), getPassword()); - } - - /** - * the data source test connection - * @param info Properties - * @return Connection Connection - * @throws Exception Exception - */ - public Connection getConnection(Properties info) throws Exception { - Class.forName(driverClassSelector()); - info.setProperty(USER, getUser()); - info.setProperty(PASSWORD, getPassword()); - return DriverManager.getConnection(getJdbcUrl(), info); - } - - protected String filterOther(String otherParams) { - return otherParams; - } - - public String getUser() { - return user; - } - - public void setUser(String user) { - this.user = user; - } - - /** - * password need decode - * @return - */ - public String getPassword() { - return CommonUtils.decodePassword(password); - } - - public void setPassword(String password) { - this.password = password; - } - - public void setAddress(String address) { - this.address = address; - } - - public String getAddress() { - return address; - } - - public String getDatabase() { - return database; - } - - public void setDatabase(String database) { - this.database = database; - } - - public String getOther() { - return other; - } - - public void setOther(String other) { - this.other = other; - } - - public void setConnParams(String connParams) { - - } - - public String getJavaSecurityKrb5Conf() { - return javaSecurityKrb5Conf; - } - - public void setJavaSecurityKrb5Conf(String javaSecurityKrb5Conf) { - this.javaSecurityKrb5Conf = javaSecurityKrb5Conf; - } - - public String getLoginUserKeytabUsername() { - return loginUserKeytabUsername; - } - - public void setLoginUserKeytabUsername(String loginUserKeytabUsername) { - this.loginUserKeytabUsername = loginUserKeytabUsername; - } - - public String getLoginUserKeytabPath() { - return loginUserKeytabPath; - } - - public void setLoginUserKeytabPath(String loginUserKeytabPath) { - this.loginUserKeytabPath = loginUserKeytabPath; - } -} diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/DataSourceFactory.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/DataSourceFactory.java deleted file mode 100644 index f5d07ea693..0000000000 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/DataSourceFactory.java +++ /dev/null @@ -1,108 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.dolphinscheduler.dao.datasource; - -import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.common.enums.DbType; -import org.apache.dolphinscheduler.common.utils.*; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * produce datasource in this custom defined datasource factory. - */ -public class DataSourceFactory { - - private static final Logger logger = LoggerFactory.getLogger(DataSourceFactory.class); - - /** - * getDatasource - * @param dbType dbType - * @param parameter parameter - * @return getDatasource - */ - public static BaseDataSource getDatasource(DbType dbType, String parameter) { - try { - switch (dbType) { - case MYSQL: - return JSONUtils.parseObject(parameter, MySQLDataSource.class); - case POSTGRESQL: - return JSONUtils.parseObject(parameter, PostgreDataSource.class); - case HIVE: - return JSONUtils.parseObject(parameter, HiveDataSource.class); - case SPARK: - return JSONUtils.parseObject(parameter, SparkDataSource.class); - case CLICKHOUSE: - return JSONUtils.parseObject(parameter, ClickHouseDataSource.class); - case ORACLE: - return JSONUtils.parseObject(parameter, OracleDataSource.class); - case SQLSERVER: - return JSONUtils.parseObject(parameter, SQLServerDataSource.class); - case DB2: - return JSONUtils.parseObject(parameter, DB2ServerDataSource.class); - case PRESTO: - return JSONUtils.parseObject(parameter, PrestoDataSource.class); - default: - return null; - } - } catch (Exception e) { - logger.error("get datasource object error", e); - return null; - } - } - - /** - * load class - * @param dbType - * @throws Exception - */ - public static void loadClass(DbType dbType) throws Exception{ - switch (dbType){ - case MYSQL : - Class.forName(Constants.COM_MYSQL_JDBC_DRIVER); - break; - case POSTGRESQL : - Class.forName(Constants.ORG_POSTGRESQL_DRIVER); - break; - case HIVE : - Class.forName(Constants.ORG_APACHE_HIVE_JDBC_HIVE_DRIVER); - break; - case SPARK : - Class.forName(Constants.ORG_APACHE_HIVE_JDBC_HIVE_DRIVER); - break; - case CLICKHOUSE : - Class.forName(Constants.COM_CLICKHOUSE_JDBC_DRIVER); - break; - case ORACLE : - Class.forName(Constants.COM_ORACLE_JDBC_DRIVER); - break; - case SQLSERVER: - Class.forName(Constants.COM_SQLSERVER_JDBC_DRIVER); - break; - case DB2: - Class.forName(Constants.COM_DB2_JDBC_DRIVER); - break; - case PRESTO: - Class.forName(Constants.COM_PRESTO_JDBC_DRIVER); - break; - default: - logger.error("not support sql type: {},can't load class", dbType); - throw new IllegalArgumentException("not support sql type,can't load class"); - - } - } -} diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/HiveDataSource.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/HiveDataSource.java deleted file mode 100644 index 8fccb08bb7..0000000000 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/HiveDataSource.java +++ /dev/null @@ -1,120 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.dao.datasource; - -import static org.apache.dolphinscheduler.common.Constants.SEMICOLON; - -import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.common.enums.DbType; -import org.apache.dolphinscheduler.common.utils.CollectionUtils; -import org.apache.dolphinscheduler.common.utils.CommonUtils; -import org.apache.dolphinscheduler.common.utils.HiveConfUtils; -import org.apache.dolphinscheduler.common.utils.StringUtils; - -import java.sql.Connection; -import java.util.Map; - -/** - * data source of hive - */ -public class HiveDataSource extends BaseDataSource { - - /** - * gets the JDBC url for the data source connection - * @return jdbc url - */ - @Override - public String driverClassSelector() { - return Constants.ORG_APACHE_HIVE_JDBC_HIVE_DRIVER; - } - - /** - * @return db type - */ - @Override - public DbType dbTypeSelector() { - return DbType.HIVE; - } - - /** - * build hive jdbc params,append : ?hive_conf_list - * - * hive jdbc url template: - * - * jdbc:hive2://:,:/dbName;initFile=;sess_var_list?hive_conf_list#hive_var_list - * - * @param otherParams otherParams - * @return filter otherParams - */ - @Override - protected String filterOther(String otherParams) { - if (StringUtils.isBlank(otherParams)) { - return ""; - } - - StringBuilder hiveConfListSb = new StringBuilder(); - hiveConfListSb.append("?"); - StringBuilder sessionVarListSb = new StringBuilder(); - - String[] otherArray = otherParams.split(";", -1); - - for (String conf : otherArray) { - if (HiveConfUtils.isHiveConfVar(conf)) { - hiveConfListSb.append(conf).append(";"); - } else { - sessionVarListSb.append(conf).append(";"); - } - } - - // remove the last ";" - if (sessionVarListSb.length() > 0) { - sessionVarListSb.deleteCharAt(sessionVarListSb.length() - 1); - } - - if (hiveConfListSb.length() > 0) { - hiveConfListSb.deleteCharAt(hiveConfListSb.length() - 1); - } - - return sessionVarListSb.toString() + hiveConfListSb.toString(); - } - - /** - * the data source test connection - * @return Connection Connection - * @throws Exception Exception - */ - @Override - public Connection getConnection() throws Exception { - CommonUtils.loadKerberosConf(getJavaSecurityKrb5Conf(), getLoginUserKeytabUsername(), getLoginUserKeytabPath()); - return super.getConnection(); - } - - @Override - public void setConnParams(String connParams) { - // Verification parameters - Map connParamMap = CollectionUtils.stringToMap(connParams, SEMICOLON); - if (connParamMap.isEmpty()) { - return; - } - - StringBuilder otherSb = new StringBuilder(); - connParamMap.forEach((k, v) -> otherSb.append(String.format("%s=%s%s", k, v, SEMICOLON))); - StringBuilder otherAppend = StringUtils.isNotBlank(getOther()) ? otherSb.append(getOther()) : otherSb.deleteCharAt(otherSb.length() - 1); - super.setOther(otherAppend.toString()); - } -} diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/MySQLDataSource.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/MySQLDataSource.java deleted file mode 100644 index d3e3927a97..0000000000 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/MySQLDataSource.java +++ /dev/null @@ -1,120 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.dao.datasource; - -import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.common.enums.DbType; -import org.apache.dolphinscheduler.common.utils.JSONUtils; -import org.apache.dolphinscheduler.common.utils.StringUtils; - -import org.apache.commons.collections4.MapUtils; - -import java.util.HashMap; -import java.util.Map; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * data source of mySQL - */ -public class MySQLDataSource extends BaseDataSource { - - private static final Logger logger = LoggerFactory.getLogger(MySQLDataSource.class); - - private static final String ALLOW_LOAD_LOCAL_IN_FILE_NAME = "allowLoadLocalInfile"; - - private static final String AUTO_DESERIALIZE = "autoDeserialize"; - - private static final String ALLOW_LOCAL_IN_FILE_NAME = "allowLocalInfile"; - - private static final String ALLOW_URL_IN_LOCAL_IN_FILE_NAME = "allowUrlInLocalInfile"; - - private static final String APPEND_PARAMS = "allowLoadLocalInfile=false&autoDeserialize=false&allowLocalInfile=false&allowUrlInLocalInfile=false"; - - private static boolean checkKeyIsLegitimate(String key) { - return !key.contains(ALLOW_LOAD_LOCAL_IN_FILE_NAME) && !key.contains(AUTO_DESERIALIZE) && !key.contains(ALLOW_LOCAL_IN_FILE_NAME) && !key.contains(ALLOW_URL_IN_LOCAL_IN_FILE_NAME); - } - - /** - * gets the JDBC url for the data source connection - * - * @return jdbc url - */ - @Override - public String driverClassSelector() { - return Constants.COM_MYSQL_JDBC_DRIVER; - } - - /** - * @return db type - */ - @Override - public DbType dbTypeSelector() { - return DbType.MYSQL; - } - - public static Map buildOtherParams(String other) { - if (StringUtils.isBlank(other)) { - return null; - } - Map paramMap = JSONUtils.toMap(other); - if (MapUtils.isEmpty(paramMap)) { - return null; - } - Map newParamMap = new HashMap<>(); - paramMap.forEach((k, v) -> { - if (!checkKeyIsLegitimate(k)) { - return; - } - newParamMap.put(k, v); - - }); - return newParamMap; - } - - @Override - public String getUser() { - if (user.contains(AUTO_DESERIALIZE)) { - logger.warn("sensitive param : {} in username field is filtered", AUTO_DESERIALIZE); - user = user.replace(AUTO_DESERIALIZE, ""); - } - logger.debug("username : {}", user); - return user; - } - - @Override - protected String filterOther(String otherParams) { - if (StringUtils.isBlank(otherParams)) { - return APPEND_PARAMS; - } - char symbol = '&'; - return otherParams + symbol + APPEND_PARAMS; - } - - @Override - public String getPassword() { - // password need decode - password = super.getPassword(); - if (password.contains(AUTO_DESERIALIZE)) { - logger.warn("sensitive param : {} in password field is filtered", AUTO_DESERIALIZE); - password = password.replace(AUTO_DESERIALIZE, ""); - } - return password; - } -} diff --git a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/CommandMapper.xml b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/CommandMapper.xml index abbc9a17a1..c0728f2e43 100644 --- a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/CommandMapper.xml +++ b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/CommandMapper.xml @@ -20,7 +20,7 @@