natas_hw
/
dolphinscheduler
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Move datasource params to datasouce plugin module (#10353) 

* Move datasource params to datasouce plugin module
This commit is contained in:
Wenjun Ruan 2022-06-03 12:15:53 +08:00 committed by GitHub
parent 12fdd9bbd9
commit 0bf4043d5f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
64 changed files with 242 additions and 181 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/DataSourceControllerTest.java
View File

@ -25,7 +25,7 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.
import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.mysql.MySQLDataSourceParamDTO;
import org.apache.dolphinscheduler.plugin.datasource.mysql.param.MySQLDataSourceParamDTO;
import java.util.HashMap;

14
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataSourceServiceTest.java
View File

@ -17,6 +17,9 @@
package org.apache.dolphinscheduler.api.service;
import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.DATASOURCE_DELETE;
import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.DATASOURCE_LIST;
import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.service.impl.BaseServiceImpl;
import org.apache.dolphinscheduler.api.service.impl.DataSourceServiceImpl;
@ -29,14 +32,14 @@ import org.apache.dolphinscheduler.dao.entity.DataSource;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.mapper.DataSourceMapper;
import org.apache.dolphinscheduler.dao.mapper.DataSourceUserMapper;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.hive.HiveDataSourceParamDTO;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.mysql.MySQLDataSourceParamDTO;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.oracle.OracleDataSourceParamDTO;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.postgresql.PostgreSQLDataSourceParamDTO;
import org.apache.dolphinscheduler.plugin.datasource.api.plugin.DataSourceClientProvider;
import org.apache.dolphinscheduler.plugin.datasource.api.utils.CommonUtils;
import org.apache.dolphinscheduler.plugin.datasource.api.utils.DataSourceUtils;
import org.apache.dolphinscheduler.plugin.datasource.api.utils.PasswordUtils;
import org.apache.dolphinscheduler.plugin.datasource.hive.param.HiveDataSourceParamDTO;
import org.apache.dolphinscheduler.plugin.datasource.mysql.param.MySQLDataSourceParamDTO;
import org.apache.dolphinscheduler.plugin.datasource.oracle.param.OracleDataSourceParamDTO;
import org.apache.dolphinscheduler.plugin.datasource.postgresql.param.PostgreSQLDataSourceParamDTO;
import org.apache.dolphinscheduler.service.permission.ResourcePermissionCheckService;
import org.apache.dolphinscheduler.spi.datasource.ConnectionParam;
import org.apache.dolphinscheduler.spi.enums.DbConnectType;
@ -67,9 +70,6 @@ import org.powermock.modules.junit4.PowerMockRunner;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.DATASOURCE_DELETE;
import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.DATASOURCE_LIST;
/**
* data source service test
*/

5
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/pom.xml
View File

@ -44,7 +44,6 @@
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
@ -66,7 +65,6 @@
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-collections4</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
@ -130,7 +128,6 @@
<groupId>log4j</groupId>
</exclusion>
</exclusions>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
@ -139,8 +136,6 @@
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.zaxxer</groupId>
<artifactId>HikariCP</artifactId>

23
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/BaseDataSourceParamDTO.java
View File

@ -17,34 +17,13 @@
package org.apache.dolphinscheduler.plugin.datasource.api.datasource;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.clickhouse.ClickHouseDataSourceParamDTO;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.db2.Db2DataSourceParamDTO;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.hive.HiveDataSourceParamDTO;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.mysql.MySQLDataSourceParamDTO;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.oracle.OracleDataSourceParamDTO;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.postgresql.PostgreSQLDataSourceParamDTO;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.presto.PrestoDataSourceParamDTO;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.redshift.RedshiftDataSourceParamDTO;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.spark.SparkDataSourceParamDTO;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.sqlserver.SQLServerDataSourceParamDTO;
import org.apache.dolphinscheduler.spi.enums.DbType;
import java.io.Serializable;
import java.util.Map;
/**
* Basic datasource params submitted to api.
* <p>
* see {@link MySQLDataSourceParamDTO}
* see {@link PostgreSQLDataSourceParamDTO}
* see {@link HiveDataSourceParamDTO}
* see {@link SparkDataSourceParamDTO}
* see {@link ClickHouseDataSourceParamDTO}
* see {@link OracleDataSourceParamDTO}
* see {@link SQLServerDataSourceParamDTO}
* see {@link Db2DataSourceParamDTO}
* see {@link PrestoDataSourceParamDTO}
* see {@link RedshiftDataSourceParamDTO}
* Basic datasource params submitted to api, each datasource plugin should have implementation.
*/
public abstract class BaseDataSourceParamDTO implements Serializable {

5
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/plugin/DataSourcePluginManager.java
View File

@ -23,10 +23,8 @@ import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel;
import org.apache.dolphinscheduler.spi.datasource.DataSourceChannelFactory;
import java.util.Collections;
import java.util.HashSet;
import java.util.Map;
import java.util.ServiceLoader;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import org.slf4j.Logger;
@ -42,14 +40,13 @@ public class DataSourcePluginManager {
}
public void installPlugin() {
final Set<String> names = new HashSet<>();
ServiceLoader.load(DataSourceChannelFactory.class).forEach(factory -> {
final String name = factory.getName();
logger.info("Registering datasource plugin: {}", name);
if (!names.add(name)) {
if (datasourceClientMap.containsKey(name)) {
throw new IllegalStateException(format("Duplicate datasource plugins named '%s'", name));
}

12
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/plugin/DataSourceProcessorManager.java
View File

@ -17,18 +17,17 @@
package org.apache.dolphinscheduler.plugin.datasource.api.plugin;
import static java.lang.String.format;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.DataSourceProcessor;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Collections;
import java.util.HashSet;
import java.util.Map;
import java.util.ServiceLoader;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import static java.lang.String.format;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class DataSourceProcessorManager {
private static final Logger logger = LoggerFactory.getLogger(DataSourceProcessorManager.class);
@ -40,13 +39,12 @@ public class DataSourceProcessorManager {
}
public void installProcessor() {
final Set<String> names = new HashSet<>();
ServiceLoader.load(DataSourceProcessor.class).forEach(factory -> {
final String name = factory.getDbType().name();
logger.info("start register processor: {}", name);
if (!names.add(name)) {
if (dataSourceProcessorMap.containsKey(name)) {
throw new IllegalStateException(format("Duplicate datasource plugins named '%s'", name));
}
loadDatasourceClient(factory);

2
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/client/CommonDataSourceClientTest.java
View File

@ -17,7 +17,7 @@
package org.apache.dolphinscheduler.plugin.datasource.api.client;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.mysql.MySQLConnectionParam;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.MySQLConnectionParam;
import org.apache.dolphinscheduler.plugin.datasource.api.provider.JDBCDataSourceProvider;
import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam;
import org.apache.dolphinscheduler.spi.enums.DbType;

5
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/mysql/MySQLConnectionParam.java → dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/MySQLConnectionParam.java
View File

@ -15,10 +15,13 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.datasource.api.datasource.mysql;
package org.apache.dolphinscheduler.plugin.datasource.api.datasource;
import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam;
/**
* Used for test.
*/
public class MySQLConnectionParam extends BaseConnectionParam {
@Override
public String toString() {

2
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/clickhouse/ClickHouseConnectionParam.java → dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-clickhouse/src/main/java/org/apache/dolphinscheduler/plugin/datasource/clickhouse/param/ClickHouseConnectionParam.java
View File

@ -15,7 +15,7 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.datasource.api.datasource.clickhouse;
package org.apache.dolphinscheduler.plugin.datasource.clickhouse.param;
import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam;

2
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/clickhouse/ClickHouseDataSourceParamDTO.java → dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-clickhouse/src/main/java/org/apache/dolphinscheduler/plugin/datasource/clickhouse/param/ClickHouseDataSourceParamDTO.java
View File

@ -15,7 +15,7 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.datasource.api.datasource.clickhouse;
package org.apache.dolphinscheduler.plugin.datasource.clickhouse.param;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO;
import org.apache.dolphinscheduler.spi.enums.DbType;

12
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/clickhouse/ClickHouseDataSourceProcessor.java → dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-clickhouse/src/main/java/org/apache/dolphinscheduler/plugin/datasource/clickhouse/param/ClickHouseDataSourceProcessor.java
View File

@ -15,10 +15,8 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.datasource.api.datasource.clickhouse;
package org.apache.dolphinscheduler.plugin.datasource.clickhouse.param;
import com.google.auto.service.AutoService;
import org.apache.commons.collections4.MapUtils;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.AbstractDataSourceProcessor;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.DataSourceProcessor;
@ -29,12 +27,16 @@ import org.apache.dolphinscheduler.spi.utils.Constants;
import org.apache.dolphinscheduler.spi.utils.JSONUtils;
import org.apache.dolphinscheduler.spi.utils.StringUtils;
import org.apache.commons.collections4.MapUtils;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.util.LinkedHashMap;
import java.util.Map;
import com.google.auto.service.AutoService;
@AutoService(DataSourceProcessor.class)
public class ClickHouseDataSourceProcessor extends AbstractDataSourceProcessor {
@ -43,12 +45,12 @@ public class ClickHouseDataSourceProcessor extends AbstractDataSourceProcessor {
return JSONUtils.parseObject(paramJson, ClickHouseDataSourceParamDTO.class);
}
@Override
public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) {
ClickHouseConnectionParam connectionParams = (ClickHouseConnectionParam) createConnectionParams(connectionJson);
ClickHouseDataSourceParamDTO clickHouseDatasourceParamDTO = new ClickHouseDataSourceParamDTO();
ClickHouseDataSourceParamDTO
clickHouseDatasourceParamDTO = new ClickHouseDataSourceParamDTO();
clickHouseDatasourceParamDTO.setDatabase(connectionParams.getDatabase());
clickHouseDatasourceParamDTO.setUserName(connectionParams.getUser());
clickHouseDatasourceParamDTO.setOther(parseOther(connectionParams.getOther()));

2
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-clickhouse/src/test/java/org/apache/dolphinscheduler/plugin/datasource/clickhouse/ClickHouseDataSourceChannelTest.java
View File

@ -17,7 +17,7 @@
package org.apache.dolphinscheduler.plugin.datasource.clickhouse;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.clickhouse.ClickHouseConnectionParam;
import org.apache.dolphinscheduler.plugin.datasource.clickhouse.param.ClickHouseConnectionParam;
import org.apache.dolphinscheduler.spi.enums.DbType;
import org.junit.Assert;

2
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/db2/Db2ConnectionParam.java → dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-db2/src/main/java/org/apache/dolphinscheduler/plugin/datasource/db2/param/Db2ConnectionParam.java
View File

@ -15,7 +15,7 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.datasource.api.datasource.db2;
package org.apache.dolphinscheduler.plugin.datasource.db2.param;
import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam;

2
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/db2/Db2DataSourceParamDTO.java → dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-db2/src/main/java/org/apache/dolphinscheduler/plugin/datasource/db2/param/Db2DataSourceParamDTO.java
View File

@ -15,7 +15,7 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.datasource.api.datasource.db2;
package org.apache.dolphinscheduler.plugin.datasource.db2.param;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO;
import org.apache.dolphinscheduler.spi.enums.DbType;

11
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/db2/Db2DataSourceProcessor.java → dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-db2/src/main/java/org/apache/dolphinscheduler/plugin/datasource/db2/param/Db2DataSourceProcessor.java
View File

@ -15,10 +15,8 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.datasource.api.datasource.db2;
package org.apache.dolphinscheduler.plugin.datasource.db2.param;
import com.google.auto.service.AutoService;
import org.apache.commons.collections4.MapUtils;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.AbstractDataSourceProcessor;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.DataSourceProcessor;
@ -30,12 +28,16 @@ import org.apache.dolphinscheduler.spi.utils.Constants;
import org.apache.dolphinscheduler.spi.utils.JSONUtils;
import org.apache.dolphinscheduler.spi.utils.StringUtils;
import org.apache.commons.collections4.MapUtils;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.util.LinkedHashMap;
import java.util.Map;
import com.google.auto.service.AutoService;
@AutoService(DataSourceProcessor.class)
public class Db2DataSourceProcessor extends AbstractDataSourceProcessor {
@ -48,7 +50,8 @@ public class Db2DataSourceProcessor extends AbstractDataSourceProcessor {
public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) {
Db2ConnectionParam connectionParams = (Db2ConnectionParam) createConnectionParams(connectionJson);
Db2DataSourceParamDTO db2DatasourceParamDTO = new Db2DataSourceParamDTO();
Db2DataSourceParamDTO
db2DatasourceParamDTO = new Db2DataSourceParamDTO();
db2DatasourceParamDTO.setDatabase(connectionParams.getDatabase());
db2DatasourceParamDTO.setOther(parseOther(connectionParams.getOther()));
db2DatasourceParamDTO.setUserName(db2DatasourceParamDTO.getUserName());

2
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-db2/src/test/java/org/apache/dolphinscheduler/plugin/datasource/db2/DB2DataSourceChannelTest.java
View File

@ -17,7 +17,7 @@
package org.apache.dolphinscheduler.plugin.datasource.db2;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.db2.Db2ConnectionParam;
import org.apache.dolphinscheduler.plugin.datasource.db2.param.Db2ConnectionParam;
import org.apache.dolphinscheduler.spi.enums.DbType;
import org.junit.Assert;

2
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/db2/Db2DataSourceProcessorTest.java → dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-db2/src/test/java/org/apache/dolphinscheduler/plugin/datasource/db2/param/Db2DataSourceProcessorTest.java
View File

@ -15,7 +15,7 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.datasource.api.datasource.db2;
package org.apache.dolphinscheduler.plugin.datasource.db2.param;
import org.apache.dolphinscheduler.plugin.datasource.api.plugin.DataSourceClientProvider;
import org.apache.dolphinscheduler.plugin.datasource.api.utils.CommonUtils;

2
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/main/java/org/apache/dolphinscheduler/plugin/datasource/hive/HiveDataSourceClient.java
View File

@ -20,7 +20,7 @@ package org.apache.dolphinscheduler.plugin.datasource.hive;
import com.zaxxer.hikari.HikariDataSource;
import org.apache.dolphinscheduler.plugin.datasource.api.client.CommonDataSourceClient;
import org.apache.dolphinscheduler.plugin.datasource.api.provider.JDBCDataSourceProvider;
import org.apache.dolphinscheduler.plugin.datasource.utils.CommonUtil;
import org.apache.dolphinscheduler.plugin.datasource.hive.utils.CommonUtil;
import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam;
import org.apache.dolphinscheduler.spi.enums.DbType;
import org.apache.dolphinscheduler.spi.utils.Constants;

2
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/hive/HiveConnectionParam.java → dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/main/java/org/apache/dolphinscheduler/plugin/datasource/hive/param/HiveConnectionParam.java
View File

@ -15,7 +15,7 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.datasource.api.datasource.hive;
package org.apache.dolphinscheduler.plugin.datasource.hive.param;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseHDFSConnectionParam;

2
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/hive/HiveDataSourceParamDTO.java → dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/main/java/org/apache/dolphinscheduler/plugin/datasource/hive/param/HiveDataSourceParamDTO.java
View File

@ -15,7 +15,7 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.datasource.api.datasource.hive;
package org.apache.dolphinscheduler.plugin.datasource.hive.param;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseHDFSDataSourceParamDTO;
import org.apache.dolphinscheduler.spi.enums.DbType;

11
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/hive/HiveDataSourceProcessor.java → dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/main/java/org/apache/dolphinscheduler/plugin/datasource/hive/param/HiveDataSourceProcessor.java
View File

@ -15,10 +15,8 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.datasource.api.datasource.hive;
package org.apache.dolphinscheduler.plugin.datasource.hive.param;
import com.google.auto.service.AutoService;
import org.apache.commons.collections4.MapUtils;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.AbstractDataSourceProcessor;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.DataSourceProcessor;
@ -31,6 +29,8 @@ import org.apache.dolphinscheduler.spi.utils.Constants;
import org.apache.dolphinscheduler.spi.utils.JSONUtils;
import org.apache.dolphinscheduler.spi.utils.StringUtils;
import org.apache.commons.collections4.MapUtils;
import java.io.IOException;
import java.sql.Connection;
import java.sql.DriverManager;
@ -38,6 +38,8 @@ import java.sql.SQLException;
import java.util.LinkedHashMap;
import java.util.Map;
import com.google.auto.service.AutoService;
@AutoService(DataSourceProcessor.class)
public class HiveDataSourceProcessor extends AbstractDataSourceProcessor {
@ -48,7 +50,8 @@ public class HiveDataSourceProcessor extends AbstractDataSourceProcessor {
@Override
public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) {
HiveDataSourceParamDTO hiveDataSourceParamDTO = new HiveDataSourceParamDTO();
HiveDataSourceParamDTO
hiveDataSourceParamDTO = new HiveDataSourceParamDTO();
HiveConnectionParam hiveConnectionParam = (HiveConnectionParam) createConnectionParams(connectionJson);
hiveDataSourceParamDTO.setDatabase(hiveConnectionParam.getDatabase());

2
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/main/java/org/apache/dolphinscheduler/plugin/datasource/utils/CommonUtil.java → dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/main/java/org/apache/dolphinscheduler/plugin/datasource/hive/utils/CommonUtil.java
View File

@ -15,7 +15,7 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.datasource.utils;
package org.apache.dolphinscheduler.plugin.datasource.hive.utils;
import static org.apache.dolphinscheduler.spi.utils.Constants.JAVA_SECURITY_KRB5_CONF;

2
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/test/java/org/apache/dolphinscheduler/plugin/datasource/hive/HiveDataSourceChannelTest.java
View File

@ -17,7 +17,7 @@
package org.apache.dolphinscheduler.plugin.datasource.hive;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.hive.HiveConnectionParam;
import org.apache.dolphinscheduler.plugin.datasource.hive.param.HiveConnectionParam;
import org.apache.dolphinscheduler.spi.enums.DbType;
import org.junit.Assert;

2
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/hive/HiveDataSourceProcessorTest.java → dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/test/java/org/apache/dolphinscheduler/plugin/datasource/hive/param/HiveDataSourceProcessorTest.java
View File

@ -15,7 +15,7 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.datasource.api.datasource.hive;
package org.apache.dolphinscheduler.plugin.datasource.hive.param;
import org.apache.dolphinscheduler.plugin.datasource.api.plugin.DataSourceClientProvider;
import org.apache.dolphinscheduler.plugin.datasource.api.utils.CommonUtils;

37
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-mysql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/mysql/param/MySQLConnectionParam.java Normal file
View File

@ -0,0 +1,37 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.datasource.mysql.param;
import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam;
public class MySQLConnectionParam extends BaseConnectionParam {
@Override
public String toString() {
return "MySQLConnectionParam{"
+ "user='" + user + '\''
+ ", password='" + password + '\''
+ ", address='" + address + '\''
+ ", database='" + database + '\''
+ ", jdbcUrl='" + jdbcUrl + '\''
+ ", driverLocation='" + driverLocation + '\''
+ ", driverClassName='" + driverClassName + '\''
+ ", validationQuery='" + validationQuery + '\''
+ ", other='" + other + '\''
+ '}';
}
}

2
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/mysql/MySQLDataSourceParamDTO.java → dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-mysql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/mysql/param/MySQLDataSourceParamDTO.java
View File

@ -15,7 +15,7 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.datasource.api.datasource.mysql;
package org.apache.dolphinscheduler.plugin.datasource.mysql.param;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO;
import org.apache.dolphinscheduler.spi.enums.DbType;

25
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/mysql/MySQLDataSourceProcessor.java → dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-mysql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/mysql/param/MySQLDataSourceProcessor.java
View File

@ -15,10 +15,8 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.datasource.api.datasource.mysql;
package org.apache.dolphinscheduler.plugin.datasource.mysql.param;
import com.google.auto.service.AutoService;
import org.apache.commons.collections4.MapUtils;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.AbstractDataSourceProcessor;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.DataSourceProcessor;
@ -29,8 +27,8 @@ import org.apache.dolphinscheduler.spi.enums.DbType;
import org.apache.dolphinscheduler.spi.utils.Constants;
import org.apache.dolphinscheduler.spi.utils.JSONUtils;
import org.apache.dolphinscheduler.spi.utils.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.commons.collections4.MapUtils;
import java.sql.Connection;
import java.sql.DriverManager;
@ -39,6 +37,11 @@ import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.Map;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.auto.service.AutoService;
@AutoService(DataSourceProcessor.class)
public class MySQLDataSourceProcessor extends AbstractDataSourceProcessor {
@ -61,8 +64,10 @@ public class MySQLDataSourceProcessor extends AbstractDataSourceProcessor {
@Override
public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) {
MySQLConnectionParam connectionParams = (MySQLConnectionParam) createConnectionParams(connectionJson);
MySQLDataSourceParamDTO mysqlDatasourceParamDTO = new MySQLDataSourceParamDTO();
MySQLConnectionParam
connectionParams = (MySQLConnectionParam) createConnectionParams(connectionJson);
MySQLDataSourceParamDTO
mysqlDatasourceParamDTO = new MySQLDataSourceParamDTO();
mysqlDatasourceParamDTO.setUserName(connectionParams.getUser());
mysqlDatasourceParamDTO.setDatabase(connectionParams.getDatabase());
@ -83,7 +88,8 @@ public class MySQLDataSourceProcessor extends AbstractDataSourceProcessor {
String address = String.format("%s%s:%s", Constants.JDBC_MYSQL, mysqlDatasourceParam.getHost(), mysqlDatasourceParam.getPort());
String jdbcUrl = String.format("%s/%s", address, mysqlDatasourceParam.getDatabase());
MySQLConnectionParam mysqlConnectionParam = new MySQLConnectionParam();
MySQLConnectionParam
mysqlConnectionParam = new MySQLConnectionParam();
mysqlConnectionParam.setJdbcUrl(jdbcUrl);
mysqlConnectionParam.setDatabase(mysqlDatasourceParam.getDatabase());
mysqlConnectionParam.setAddress(address);
@ -114,7 +120,8 @@ public class MySQLDataSourceProcessor extends AbstractDataSourceProcessor {
@Override
public String getJdbcUrl(ConnectionParam connectionParam) {
MySQLConnectionParam mysqlConnectionParam = (MySQLConnectionParam) connectionParam;
MySQLConnectionParam
mysqlConnectionParam = (MySQLConnectionParam) connectionParam;
String jdbcUrl = mysqlConnectionParam.getJdbcUrl();
if (!StringUtils.isEmpty(mysqlConnectionParam.getOther())) {
return String.format("%s?%s&%s", jdbcUrl, mysqlConnectionParam.getOther(), APPEND_PARAMS);

2
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-mysql/src/test/java/org/apache/dolphinscheduler/plugin/datasource/mysql/MySQLDataSourceChannelTest.java
View File

@ -17,7 +17,7 @@
package org.apache.dolphinscheduler.plugin.datasource.mysql;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.mysql.MySQLConnectionParam;
import org.apache.dolphinscheduler.plugin.datasource.mysql.param.MySQLConnectionParam;
import org.apache.dolphinscheduler.spi.enums.DbType;
import org.junit.Assert;

2
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/mysql/MySQLDataSourceProcessorTest.java → dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-mysql/src/test/java/org/apache/dolphinscheduler/plugin/datasource/mysql/param/MySQLDataSourceProcessorTest.java
View File

@ -15,7 +15,7 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.datasource.api.datasource.mysql;
package org.apache.dolphinscheduler.plugin.datasource.mysql.param;
import org.apache.dolphinscheduler.plugin.datasource.api.plugin.DataSourceClientProvider;
import org.apache.dolphinscheduler.plugin.datasource.api.utils.CommonUtils;

5
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/provider/JDBCDataSourceProviderTest.java → dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-mysql/src/test/java/org/apache/dolphinscheduler/plugin/datasource/mysql/provider/JDBCDataSourceProviderTest.java
View File

@ -15,9 +15,10 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.datasource.api.provider;
package org.apache.dolphinscheduler.plugin.datasource.mysql.provider;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.mysql.MySQLConnectionParam;
import org.apache.dolphinscheduler.plugin.datasource.api.provider.JDBCDataSourceProvider;
import org.apache.dolphinscheduler.plugin.datasource.mysql.param.MySQLConnectionParam;
import org.apache.dolphinscheduler.spi.enums.DbType;
import org.junit.Assert;

25
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/utils/DataSourceUtilsTest.java → dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-mysql/src/test/java/org/apache/dolphinscheduler/plugin/datasource/mysql/utils/DataSourceUtilsTest.java
View File

@ -15,17 +15,19 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.datasource.api.utils;
package org.apache.dolphinscheduler.plugin.datasource.mysql.utils;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.mysql.MySQLConnectionParam;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.mysql.MySQLDataSourceParamDTO;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.mysql.MySQLDataSourceProcessor;
import org.apache.dolphinscheduler.plugin.datasource.api.plugin.DataSourceClientProvider;
import org.apache.dolphinscheduler.plugin.task.api.TaskConstants;
import org.apache.dolphinscheduler.plugin.datasource.api.utils.CommonUtils;
import org.apache.dolphinscheduler.plugin.datasource.api.utils.DataSourceUtils;
import org.apache.dolphinscheduler.plugin.datasource.api.utils.PasswordUtils;
import org.apache.dolphinscheduler.plugin.datasource.mysql.param.MySQLConnectionParam;
import org.apache.dolphinscheduler.plugin.datasource.mysql.param.MySQLDataSourceParamDTO;
import org.apache.dolphinscheduler.plugin.datasource.mysql.param.MySQLDataSourceProcessor;
import org.apache.dolphinscheduler.spi.datasource.ConnectionParam;
import org.apache.dolphinscheduler.spi.enums.DbType;
import org.apache.dolphinscheduler.spi.utils.Constants;
import org.apache.dolphinscheduler.spi.utils.JSONUtils;
import org.apache.dolphinscheduler.spi.utils.PropertyUtils;
import java.sql.Connection;
import java.sql.DriverManager;
@ -33,7 +35,6 @@ import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.ExecutionException;
import org.apache.dolphinscheduler.spi.utils.PropertyUtils;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
@ -93,7 +94,7 @@ public class DataSourceUtilsTest {
@Test
public void testGetConnection() throws ExecutionException {
PowerMockito.mockStatic(PropertyUtils.class);
PowerMockito.when(PropertyUtils.getLong(TaskConstants.KERBEROS_EXPIRE_TIME, 24L)).thenReturn(24L);
PowerMockito.when(PropertyUtils.getLong("kerberos.expire.time", 24L)).thenReturn(24L);
PowerMockito.mockStatic(DataSourceClientProvider.class);
DataSourceClientProvider clientProvider = PowerMockito.mock(DataSourceClientProvider.class);
PowerMockito.when(DataSourceClientProvider.getInstance()).thenReturn(clientProvider);
@ -134,14 +135,6 @@ public class DataSourceUtilsTest {
@Test
public void testGetDatasourceProcessor() {
Assert.assertNotNull(DataSourceUtils.getDatasourceProcessor(DbType.MYSQL));
Assert.assertNotNull(DataSourceUtils.getDatasourceProcessor(DbType.POSTGRESQL));
Assert.assertNotNull(DataSourceUtils.getDatasourceProcessor(DbType.HIVE));
Assert.assertNotNull(DataSourceUtils.getDatasourceProcessor(DbType.SPARK));
Assert.assertNotNull(DataSourceUtils.getDatasourceProcessor(DbType.CLICKHOUSE));
Assert.assertNotNull(DataSourceUtils.getDatasourceProcessor(DbType.ORACLE));
Assert.assertNotNull(DataSourceUtils.getDatasourceProcessor(DbType.SQLSERVER));
Assert.assertNotNull(DataSourceUtils.getDatasourceProcessor(DbType.DB2));
Assert.assertNotNull(DataSourceUtils.getDatasourceProcessor(DbType.PRESTO));
}
@Test(expected = Exception.class)

2
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/oracle/OracleConnectionParam.java → dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oracle/src/main/java/org/apache/dolphinscheduler/plugin/datasource/oracle/param/OracleConnectionParam.java
View File

@ -15,7 +15,7 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.datasource.api.datasource.oracle;
package org.apache.dolphinscheduler.plugin.datasource.oracle.param;
import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam;
import org.apache.dolphinscheduler.spi.enums.DbConnectType;

2
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/oracle/OracleDataSourceParamDTO.java → dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oracle/src/main/java/org/apache/dolphinscheduler/plugin/datasource/oracle/param/OracleDataSourceParamDTO.java
View File

@ -15,7 +15,7 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.datasource.api.datasource.oracle;
package org.apache.dolphinscheduler.plugin.datasource.oracle.param;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO;
import org.apache.dolphinscheduler.spi.enums.DbConnectType;

10
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/oracle/OracleDataSourceProcessor.java → dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oracle/src/main/java/org/apache/dolphinscheduler/plugin/datasource/oracle/param/OracleDataSourceProcessor.java
View File

@ -15,11 +15,8 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.datasource.api.datasource.oracle;
package org.apache.dolphinscheduler.plugin.datasource.oracle.param;
import com.google.auto.service.AutoService;
import org.apache.commons.collections4.MapUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.AbstractDataSourceProcessor;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.DataSourceProcessor;
@ -30,6 +27,9 @@ import org.apache.dolphinscheduler.spi.enums.DbConnectType;
import org.apache.dolphinscheduler.spi.enums.DbType;
import org.apache.dolphinscheduler.spi.utils.Constants;
import org.apache.dolphinscheduler.spi.utils.JSONUtils;
import org.apache.dolphinscheduler.spi.utils.StringUtils;
import org.apache.commons.collections4.MapUtils;
import java.sql.Connection;
import java.sql.DriverManager;
@ -39,6 +39,8 @@ import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import com.google.auto.service.AutoService;
@AutoService(DataSourceProcessor.class)
public class OracleDataSourceProcessor extends AbstractDataSourceProcessor {

2
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oracle/src/test/java/org/apache/dolphinscheduler/plugin/datasource/oracle/OracleDataSourceChannelTest.java
View File

@ -17,7 +17,7 @@
package org.apache.dolphinscheduler.plugin.datasource.oracle;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.oracle.OracleConnectionParam;
import org.apache.dolphinscheduler.plugin.datasource.oracle.param.OracleConnectionParam;
import org.apache.dolphinscheduler.spi.enums.DbType;
import org.junit.Assert;

4
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/oracle/OracleDataSourceProcessorTest.java → dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oracle/src/test/java/org/apache/dolphinscheduler/plugin/datasource/oracle/param/OracleDataSourceProcessorTest.java
View File

@ -15,7 +15,7 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.datasource.api.datasource.oracle;
package org.apache.dolphinscheduler.plugin.datasource.oracle.param;
import org.apache.dolphinscheduler.plugin.datasource.api.plugin.DataSourceClientProvider;
import org.apache.dolphinscheduler.plugin.datasource.api.utils.CommonUtils;
@ -97,4 +97,4 @@ public class OracleDataSourceProcessorTest {
public void testGetValidationQuery() {
Assert.assertEquals(Constants.ORACLE_VALIDATION_QUERY, oracleDatasourceProcessor.getValidationQuery());
}
}
}

2
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/postgresql/PostgreSQLConnectionParam.java → dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-postgresql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/postgresql/param/PostgreSQLConnectionParam.java
View File

@ -15,7 +15,7 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.datasource.api.datasource.postgresql;
package org.apache.dolphinscheduler.plugin.datasource.postgresql.param;
import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam;

2
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/postgresql/PostgreSQLDataSourceParamDTO.java → dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-postgresql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/postgresql/param/PostgreSQLDataSourceParamDTO.java
View File

@ -15,7 +15,7 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.datasource.api.datasource.postgresql;
package org.apache.dolphinscheduler.plugin.datasource.postgresql.param;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO;
import org.apache.dolphinscheduler.spi.enums.DbType;

12
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/postgresql/PostgreSQLDataSourceProcessor.java → dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-postgresql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/postgresql/param/PostgreSQLDataSourceProcessor.java
View File

@ -15,11 +15,8 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.datasource.api.datasource.postgresql;
package org.apache.dolphinscheduler.plugin.datasource.postgresql.param;
import com.google.auto.service.AutoService;
import org.apache.commons.collections4.MapUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.AbstractDataSourceProcessor;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.DataSourceProcessor;
@ -30,11 +27,15 @@ import org.apache.dolphinscheduler.spi.enums.DbType;
import org.apache.dolphinscheduler.spi.utils.Constants;
import org.apache.dolphinscheduler.spi.utils.JSONUtils;
import org.apache.commons.collections4.MapUtils;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.util.LinkedHashMap;
import java.util.Map;
import com.google.auto.service.AutoService;
import com.google.common.base.Strings;
@AutoService(DataSourceProcessor.class)
@ -48,7 +49,8 @@ public class PostgreSQLDataSourceProcessor extends AbstractDataSourceProcessor {
@Override
public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) {
PostgreSQLConnectionParam connectionParams = (PostgreSQLConnectionParam) createConnectionParams(connectionJson);
PostgreSQLDataSourceParamDTO postgreSqlDatasourceParamDTO = new PostgreSQLDataSourceParamDTO();
PostgreSQLDataSourceParamDTO
postgreSqlDatasourceParamDTO = new PostgreSQLDataSourceParamDTO();
postgreSqlDatasourceParamDTO.setDatabase(connectionParams.getDatabase());
postgreSqlDatasourceParamDTO.setUserName(connectionParams.getUser());
postgreSqlDatasourceParamDTO.setOther(parseOther(connectionParams.getOther()));

2
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-postgresql/src/test/java/org/apache/dolphinscheduler/plugin/datasource/postgresql/PostgreSQLDataSourceChannelTest.java
View File

@ -17,7 +17,7 @@
package org.apache.dolphinscheduler.plugin.datasource.postgresql;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.postgresql.PostgreSQLConnectionParam;
import org.apache.dolphinscheduler.plugin.datasource.postgresql.param.PostgreSQLConnectionParam;
import org.apache.dolphinscheduler.spi.enums.DbType;
import org.junit.Assert;

2
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/postgresql/PostgreSQLDataSourceProcessorTest.java → dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-postgresql/src/test/java/org/apache/dolphinscheduler/plugin/datasource/postgresql/param/PostgreSQLDataSourceProcessorTest.java
View File

@ -15,7 +15,7 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.datasource.api.datasource.postgresql;
package org.apache.dolphinscheduler.plugin.datasource.postgresql.param;
import org.apache.dolphinscheduler.plugin.datasource.api.plugin.DataSourceClientProvider;
import org.apache.dolphinscheduler.plugin.datasource.api.utils.CommonUtils;

2
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/presto/PrestoConnectionParam.java → dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-presto/src/main/java/org/apache/dolphinscheduler/plugin/datasource/presto/param/PrestoConnectionParam.java
View File

@ -15,7 +15,7 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.datasource.api.datasource.presto;
package org.apache.dolphinscheduler.plugin.datasource.presto.param;
import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam;

2
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/presto/PrestoDataSourceParamDTO.java → dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-presto/src/main/java/org/apache/dolphinscheduler/plugin/datasource/presto/param/PrestoDataSourceParamDTO.java
View File

@ -15,7 +15,7 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.datasource.api.datasource.presto;
package org.apache.dolphinscheduler.plugin.datasource.presto.param;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO;
import org.apache.dolphinscheduler.spi.enums.DbType;

13
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/presto/PrestoDataSourceProcessor.java → dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-presto/src/main/java/org/apache/dolphinscheduler/plugin/datasource/presto/param/PrestoDataSourceProcessor.java
View File

@ -15,11 +15,8 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.datasource.api.datasource.presto;
package org.apache.dolphinscheduler.plugin.datasource.presto.param;
import com.google.auto.service.AutoService;
import org.apache.commons.collections4.MapUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.AbstractDataSourceProcessor;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.DataSourceProcessor;
@ -29,6 +26,9 @@ import org.apache.dolphinscheduler.spi.datasource.ConnectionParam;
import org.apache.dolphinscheduler.spi.enums.DbType;
import org.apache.dolphinscheduler.spi.utils.Constants;
import org.apache.dolphinscheduler.spi.utils.JSONUtils;
import org.apache.dolphinscheduler.spi.utils.StringUtils;
import org.apache.commons.collections4.MapUtils;
import java.sql.Connection;
import java.sql.DriverManager;
@ -38,6 +38,8 @@ import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import com.google.auto.service.AutoService;
@AutoService(DataSourceProcessor.class)
public class PrestoDataSourceProcessor extends AbstractDataSourceProcessor {
@ -53,7 +55,8 @@ public class PrestoDataSourceProcessor extends AbstractDataSourceProcessor {
String[] hostSeperator = connectionParams.getAddress().split(Constants.DOUBLE_SLASH);
String[] hostPortArray = hostSeperator[hostSeperator.length - 1].split(Constants.COMMA);
PrestoDataSourceParamDTO prestoDatasourceParamDTO = new PrestoDataSourceParamDTO();
PrestoDataSourceParamDTO
prestoDatasourceParamDTO = new PrestoDataSourceParamDTO();
prestoDatasourceParamDTO.setPort(Integer.parseInt(hostPortArray[0].split(Constants.COLON)[1]));
prestoDatasourceParamDTO.setHost(hostPortArray[0].split(Constants.COLON)[0]);
prestoDatasourceParamDTO.setDatabase(connectionParams.getDatabase());

2
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-presto/src/test/java/org/apache/dolphinscheduler/plugin/datasource/presto/PrestoDataSourceChannelTest.java
View File

@ -17,7 +17,7 @@
package org.apache.dolphinscheduler.plugin.datasource.presto;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.presto.PrestoConnectionParam;
import org.apache.dolphinscheduler.plugin.datasource.presto.param.PrestoConnectionParam;
import org.apache.dolphinscheduler.spi.enums.DbType;
import org.junit.Assert;

2
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/presto/PrestoDataSourceProcessorTest.java → dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-presto/src/test/java/org/apache/dolphinscheduler/plugin/datasource/presto/param/PrestoDataSourceProcessorTest.java
View File

@ -15,7 +15,7 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.datasource.api.datasource.presto;
package org.apache.dolphinscheduler.plugin.datasource.presto.param;
import org.apache.dolphinscheduler.plugin.datasource.api.plugin.DataSourceClientProvider;
import org.apache.dolphinscheduler.plugin.datasource.api.utils.CommonUtils;

2
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/redshift/RedshiftConnectionParam.java → dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/src/main/java/org/apache/dolphinscheduler/plugin/datasource/redshift/param/RedshiftConnectionParam.java
View File

@ -15,7 +15,7 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.datasource.api.datasource.redshift;
package org.apache.dolphinscheduler.plugin.datasource.redshift.param;
import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam;

2
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/redshift/RedshiftDataSourceParamDTO.java → dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/src/main/java/org/apache/dolphinscheduler/plugin/datasource/redshift/param/RedshiftDataSourceParamDTO.java
View File

@ -15,7 +15,7 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.datasource.api.datasource.redshift;
package org.apache.dolphinscheduler.plugin.datasource.redshift.param;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO;
import org.apache.dolphinscheduler.spi.enums.DbType;

10
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/redshift/RedshiftDataSourceProcessor.java → dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/src/main/java/org/apache/dolphinscheduler/plugin/datasource/redshift/param/RedshiftDataSourceProcessor.java
View File

@ -15,11 +15,8 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.datasource.api.datasource.redshift;
package org.apache.dolphinscheduler.plugin.datasource.redshift.param;
import com.google.auto.service.AutoService;
import org.apache.commons.collections4.MapUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.AbstractDataSourceProcessor;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.DataSourceProcessor;
@ -29,6 +26,9 @@ import org.apache.dolphinscheduler.spi.datasource.ConnectionParam;
import org.apache.dolphinscheduler.spi.enums.DbType;
import org.apache.dolphinscheduler.spi.utils.Constants;
import org.apache.dolphinscheduler.spi.utils.JSONUtils;
import org.apache.dolphinscheduler.spi.utils.StringUtils;
import org.apache.commons.collections4.MapUtils;
import java.sql.Connection;
import java.sql.DriverManager;
@ -38,6 +38,8 @@ import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import com.google.auto.service.AutoService;
@AutoService(DataSourceProcessor.class)
public class RedshiftDataSourceProcessor extends AbstractDataSourceProcessor {

10
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/redshift/RedshiftDataSourceProcessorTest.java → dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/src/test/java/org/apache/dolphinscheduler/plugin/datasource/redshift/param/RedshiftDataSourceProcessorTest.java
View File

@ -15,7 +15,7 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.datasource.api.datasource.redshift;
package org.apache.dolphinscheduler.plugin.datasource.redshift.param;
import org.apache.dolphinscheduler.plugin.datasource.api.plugin.DataSourceClientProvider;
import org.apache.dolphinscheduler.plugin.datasource.api.utils.CommonUtils;
@ -56,7 +56,7 @@ public class RedshiftDataSourceProcessorTest {
PowerMockito.mockStatic(PasswordUtils.class);
PowerMockito.when(PasswordUtils.encodePassword(Mockito.anyString())).thenReturn("test");
RedshiftConnectionParam connectionParams = (RedshiftConnectionParam) redshiftDatasourceProcessor
.createConnectionParams(redshiftDatasourceParamDTO);
.createConnectionParams(redshiftDatasourceParamDTO);
Assert.assertEquals("jdbc:redshift://localhost:5439", connectionParams.getAddress());
Assert.assertEquals("jdbc:redshift://localhost:5439/dev", connectionParams.getJdbcUrl());
}
@ -64,9 +64,9 @@ public class RedshiftDataSourceProcessorTest {
@Test
public void testCreateConnectionParams2() {
String connectionJson = "{\"user\":\"awsuser\",\"password\":\"123456\",\"address\":\"jdbc:redshift://localhost:5439\""
+ ",\"database\":\"dev\",\"jdbcUrl\":\"jdbc:redshift://localhost:5439/dev\"}";
+ ",\"database\":\"dev\",\"jdbcUrl\":\"jdbc:redshift://localhost:5439/dev\"}";
RedshiftConnectionParam connectionParams = (RedshiftConnectionParam) redshiftDatasourceProcessor
.createConnectionParams(connectionJson);
.createConnectionParams(connectionJson);
Assert.assertNotNull(connectionParams);
Assert.assertEquals("awsuser", connectionParams.getUser());
}
@ -82,7 +82,7 @@ public class RedshiftDataSourceProcessorTest {
redshiftConnectionParam.setJdbcUrl("jdbc:redshift://localhost:5439/default");
redshiftConnectionParam.setOther("DSILogLevel=6;defaultRowFetchSize=100");
Assert.assertEquals("jdbc:redshift://localhost:5439/default?DSILogLevel=6;defaultRowFetchSize=100",
redshiftDatasourceProcessor.getJdbcUrl(redshiftConnectionParam));
redshiftDatasourceProcessor.getJdbcUrl(redshiftConnectionParam));
}

2
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/spark/SparkConnectionParam.java → dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-spark/src/main/java/org/apache/dolphinscheduler/plugin/datasource/spark/param/SparkConnectionParam.java
View File

@ -15,7 +15,7 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.datasource.api.datasource.spark;
package org.apache.dolphinscheduler.plugin.datasource.spark.param;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseHDFSConnectionParam;

2
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/spark/SparkDataSourceParamDTO.java → dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-spark/src/main/java/org/apache/dolphinscheduler/plugin/datasource/spark/param/SparkDataSourceParamDTO.java
View File

@ -15,7 +15,7 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.datasource.api.datasource.spark;
package org.apache.dolphinscheduler.plugin.datasource.spark.param;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseHDFSDataSourceParamDTO;
import org.apache.dolphinscheduler.spi.enums.DbType;

20
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/spark/SparkDataSourceProcessor.java → dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-spark/src/main/java/org/apache/dolphinscheduler/plugin/datasource/spark/param/SparkDataSourceProcessor.java
View File

@ -15,10 +15,8 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.datasource.api.datasource.spark;
package org.apache.dolphinscheduler.plugin.datasource.spark.param;
import com.google.auto.service.AutoService;
import org.apache.commons.collections4.MapUtils;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.AbstractDataSourceProcessor;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.DataSourceProcessor;
@ -31,6 +29,8 @@ import org.apache.dolphinscheduler.spi.utils.Constants;
import org.apache.dolphinscheduler.spi.utils.JSONUtils;
import org.apache.dolphinscheduler.spi.utils.StringUtils;
import org.apache.commons.collections4.MapUtils;
import java.io.IOException;
import java.sql.Connection;
import java.sql.DriverManager;
@ -41,6 +41,8 @@ import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import com.google.auto.service.AutoService;
@AutoService(DataSourceProcessor.class)
public class SparkDataSourceProcessor extends AbstractDataSourceProcessor {
@ -51,9 +53,11 @@ public class SparkDataSourceProcessor extends AbstractDataSourceProcessor {
@Override
public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) {
SparkConnectionParam connectionParams = (SparkConnectionParam) createConnectionParams(connectionJson);
SparkConnectionParam
connectionParams = (SparkConnectionParam) createConnectionParams(connectionJson);
SparkDataSourceParamDTO sparkDatasourceParamDTO = new SparkDataSourceParamDTO();
SparkDataSourceParamDTO
sparkDatasourceParamDTO = new SparkDataSourceParamDTO();
sparkDatasourceParamDTO.setDatabase(connectionParams.getDatabase());
sparkDatasourceParamDTO.setUserName(connectionParams.getUser());
sparkDatasourceParamDTO.setOther(parseOther(connectionParams.getOther()));
@ -85,7 +89,8 @@ public class SparkDataSourceProcessor extends AbstractDataSourceProcessor {
String jdbcUrl = address + "/" + sparkDatasourceParam.getDatabase();
SparkConnectionParam sparkConnectionParam = new SparkConnectionParam();
SparkConnectionParam
sparkConnectionParam = new SparkConnectionParam();
sparkConnectionParam.setPassword(PasswordUtils.encodePassword(sparkDatasourceParam.getPassword()));
sparkConnectionParam.setUser(sparkDatasourceParam.getUserName());
sparkConnectionParam.setOther(transformOther(sparkDatasourceParam.getOther()));
@ -123,7 +128,8 @@ public class SparkDataSourceProcessor extends AbstractDataSourceProcessor {
@Override
public String getJdbcUrl(ConnectionParam connectionParam) {
SparkConnectionParam sparkConnectionParam = (SparkConnectionParam) connectionParam;
SparkConnectionParam
sparkConnectionParam = (SparkConnectionParam) connectionParam;
if (!StringUtils.isEmpty(sparkConnectionParam.getOther())) {
return String.format("%s;%s", sparkConnectionParam.getJdbcUrl(), sparkConnectionParam.getOther());
}

2
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-spark/src/test/java/org/apache/dolphinscheduler/plugin/datasource/spark/SparkDataSourceChannelTest.java
View File

@ -17,7 +17,7 @@
package org.apache.dolphinscheduler.plugin.datasource.spark;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.spark.SparkConnectionParam;
import org.apache.dolphinscheduler.plugin.datasource.spark.param.SparkConnectionParam;
import org.apache.dolphinscheduler.spi.enums.DbType;
import org.junit.Assert;

2
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/spark/SparkDataSourceProcessorTest.java → dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-spark/src/test/java/org/apache/dolphinscheduler/plugin/datasource/spark/param/SparkDataSourceProcessorTest.java
View File

@ -15,7 +15,7 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.datasource.api.datasource.spark;
package org.apache.dolphinscheduler.plugin.datasource.spark.param;
import org.apache.dolphinscheduler.plugin.datasource.api.plugin.DataSourceClientProvider;
import org.apache.dolphinscheduler.plugin.datasource.api.utils.CommonUtils;

2
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/sqlserver/SQLServerConnectionParam.java → dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sqlserver/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sqlserver/param/SQLServerConnectionParam.java
View File

@ -15,7 +15,7 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.datasource.api.datasource.sqlserver;
package org.apache.dolphinscheduler.plugin.datasource.sqlserver.param;
import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam;

2
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/sqlserver/SQLServerDataSourceParamDTO.java → dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sqlserver/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sqlserver/param/SQLServerDataSourceParamDTO.java
View File

@ -15,7 +15,7 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.datasource.api.datasource.sqlserver;
package org.apache.dolphinscheduler.plugin.datasource.sqlserver.param;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO;
import org.apache.dolphinscheduler.spi.enums.DbType;

10
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/sqlserver/SQLServerDataSourceProcessor.java → dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sqlserver/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sqlserver/param/SQLServerDataSourceProcessor.java
View File

@ -15,11 +15,8 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.datasource.api.datasource.sqlserver;
package org.apache.dolphinscheduler.plugin.datasource.sqlserver.param;
import com.google.auto.service.AutoService;
import org.apache.commons.collections4.MapUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.AbstractDataSourceProcessor;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.DataSourceProcessor;
@ -29,6 +26,9 @@ import org.apache.dolphinscheduler.spi.datasource.ConnectionParam;
import org.apache.dolphinscheduler.spi.enums.DbType;
import org.apache.dolphinscheduler.spi.utils.Constants;
import org.apache.dolphinscheduler.spi.utils.JSONUtils;
import org.apache.dolphinscheduler.spi.utils.StringUtils;
import org.apache.commons.collections4.MapUtils;
import java.sql.Connection;
import java.sql.DriverManager;
@ -36,6 +36,8 @@ import java.sql.SQLException;
import java.util.LinkedHashMap;
import java.util.Map;
import com.google.auto.service.AutoService;
@AutoService(DataSourceProcessor.class)
public class SQLServerDataSourceProcessor extends AbstractDataSourceProcessor {

2
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sqlserver/src/test/java/org/apache/dolphinscheduler/plugin/datasource/sqlserver/SQLServerDataSourceChannelTest.java
View File

@ -17,7 +17,7 @@
package org.apache.dolphinscheduler.plugin.datasource.sqlserver;
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.sqlserver.SQLServerConnectionParam;
import org.apache.dolphinscheduler.plugin.datasource.sqlserver.param.SQLServerConnectionParam;
import org.apache.dolphinscheduler.spi.enums.DbType;
import org.junit.Assert;

2
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/sqlserver/SQLServerDataSourceProcessorTest.java → dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sqlserver/src/test/java/org/apache/dolphinscheduler/plugin/datasource/sqlserver/param/SQLServerDataSourceProcessorTest.java
View File

@ -15,7 +15,7 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.datasource.api.datasource.sqlserver;
package org.apache.dolphinscheduler.plugin.datasource.sqlserver.param;
import org.apache.dolphinscheduler.plugin.datasource.api.plugin.DataSourceClientProvider;
import org.apache.dolphinscheduler.plugin.datasource.api.utils.CommonUtils;

2
dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/permission/ResourcePermissionCheckService.java
View File

@ -41,7 +41,7 @@ public interface ResourcePermissionCheckService<T>{
* @param <T>
* @return
*/
<T> Set<T> userOwnedResourceIdsAcquisition(AuthorizationType authorizationType, Integer userId, Logger logger);
Set<T> userOwnedResourceIdsAcquisition(AuthorizationType authorizationType, Integer userId, Logger logger);
/**
* operationpermissionCheck

60
dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/permission/ResourcePermissionCheckServiceImpl.java
View File

@ -33,12 +33,42 @@
package org.apache.dolphinscheduler.service.permission;
import static java.util.stream.Collectors.toSet;
import org.apache.dolphinscheduler.common.enums.AuthorizationType;
import org.apache.dolphinscheduler.common.enums.UserType;
import org.apache.dolphinscheduler.dao.entity.*;
import org.apache.dolphinscheduler.dao.entity.Queue;
import org.apache.dolphinscheduler.dao.mapper.*;
import org.apache.dolphinscheduler.dao.entity.AccessToken;
import org.apache.dolphinscheduler.dao.entity.AlertGroup;
import org.apache.dolphinscheduler.dao.entity.DataSource;
import org.apache.dolphinscheduler.dao.entity.Project;
import org.apache.dolphinscheduler.dao.entity.Resource;
import org.apache.dolphinscheduler.dao.entity.UdfFunc;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.mapper.AccessTokenMapper;
import org.apache.dolphinscheduler.dao.mapper.AlertGroupMapper;
import org.apache.dolphinscheduler.dao.mapper.AlertPluginInstanceMapper;
import org.apache.dolphinscheduler.dao.mapper.CommandMapper;
import org.apache.dolphinscheduler.dao.mapper.DataSourceMapper;
import org.apache.dolphinscheduler.dao.mapper.DqRuleMapper;
import org.apache.dolphinscheduler.dao.mapper.EnvironmentMapper;
import org.apache.dolphinscheduler.dao.mapper.K8sNamespaceMapper;
import org.apache.dolphinscheduler.dao.mapper.ProjectMapper;
import org.apache.dolphinscheduler.dao.mapper.QueueMapper;
import org.apache.dolphinscheduler.dao.mapper.ResourceMapper;
import org.apache.dolphinscheduler.dao.mapper.TenantMapper;
import org.apache.dolphinscheduler.dao.mapper.UdfFuncMapper;
import org.apache.dolphinscheduler.dao.mapper.WorkerGroupMapper;
import org.apache.dolphinscheduler.service.process.ProcessService;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import org.slf4j.Logger;
import org.springframework.beans.BeansException;
import org.springframework.beans.factory.annotation.Autowired;
@ -46,11 +76,6 @@ import org.springframework.context.ApplicationContext;
import org.springframework.context.ApplicationContextAware;
import org.springframework.stereotype.Component;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import static java.util.stream.Collectors.toSet;
@Component
public class ResourcePermissionCheckServiceImpl implements ResourcePermissionCheckService<Object>, ApplicationContextAware {
@ -69,9 +94,9 @@ public class ResourcePermissionCheckServiceImpl implements ResourcePermissionChe
@Override
public boolean resourcePermissionCheck(AuthorizationType authorizationType, Object[] needChecks, Integer userId, Logger logger) {
if (Objects.nonNull(needChecks) && needChecks.length > 0){
Set<Object> originResSet = new HashSet<>(Arrays.asList(needChecks));
Set<Object> ownResSets = RESOURCE_LIST_MAP.get(authorizationType).listAuthorizedResource(userId, logger);
if (Objects.nonNull(needChecks) && needChecks.length > 0) {
Set<?> originResSet = new HashSet<>(Arrays.asList(needChecks));
Set<?> ownResSets = RESOURCE_LIST_MAP.get(authorizationType).listAuthorizedResource(userId, logger);
originResSet.removeAll(ownResSets);
return originResSet.isEmpty();
}
@ -94,13 +119,14 @@ public class ResourcePermissionCheckServiceImpl implements ResourcePermissionChe
}
@Override
public <T> Set<T> userOwnedResourceIdsAcquisition(AuthorizationType authorizationType, Integer userId, Logger logger) {
public Set<Object> userOwnedResourceIdsAcquisition(AuthorizationType authorizationType, Integer userId, Logger logger) {
User user = processService.getUserById(userId);
if (user == null){
if (user == null) {
logger.error("user id {} doesn't exist", userId);
return Collections.emptySet();
}
return RESOURCE_LIST_MAP.get(authorizationType).listAuthorizedResource(user.getUserType().equals(UserType.ADMIN_USER) ? 0 : userId, logger);
return (Set<Object>) RESOURCE_LIST_MAP.get(authorizationType).listAuthorizedResource(
user.getUserType().equals(UserType.ADMIN_USER) ? 0 : userId, logger);
}
@Component
@ -138,7 +164,7 @@ public class ResourcePermissionCheckServiceImpl implements ResourcePermissionChe
}
@Override
public <T> Set<T> listAuthorizedResource(int userId, Logger logger) {
public Set<Integer> listAuthorizedResource(int userId, Logger logger) {
return null;
}
@ -550,11 +576,11 @@ public class ResourcePermissionCheckServiceImpl implements ResourcePermissionChe
/**
* get all resources under the user (no admin)
*
* @param userId
* @param <T>
* @return
*/
<T> Set<T> listAuthorizedResource(int userId, Logger logger);
Set<T> listAuthorizedResource(int userId, Logger logger);
/**
* permission check

2
dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/pom.xml
View File

@ -41,7 +41,7 @@
</dependency>
<dependency>
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler-datasource-api</artifactId>
<artifactId>dolphinscheduler-datasource-all</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>

2
dolphinscheduler-task-plugin/dolphinscheduler-task-sql/pom.xml
View File

@ -41,7 +41,7 @@
</dependency>
<dependency>
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler-datasource-api</artifactId>
<artifactId>dolphinscheduler-datasource-all</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>