natas_hw
/
dolphinscheduler
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Revert "Fix datasource jdbc url param doesn't work (#13098)" 

This reverts commit 750cf62641.
This commit is contained in:
Jay Chung 2022-12-28 18:09:47 +08:00
parent dbccc23c54
commit 1bb9b86f5b
22 changed files with 337 additions and 236 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataSourceServiceTest.java
View File

@ -381,30 +381,21 @@ public class DataSourceServiceTest {
@Test
public void buildParameterWithDecodePassword() {
try (MockedStatic<PropertyUtils> mockedStaticPropertyUtils = Mockito.mockStatic(PropertyUtils.class)) {
mockedStaticPropertyUtils
.when(() -> PropertyUtils.getBoolean(DataSourceConstants.DATASOURCE_ENCRYPTION_ENABLE, false))
.thenReturn(true);
Map<String, String> other = new HashMap<>();
other.put("autoDeserialize", "yes");
other.put("allowUrlInLocalInfile", "true");
other.put("useSSL", "true");
MySQLDataSourceParamDTO mysqlDatasourceParamDTO = new MySQLDataSourceParamDTO();
mysqlDatasourceParamDTO.setHost("192.168.9.1");
mysqlDatasourceParamDTO.setPort(1521);
mysqlDatasourceParamDTO.setDatabase("im");
mysqlDatasourceParamDTO.setUserName("test");
mysqlDatasourceParamDTO.setPassword("123456");
mysqlDatasourceParamDTO.setOther(other);
ConnectionParam connectionParam = DataSourceUtils.buildConnectionParams(mysqlDatasourceParamDTO);
String expected =
"{\"user\":\"test\",\"password\":\"bnVsbE1USXpORFUy\",\"address\":\"jdbc:mysql://192.168.9.1:1521\","
+
"\"database\":\"im\",\"jdbcUrl\":\"jdbc:mysql://192.168.9.1:1521/im\",\"driverClassName\":\"com.mysql.cj.jdbc.Driver\","
+
"\"validationQuery\":\"select 1\",\"other\":{\"autoDeserialize\":\"yes\",\"allowUrlInLocalInfile\":\"true\",\"useSSL\":\"true\"}}";
Assertions.assertEquals(expected, JSONUtils.toJsonString(connectionParam));
}
PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE, "true");
Map<String, String> other = new HashMap<>();
other.put("autoDeserialize", "yes");
other.put("allowUrlInLocalInfile", "true");
MySQLDataSourceParamDTO mysqlDatasourceParamDTO = new MySQLDataSourceParamDTO();
mysqlDatasourceParamDTO.setHost("192.168.9.1");
mysqlDatasourceParamDTO.setPort(1521);
mysqlDatasourceParamDTO.setDatabase("im");
mysqlDatasourceParamDTO.setUserName("test");
mysqlDatasourceParamDTO.setPassword("123456");
mysqlDatasourceParamDTO.setOther(other);
ConnectionParam connectionParam = DataSourceUtils.buildConnectionParams(mysqlDatasourceParamDTO);
String expected = "{\"user\":\"test\",\"password\":\"IUAjJCVeJipNVEl6TkRVMg==\",\"address\":\"jdbc:mysql://192.168.9.1:1521\",\"database\":\"im\",\"jdbcUrl\":\"jdbc:mysql://192.168.9.1:1521/"
+ "im\",\"driverClassName\":\"com.mysql.cj.jdbc.Driver\",\"validationQuery\":\"select 1\",\"props\":{\"autoDeserialize\":\"yes\",\"allowUrlInLocalInfile\":\"true\"}}";
Assert.assertEquals(expected, JSONUtils.toJsonString(connectionParam));
PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE, "false");
mysqlDatasourceParamDTO = new MySQLDataSourceParamDTO();

14
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/AbstractDataSourceProcessor.java
View File

@ -17,23 +17,17 @@
package org.apache.dolphinscheduler.plugin.datasource.api.datasource;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.plugin.datasource.api.utils.PasswordUtils;
import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam;
import org.apache.dolphinscheduler.spi.datasource.ConnectionParam;
import org.apache.dolphinscheduler.spi.enums.DbType;
import org.apache.commons.collections4.MapUtils;
import org.apache.commons.lang3.StringUtils;
import java.text.MessageFormat;
import java.util.Collections;
import java.util.Map;
import java.util.regex.Pattern;
import com.fasterxml.jackson.core.type.TypeReference;
import com.google.common.collect.Sets;
public abstract class AbstractDataSourceProcessor implements DataSourceProcessor {
private static final Pattern IPV4_PATTERN = Pattern.compile("^[a-zA-Z0-9\\_\\-\\.\\,]+$");
@ -88,14 +82,6 @@ public abstract class AbstractDataSourceProcessor implements DataSourceProcessor
}
}
protected Map<String, String> transformOtherParamToMap(String other) {
if (StringUtils.isBlank(other)) {
return Collections.emptyMap();
}
return JSONUtils.parseObject(other, new TypeReference<Map<String, String>>() {
});
}
@Override
public String getDatasourceUniqueId(ConnectionParam connectionParam, DbType dbType) {
BaseConnectionParam baseConnectionParam = (BaseConnectionParam) connectionParam;

29
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/clickhouse/ClickHouseDataSourceProcessor.java
View File

@ -31,8 +31,7 @@ import org.apache.commons.collections4.MapUtils;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
import java.util.LinkedHashMap;
import java.util.Map;
public class ClickHouseDataSourceProcessor extends AbstractDataSourceProcessor {
@ -44,7 +43,7 @@ public class ClickHouseDataSourceProcessor extends AbstractDataSourceProcessor {
ClickHouseDataSourceParamDTO clickHouseDatasourceParamDTO = new ClickHouseDataSourceParamDTO();
clickHouseDatasourceParamDTO.setDatabase(connectionParams.getDatabase());
clickHouseDatasourceParamDTO.setUserName(connectionParams.getUser());
clickHouseDatasourceParamDTO.setOther(connectionParams.getOther());
clickHouseDatasourceParamDTO.setOther(parseOther(connectionParams.getOther()));
String[] hostSeperator = connectionParams.getAddress().split(Constants.DOUBLE_SLASH);
String[] hostPortArray = hostSeperator[hostSeperator.length - 1].split(Constants.COMMA);
@ -68,7 +67,8 @@ public class ClickHouseDataSourceProcessor extends AbstractDataSourceProcessor {
clickhouseConnectionParam.setPassword(PasswordUtils.encodePassword(clickHouseParam.getPassword()));
clickhouseConnectionParam.setDriverClassName(getDatasourceDriver());
clickhouseConnectionParam.setValidationQuery(getValidationQuery());
clickhouseConnectionParam.setOther(clickHouseParam.getOther());
clickhouseConnectionParam.setOther(transformOther(clickHouseParam.getOther()));
clickhouseConnectionParam.setProps(clickHouseParam.getOther());
return clickhouseConnectionParam;
}
@ -91,8 +91,8 @@ public class ClickHouseDataSourceProcessor extends AbstractDataSourceProcessor {
public String getJdbcUrl(ConnectionParam connectionParam) {
ClickHouseConnectionParam clickhouseConnectionParam = (ClickHouseConnectionParam) connectionParam;
String jdbcUrl = clickhouseConnectionParam.getJdbcUrl();
if (MapUtils.isNotEmpty(clickhouseConnectionParam.getOther())) {
jdbcUrl = String.format("%s?%s", jdbcUrl, transformOther(clickhouseConnectionParam.getOther()));
if (!StringUtils.isEmpty(clickhouseConnectionParam.getOther())) {
jdbcUrl = String.format("%s?%s", jdbcUrl, clickhouseConnectionParam.getOther());
}
return jdbcUrl;
}
@ -114,9 +114,20 @@ public class ClickHouseDataSourceProcessor extends AbstractDataSourceProcessor {
if (MapUtils.isEmpty(otherMap)) {
return null;
}
List<String> otherList = new ArrayList<>();
otherMap.forEach((key, value) -> otherList.add(String.format("%s=%s", key, value)));
return String.join("&", otherList);
StringBuilder stringBuilder = new StringBuilder();
otherMap.forEach((key, value) -> stringBuilder.append(String.format("%s=%s%s", key, value, "&")));
return stringBuilder.toString();
}
private Map<String, String> parseOther(String other) {
if (other == null) {
return null;
}
Map<String, String> otherMap = new LinkedHashMap<>();
String[] configs = other.split("&");
for (String config : configs) {
otherMap.put(config.split("=")[0], config.split("=")[1]);
}
return otherMap;
}
}

30
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/db2/Db2DataSourceProcessor.java
View File

@ -32,8 +32,7 @@ import org.apache.commons.collections4.MapUtils;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
import java.util.LinkedHashMap;
import java.util.Map;
public class Db2DataSourceProcessor extends AbstractDataSourceProcessor {
@ -44,7 +43,7 @@ public class Db2DataSourceProcessor extends AbstractDataSourceProcessor {
Db2DataSourceParamDTO db2DatasourceParamDTO = new Db2DataSourceParamDTO();
db2DatasourceParamDTO.setDatabase(connectionParams.getDatabase());
db2DatasourceParamDTO.setOther(connectionParams.getOther());
db2DatasourceParamDTO.setOther(parseOther(connectionParams.getOther()));
db2DatasourceParamDTO.setUserName(db2DatasourceParamDTO.getUserName());
String[] hostSeperator = connectionParams.getAddress().split(Constants.DOUBLE_SLASH);
@ -69,7 +68,8 @@ public class Db2DataSourceProcessor extends AbstractDataSourceProcessor {
db2ConnectionParam.setPassword(PasswordUtils.encodePassword(db2Param.getPassword()));
db2ConnectionParam.setDriverClassName(getDatasourceDriver());
db2ConnectionParam.setValidationQuery(getValidationQuery());
db2ConnectionParam.setOther(db2Param.getOther());
db2ConnectionParam.setOther(transformOther(db2Param.getOther()));
db2ConnectionParam.setProps(db2Param.getOther());
return db2ConnectionParam;
}
@ -87,9 +87,8 @@ public class Db2DataSourceProcessor extends AbstractDataSourceProcessor {
@Override
public String getJdbcUrl(ConnectionParam connectionParam) {
Db2ConnectionParam db2ConnectionParam = (Db2ConnectionParam) connectionParam;
if (MapUtils.isNotEmpty(db2ConnectionParam.getOther())) {
return String.format("%s;%s", db2ConnectionParam.getJdbcUrl(),
transformOther(db2ConnectionParam.getOther()));
if (!StringUtils.isEmpty(db2ConnectionParam.getOther())) {
return String.format("%s;%s", db2ConnectionParam.getJdbcUrl(), db2ConnectionParam.getOther());
}
return db2ConnectionParam.getJdbcUrl();
}
@ -116,9 +115,20 @@ public class Db2DataSourceProcessor extends AbstractDataSourceProcessor {
if (MapUtils.isEmpty(otherMap)) {
return null;
}
List<String> otherList = new ArrayList<>();
otherMap.forEach((key, value) -> otherList.add(String.format("%s=%s", key, value)));
return String.join(";", otherList);
StringBuilder stringBuilder = new StringBuilder();
otherMap.forEach((key, value) -> stringBuilder.append(String.format("%s=%s%s", key, value, ";")));
stringBuilder.deleteCharAt(stringBuilder.length() - 1);
return stringBuilder.toString();
}
private Map<String, String> parseOther(String other) {
if (other == null) {
return null;
}
Map<String, String> otherMap = new LinkedHashMap<>();
for (String config : other.split("&")) {
otherMap.put(config.split("=")[0], config.split("=")[1]);
}
return otherMap;
}
}

59
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/hive/HiveDataSourceProcessor.java
View File

@ -34,8 +34,7 @@ import java.io.IOException;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
import java.util.LinkedHashMap;
import java.util.Map;
public class HiveDataSourceProcessor extends AbstractDataSourceProcessor {
@ -47,7 +46,7 @@ public class HiveDataSourceProcessor extends AbstractDataSourceProcessor {
hiveDataSourceParamDTO.setDatabase(hiveConnectionParam.getDatabase());
hiveDataSourceParamDTO.setUserName(hiveConnectionParam.getUser());
hiveDataSourceParamDTO.setOther(hiveConnectionParam.getOther());
hiveDataSourceParamDTO.setOther(parseOther(hiveConnectionParam.getOther()));
hiveDataSourceParamDTO.setLoginUserKeytabUsername(hiveConnectionParam.getLoginUserKeytabUsername());
hiveDataSourceParamDTO.setLoginUserKeytabPath(hiveConnectionParam.getLoginUserKeytabPath());
hiveDataSourceParamDTO.setJavaSecurityKrb5Conf(hiveConnectionParam.getJavaSecurityKrb5Conf());
@ -91,7 +90,8 @@ public class HiveDataSourceProcessor extends AbstractDataSourceProcessor {
hiveConnectionParam.setLoginUserKeytabPath(hiveParam.getLoginUserKeytabPath());
hiveConnectionParam.setLoginUserKeytabUsername(hiveParam.getLoginUserKeytabUsername());
}
hiveConnectionParam.setOther(hiveParam.getOther());
hiveConnectionParam.setOther(transformOther(hiveParam.getOther()));
hiveConnectionParam.setProps(hiveParam.getOther());
return hiveConnectionParam;
}
@ -114,10 +114,11 @@ public class HiveDataSourceProcessor extends AbstractDataSourceProcessor {
public String getJdbcUrl(ConnectionParam connectionParam) {
HiveConnectionParam hiveConnectionParam = (HiveConnectionParam) connectionParam;
String jdbcUrl = hiveConnectionParam.getJdbcUrl();
if (MapUtils.isNotEmpty(hiveConnectionParam.getOther())) {
return jdbcUrl + "?" + transformOther(hiveConnectionParam.getOther());
String otherParams = filterOther(hiveConnectionParam.getOther());
if (StringUtils.isNotEmpty(otherParams) && !"?".equals(otherParams.substring(0, 1))) {
jdbcUrl += ";";
}
return jdbcUrl;
return jdbcUrl + otherParams;
}
@Override
@ -139,9 +140,47 @@ public class HiveDataSourceProcessor extends AbstractDataSourceProcessor {
if (MapUtils.isEmpty(otherMap)) {
return null;
}
List<String> otherList = new ArrayList<>();
otherMap.forEach((key, value) -> otherList.add(String.format("%s=%s", key, value)));
return String.join(";", otherList);
StringBuilder stringBuilder = new StringBuilder();
otherMap.forEach((key, value) -> stringBuilder.append(String.format("%s=%s;", key, value)));
return stringBuilder.toString();
}
private String filterOther(String otherParams) {
if (StringUtils.isBlank(otherParams)) {
return "";
}
StringBuilder hiveConfListSb = new StringBuilder();
hiveConfListSb.append("?");
StringBuilder sessionVarListSb = new StringBuilder();
String[] otherArray = otherParams.split(";", -1);
for (String conf : otherArray) {
sessionVarListSb.append(conf).append(";");
}
// remove the last ";"
if (sessionVarListSb.length() > 0) {
sessionVarListSb.deleteCharAt(sessionVarListSb.length() - 1);
}
if (hiveConfListSb.length() > 0) {
hiveConfListSb.deleteCharAt(hiveConfListSb.length() - 1);
}
return sessionVarListSb.toString() + hiveConfListSb.toString();
}
private Map<String, String> parseOther(String other) {
if (other == null) {
return null;
}
Map<String, String> otherMap = new LinkedHashMap<>();
String[] configs = other.split(";");
for (String config : configs) {
otherMap.put(config.split("=")[0], config.split("=")[1]);
}
return otherMap;
}
}

29
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/mysql/MySQLDataSourceProcessor.java
View File

@ -32,9 +32,8 @@ import org.apache.commons.collections4.MapUtils;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.LinkedHashMap;
import java.util.Map;
import org.slf4j.Logger;
@ -61,7 +60,7 @@ public class MySQLDataSourceProcessor extends AbstractDataSourceProcessor {
mysqlDatasourceParamDTO.setUserName(connectionParams.getUser());
mysqlDatasourceParamDTO.setDatabase(connectionParams.getDatabase());
mysqlDatasourceParamDTO.setOther(connectionParams.getOther());
mysqlDatasourceParamDTO.setOther(parseOther(connectionParams.getOther()));
String address = connectionParams.getAddress();
String[] hostSeperator = address.split(Constants.DOUBLE_SLASH);
@ -86,7 +85,8 @@ public class MySQLDataSourceProcessor extends AbstractDataSourceProcessor {
mysqlConnectionParam.setPassword(PasswordUtils.encodePassword(mysqlDatasourceParam.getPassword()));
mysqlConnectionParam.setDriverClassName(getDatasourceDriver());
mysqlConnectionParam.setValidationQuery(getValidationQuery());
mysqlConnectionParam.setOther(mysqlDatasourceParam.getOther());
mysqlConnectionParam.setOther(transformOther(mysqlDatasourceParam.getOther()));
mysqlConnectionParam.setProps(mysqlDatasourceParam.getOther());
return mysqlConnectionParam;
}
@ -110,8 +110,8 @@ public class MySQLDataSourceProcessor extends AbstractDataSourceProcessor {
public String getJdbcUrl(ConnectionParam connectionParam) {
MySQLConnectionParam mysqlConnectionParam = (MySQLConnectionParam) connectionParam;
String jdbcUrl = mysqlConnectionParam.getJdbcUrl();
if (MapUtils.isNotEmpty(mysqlConnectionParam.getOther())) {
return String.format("%s?%s&%s", jdbcUrl, transformOther(mysqlConnectionParam.getOther()), APPEND_PARAMS);
if (!StringUtils.isEmpty(mysqlConnectionParam.getOther())) {
return String.format("%s?%s&%s", jdbcUrl, mysqlConnectionParam.getOther(), APPEND_PARAMS);
}
return String.format("%s?%s", jdbcUrl, APPEND_PARAMS);
}
@ -152,9 +152,9 @@ public class MySQLDataSourceProcessor extends AbstractDataSourceProcessor {
if (MapUtils.isEmpty(otherMap)) {
return null;
}
List<String> otherList = new ArrayList<>();
otherMap.forEach((key, value) -> otherList.add(String.format("%s=%s", key, value)));
return String.join("&", otherList);
StringBuilder stringBuilder = new StringBuilder();
otherMap.forEach((key, value) -> stringBuilder.append(String.format("%s=%s&", key, value)));
return stringBuilder.toString();
}
private static boolean checkKeyIsLegitimate(String key) {
@ -164,4 +164,15 @@ public class MySQLDataSourceProcessor extends AbstractDataSourceProcessor {
&& !key.contains(ALLOW_URL_IN_LOCAL_IN_FILE_NAME);
}
private Map<String, String> parseOther(String other) {
if (StringUtils.isEmpty(other)) {
return null;
}
Map<String, String> otherMap = new LinkedHashMap<>();
for (String config : other.split("&")) {
otherMap.put(config.split("=")[0], config.split("=")[1]);
}
return otherMap;
}
}

28
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/oracle/OracleDataSourceProcessor.java
View File

@ -34,6 +34,7 @@ import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@ -46,7 +47,7 @@ public class OracleDataSourceProcessor extends AbstractDataSourceProcessor {
oracleDatasourceParamDTO.setDatabase(connectionParams.getDatabase());
oracleDatasourceParamDTO.setUserName(connectionParams.getUser());
oracleDatasourceParamDTO.setOther(connectionParams.getOther());
oracleDatasourceParamDTO.setOther(parseOther(connectionParams.getOther()));
String hostSeperator = Constants.DOUBLE_SLASH;
if (DbConnectType.ORACLE_SID.equals(connectionParams.connectType)) {
@ -84,7 +85,8 @@ public class OracleDataSourceProcessor extends AbstractDataSourceProcessor {
oracleConnectionParam.setConnectType(oracleParam.getConnectType());
oracleConnectionParam.setDriverClassName(getDatasourceDriver());
oracleConnectionParam.setValidationQuery(getValidationQuery());
oracleConnectionParam.setOther(oracleParam.getOther());
oracleConnectionParam.setOther(transformOther(oracleParam.getOther()));
oracleConnectionParam.setProps(oracleParam.getOther());
return oracleConnectionParam;
}
@ -107,9 +109,8 @@ public class OracleDataSourceProcessor extends AbstractDataSourceProcessor {
@Override
public String getJdbcUrl(ConnectionParam connectionParam) {
OracleConnectionParam oracleConnectionParam = (OracleConnectionParam) connectionParam;
if (MapUtils.isNotEmpty(oracleConnectionParam.getOther())) {
return String.format("%s?%s", oracleConnectionParam.getJdbcUrl(),
transformOther(oracleConnectionParam.getOther()));
if (!StringUtils.isEmpty(oracleConnectionParam.getOther())) {
return String.format("%s?%s", oracleConnectionParam.getJdbcUrl(), oracleConnectionParam.getOther());
}
return oracleConnectionParam.getJdbcUrl();
}
@ -131,9 +132,20 @@ public class OracleDataSourceProcessor extends AbstractDataSourceProcessor {
if (MapUtils.isEmpty(otherMap)) {
return null;
}
List<String> otherList = new ArrayList<>();
otherMap.forEach((key, value) -> otherList.add(String.format("%s=%s", key, value)));
return String.join("&", otherList);
List<String> list = new ArrayList<>();
otherMap.forEach((key, value) -> list.add(String.format("%s=%s", key, value)));
return String.join("&", list);
}
private Map<String, String> parseOther(String other) {
if (StringUtils.isEmpty(other)) {
return null;
}
Map<String, String> otherMap = new LinkedHashMap<>();
String[] configs = other.split("&");
for (String config : configs) {
otherMap.put(config.split("=")[0], config.split("=")[1]);
}
return otherMap;
}
}

34
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/postgresql/PostgreSQLDataSourceProcessor.java
View File

@ -32,14 +32,9 @@ import org.apache.commons.lang.StringUtils;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
import java.util.LinkedHashMap;
import java.util.Map;
import com.google.auto.service.AutoService;
import com.google.common.base.Strings;
@AutoService(DataSourceProcessor.class)
public class PostgreSQLDataSourceProcessor extends AbstractDataSourceProcessor {
@Override
@ -48,7 +43,7 @@ public class PostgreSQLDataSourceProcessor extends AbstractDataSourceProcessor {
PostgreSQLDataSourceParamDTO postgreSqlDatasourceParamDTO = new PostgreSQLDataSourceParamDTO();
postgreSqlDatasourceParamDTO.setDatabase(connectionParams.getDatabase());
postgreSqlDatasourceParamDTO.setUserName(connectionParams.getUser());
postgreSqlDatasourceParamDTO.setOther(connectionParams.getOther());
postgreSqlDatasourceParamDTO.setOther(parseOther(connectionParams.getOther()));
String address = connectionParams.getAddress();
String[] hostSeperator = address.split(Constants.DOUBLE_SLASH);
@ -73,7 +68,8 @@ public class PostgreSQLDataSourceProcessor extends AbstractDataSourceProcessor {
postgreSqlConnectionParam.setPassword(PasswordUtils.encodePassword(postgreSqlParam.getPassword()));
postgreSqlConnectionParam.setDriverClassName(getDatasourceDriver());
postgreSqlConnectionParam.setValidationQuery(getValidationQuery());
postgreSqlConnectionParam.setOther(postgreSqlParam.getOther());
postgreSqlConnectionParam.setOther(transformOther(postgreSqlParam.getOther()));
postgreSqlConnectionParam.setProps(postgreSqlParam.getOther());
return postgreSqlConnectionParam;
}
@ -96,9 +92,8 @@ public class PostgreSQLDataSourceProcessor extends AbstractDataSourceProcessor {
@Override
public String getJdbcUrl(ConnectionParam connectionParam) {
PostgreSQLConnectionParam postgreSqlConnectionParam = (PostgreSQLConnectionParam) connectionParam;
if (MapUtils.isNotEmpty(postgreSqlConnectionParam.getOther())) {
return String.format("%s?%s", postgreSqlConnectionParam.getJdbcUrl(),
transformOther(postgreSqlConnectionParam.getOther()));
if (!StringUtils.isEmpty(postgreSqlConnectionParam.getOther())) {
return String.format("%s?%s", postgreSqlConnectionParam.getJdbcUrl(), postgreSqlConnectionParam.getOther());
}
return postgreSqlConnectionParam.getJdbcUrl();
}
@ -120,9 +115,20 @@ public class PostgreSQLDataSourceProcessor extends AbstractDataSourceProcessor {
if (MapUtils.isEmpty(otherMap)) {
return null;
}
List<String> otherList = new ArrayList<>();
otherMap.forEach((key, value) -> otherList.add(String.format("%s=%s", key, value)));
return String.join("&", otherList);
StringBuilder stringBuilder = new StringBuilder();
otherMap.forEach((key, value) -> stringBuilder.append(String.format("%s=%s&", key, value)));
return stringBuilder.toString();
}
private Map<String, String> parseOther(String other) {
if (StringUtils.isEmpty(other)) {
return null;
}
Map<String, String> otherMap = new LinkedHashMap<>();
for (String config : other.split("&")) {
String[] split = config.split("=");
otherMap.put(split[0], split[1]);
}
return otherMap;
}
}

22
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/presto/PrestoDataSourceProcessor.java
View File

@ -33,6 +33,7 @@ import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@ -50,7 +51,7 @@ public class PrestoDataSourceProcessor extends AbstractDataSourceProcessor {
prestoDatasourceParamDTO.setHost(hostPortArray[0].split(Constants.COLON)[0]);
prestoDatasourceParamDTO.setDatabase(connectionParams.getDatabase());
prestoDatasourceParamDTO.setUserName(connectionParams.getUser());
prestoDatasourceParamDTO.setOther(connectionParams.getOther());
prestoDatasourceParamDTO.setOther(parseOther(connectionParams.getOther()));
return prestoDatasourceParamDTO;
}
@ -64,12 +65,13 @@ public class PrestoDataSourceProcessor extends AbstractDataSourceProcessor {
PrestoConnectionParam prestoConnectionParam = new PrestoConnectionParam();
prestoConnectionParam.setUser(prestoParam.getUserName());
prestoConnectionParam.setPassword(PasswordUtils.encodePassword(prestoParam.getPassword()));
prestoConnectionParam.setOther(prestoParam.getOther());
prestoConnectionParam.setOther(transformOther(prestoParam.getOther()));
prestoConnectionParam.setAddress(address);
prestoConnectionParam.setJdbcUrl(jdbcUrl);
prestoConnectionParam.setDatabase(prestoParam.getDatabase());
prestoConnectionParam.setDriverClassName(getDatasourceDriver());
prestoConnectionParam.setValidationQuery(getValidationQuery());
prestoConnectionParam.setProps(prestoParam.getOther());
return prestoConnectionParam;
}
@ -92,9 +94,8 @@ public class PrestoDataSourceProcessor extends AbstractDataSourceProcessor {
@Override
public String getJdbcUrl(ConnectionParam connectionParam) {
PrestoConnectionParam prestoConnectionParam = (PrestoConnectionParam) connectionParam;
if (MapUtils.isNotEmpty(prestoConnectionParam.getOther())) {
return String.format("%s?%s", prestoConnectionParam.getJdbcUrl(),
transformOther(prestoConnectionParam.getOther()));
if (!StringUtils.isEmpty(prestoConnectionParam.getOther())) {
return String.format("%s?%s", prestoConnectionParam.getJdbcUrl(), prestoConnectionParam.getOther());
}
return prestoConnectionParam.getJdbcUrl();
}
@ -121,4 +122,15 @@ public class PrestoDataSourceProcessor extends AbstractDataSourceProcessor {
return null;
}
private Map<String, String> parseOther(String other) {
if (StringUtils.isEmpty(other)) {
return null;
}
Map<String, String> otherMap = new LinkedHashMap<>();
String[] configs = other.split("&");
for (String config : configs) {
otherMap.put(config.split("=")[0], config.split("=")[1]);
}
return otherMap;
}
}

25
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/redshift/RedshiftDataSourceProcessor.java
View File

@ -33,6 +33,7 @@ import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@ -52,7 +53,7 @@ public class RedshiftDataSourceProcessor extends AbstractDataSourceProcessor {
redshiftDatasourceParamDTO.setHost(hostPortArray[0].split(Constants.COLON)[0]);
redshiftDatasourceParamDTO.setDatabase(connectionParams.getDatabase());
redshiftDatasourceParamDTO.setUserName(connectionParams.getUser());
redshiftDatasourceParamDTO.setOther(connectionParams.getOther());
redshiftDatasourceParamDTO.setOther(parseOther(connectionParams.getOther()));
return redshiftDatasourceParamDTO;
}
@ -67,12 +68,13 @@ public class RedshiftDataSourceProcessor extends AbstractDataSourceProcessor {
redshiftConnectionParam = new RedshiftConnectionParam();
redshiftConnectionParam.setUser(redshiftParam.getUserName());
redshiftConnectionParam.setPassword(PasswordUtils.encodePassword(redshiftParam.getPassword()));
redshiftConnectionParam.setOther(redshiftParam.getOther());
redshiftConnectionParam.setOther(transformOther(redshiftParam.getOther()));
redshiftConnectionParam.setAddress(address);
redshiftConnectionParam.setJdbcUrl(jdbcUrl);
redshiftConnectionParam.setDatabase(redshiftParam.getDatabase());
redshiftConnectionParam.setDriverClassName(getDatasourceDriver());
redshiftConnectionParam.setValidationQuery(getValidationQuery());
redshiftConnectionParam.setProps(redshiftParam.getOther());
return redshiftConnectionParam;
}
@ -94,10 +96,10 @@ public class RedshiftDataSourceProcessor extends AbstractDataSourceProcessor {
@Override
public String getJdbcUrl(ConnectionParam connectionParam) {
RedshiftConnectionParam redshiftConnectionParam = (RedshiftConnectionParam) connectionParam;
if (MapUtils.isNotEmpty(redshiftConnectionParam.getOther())) {
return String.format("%s?%s", redshiftConnectionParam.getJdbcUrl(),
transformOther(redshiftConnectionParam.getOther()));
RedshiftConnectionParam
redshiftConnectionParam = (RedshiftConnectionParam) connectionParam;
if (!StringUtils.isEmpty(redshiftConnectionParam.getOther())) {
return String.format("%s?%s", redshiftConnectionParam.getJdbcUrl(), redshiftConnectionParam.getOther());
}
return redshiftConnectionParam.getJdbcUrl();
}
@ -124,4 +126,15 @@ public class RedshiftDataSourceProcessor extends AbstractDataSourceProcessor {
return null;
}
private Map<String, String> parseOther(String other) {
Map<String, String> otherMap = new LinkedHashMap<>();
if (StringUtils.isEmpty(other)) {
return otherMap;
}
String[] configs = other.split(Constants.SEMICOLON);
for (String config : configs) {
otherMap.put(config.split(Constants.EQUAL_SIGN)[0], config.split(Constants.EQUAL_SIGN)[1]);
}
return otherMap;
}
}

22
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/spark/SparkDataSourceProcessor.java
View File

@ -35,6 +35,7 @@ import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.util.Arrays;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
@ -48,7 +49,7 @@ public class SparkDataSourceProcessor extends AbstractDataSourceProcessor {
SparkDataSourceParamDTO sparkDatasourceParamDTO = new SparkDataSourceParamDTO();
sparkDatasourceParamDTO.setDatabase(connectionParams.getDatabase());
sparkDatasourceParamDTO.setUserName(connectionParams.getUser());
sparkDatasourceParamDTO.setOther(connectionParams.getOther());
sparkDatasourceParamDTO.setOther(parseOther(connectionParams.getOther()));
sparkDatasourceParamDTO.setJavaSecurityKrb5Conf(connectionParams.getJavaSecurityKrb5Conf());
sparkDatasourceParamDTO.setLoginUserKeytabPath(connectionParams.getLoginUserKeytabPath());
sparkDatasourceParamDTO.setLoginUserKeytabUsername(connectionParams.getLoginUserKeytabUsername());
@ -80,12 +81,13 @@ public class SparkDataSourceProcessor extends AbstractDataSourceProcessor {
SparkConnectionParam sparkConnectionParam = new SparkConnectionParam();
sparkConnectionParam.setPassword(PasswordUtils.encodePassword(sparkDatasourceParam.getPassword()));
sparkConnectionParam.setUser(sparkDatasourceParam.getUserName());
sparkConnectionParam.setOther(sparkDatasourceParam.getOther());
sparkConnectionParam.setOther(transformOther(sparkDatasourceParam.getOther()));
sparkConnectionParam.setDatabase(sparkDatasourceParam.getDatabase());
sparkConnectionParam.setAddress(address.toString());
sparkConnectionParam.setJdbcUrl(jdbcUrl);
sparkConnectionParam.setDriverClassName(getDatasourceDriver());
sparkConnectionParam.setValidationQuery(getValidationQuery());
sparkConnectionParam.setProps(sparkDatasourceParam.getOther());
if (CommonUtils.getKerberosStartupState()) {
sparkConnectionParam.setPrincipal(sparkDatasourceParam.getPrincipal());
@ -115,9 +117,8 @@ public class SparkDataSourceProcessor extends AbstractDataSourceProcessor {
@Override
public String getJdbcUrl(ConnectionParam connectionParam) {
SparkConnectionParam sparkConnectionParam = (SparkConnectionParam) connectionParam;
if (MapUtils.isNotEmpty(sparkConnectionParam.getOther())) {
return String.format("%s;%s", sparkConnectionParam.getJdbcUrl(),
transformOther(sparkConnectionParam.getOther()));
if (!StringUtils.isEmpty(sparkConnectionParam.getOther())) {
return String.format("%s;%s", sparkConnectionParam.getJdbcUrl(), sparkConnectionParam.getOther());
}
return sparkConnectionParam.getJdbcUrl();
}
@ -146,4 +147,15 @@ public class SparkDataSourceProcessor extends AbstractDataSourceProcessor {
return String.join(";", stringBuilder);
}
private Map<String, String> parseOther(String other) {
if (StringUtils.isEmpty(other)) {
return null;
}
Map<String, String> otherMap = new LinkedHashMap<>();
String[] configs = other.split(";");
for (String config : configs) {
otherMap.put(config.split("=")[0], config.split("=")[1]);
}
return otherMap;
}
}

21
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/sqlserver/SQLServerDataSourceProcessor.java
View File

@ -32,6 +32,7 @@ import org.apache.commons.lang.StringUtils;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.util.LinkedHashMap;
import java.util.Map;
public class SQLServerDataSourceProcessor extends AbstractDataSourceProcessor {
@ -45,7 +46,7 @@ public class SQLServerDataSourceProcessor extends AbstractDataSourceProcessor {
SQLServerDataSourceParamDTO sqlServerDatasourceParamDTO = new SQLServerDataSourceParamDTO();
sqlServerDatasourceParamDTO.setDatabase(connectionParams.getDatabase());
sqlServerDatasourceParamDTO.setUserName(connectionParams.getUser());
sqlServerDatasourceParamDTO.setOther(connectionParams.getOther());
sqlServerDatasourceParamDTO.setOther(parseOther(connectionParams.getOther()));
sqlServerDatasourceParamDTO.setPort(Integer.parseInt(hostPortArray[0].split(Constants.COLON)[1]));
sqlServerDatasourceParamDTO.setHost(hostPortArray[0].split(Constants.COLON)[0]);
return sqlServerDatasourceParamDTO;
@ -61,11 +62,12 @@ public class SQLServerDataSourceProcessor extends AbstractDataSourceProcessor {
sqlServerConnectionParam.setAddress(address);
sqlServerConnectionParam.setDatabase(sqlServerParam.getDatabase());
sqlServerConnectionParam.setJdbcUrl(jdbcUrl);
sqlServerConnectionParam.setOther(sqlServerParam.getOther());
sqlServerConnectionParam.setOther(transformOther(sqlServerParam.getOther()));
sqlServerConnectionParam.setUser(sqlServerParam.getUserName());
sqlServerConnectionParam.setPassword(PasswordUtils.encodePassword(sqlServerParam.getPassword()));
sqlServerConnectionParam.setDriverClassName(getDatasourceDriver());
sqlServerConnectionParam.setValidationQuery(getValidationQuery());
sqlServerConnectionParam.setProps(sqlServerParam.getOther());
return sqlServerConnectionParam;
}
@ -88,9 +90,8 @@ public class SQLServerDataSourceProcessor extends AbstractDataSourceProcessor {
public String getJdbcUrl(ConnectionParam connectionParam) {
SQLServerConnectionParam sqlServerConnectionParam = (SQLServerConnectionParam) connectionParam;
if (MapUtils.isNotEmpty(sqlServerConnectionParam.getOther())) {
return String.format("%s;%s", sqlServerConnectionParam.getJdbcUrl(),
transformOther(sqlServerConnectionParam.getOther()));
if (!StringUtils.isEmpty(sqlServerConnectionParam.getOther())) {
return String.format("%s;%s", sqlServerConnectionParam.getJdbcUrl(), sqlServerConnectionParam.getOther());
}
return sqlServerConnectionParam.getJdbcUrl();
}
@ -117,4 +118,14 @@ public class SQLServerDataSourceProcessor extends AbstractDataSourceProcessor {
return stringBuilder.toString();
}
private Map<String, String> parseOther(String other) {
if (StringUtils.isEmpty(other)) {
return null;
}
Map<String, String> otherMap = new LinkedHashMap<>();
for (String config : other.split(";")) {
otherMap.put(config.split("=")[0], config.split("=")[1]);
}
return otherMap;
}
}

9
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/provider/JDBCDataSourceProvider.java
View File

@ -24,7 +24,6 @@ import org.apache.dolphinscheduler.spi.enums.DbType;
import org.apache.dolphinscheduler.spi.utils.Constants;
import org.apache.dolphinscheduler.spi.utils.PropertyUtils;
import org.apache.dolphinscheduler.spi.utils.StringUtils;
import org.apache.commons.collections4.MapUtils;
import java.sql.Driver;
@ -57,8 +56,8 @@ public class JDBCDataSourceProvider {
dataSource.setMaximumPoolSize(PropertyUtils.getInt(Constants.SPRING_DATASOURCE_MAX_ACTIVE, 50));
dataSource.setConnectionTestQuery(properties.getValidationQuery());
if (MapUtils.isNotEmpty(properties.getOther())) {
properties.getOther().forEach(dataSource::addDataSourceProperty);
if (properties.getProps() != null) {
properties.getProps().forEach(dataSource::addDataSourceProperty);
}
logger.info("Creating HikariDataSource pool success.");
@ -83,8 +82,8 @@ public class JDBCDataSourceProvider {
dataSource.setMaximumPoolSize(isOneSession ? 1 : PropertyUtils.getInt(Constants.SPRING_DATASOURCE_MAX_ACTIVE, 50));
dataSource.setConnectionTestQuery(properties.getValidationQuery());
if (MapUtils.isNotEmpty(properties.getOther())) {
properties.getOther().forEach(dataSource::addDataSourceProperty);
if (properties.getProps() != null) {
properties.getProps().forEach(dataSource::addDataSourceProperty);
}
logger.info("Creating OneSession HikariDataSource pool success.");

72
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/AbstractDataSourceProcessorTest.java
View File

@ -17,15 +17,10 @@
package org.apache.dolphinscheduler.plugin.datasource.api.datasource;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.mockito.Mockito.doNothing;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.mock;
import org.apache.dolphinscheduler.spi.datasource.ConnectionParam;
import org.apache.dolphinscheduler.spi.enums.DbType;
import java.sql.Connection;
import java.util.HashMap;
import java.util.Map;
@ -48,71 +43,4 @@ public class AbstractDataSourceProcessorTest {
other.put("arg0", "%");
doThrow(new IllegalArgumentException()).when(mockDataSourceProcessor).checkOther(other);
}
@Test
public void shouldNotIncludeMaliciousParams() {
AbstractDataSourceProcessor mockDataSourceProcessor = mock(AbstractDataSourceProcessor.class);
Map<String, String> other = new HashMap<>();
other.put("allowLoadLocalInfile", "whatever");
doThrow(new IllegalArgumentException()).when(mockDataSourceProcessor).checkOther(other);
}
@org.junit.Test
public void transformOtherParamToMap() {
AbstractDataSourceProcessor abstractDataSourceProcessor = new AbstractDataSourceProcessor() {
@Override
public BaseDataSourceParamDTO castDatasourceParamDTO(String paramJson) {
return null;
}
@Override
public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) {
return null;
}
@Override
public ConnectionParam createConnectionParams(BaseDataSourceParamDTO datasourceParam) {
return null;
}
@Override
public ConnectionParam createConnectionParams(String connectionJson) {
return null;
}
@Override
public String getDatasourceDriver() {
return null;
}
@Override
public String getValidationQuery() {
return null;
}
@Override
public String getJdbcUrl(ConnectionParam connectionParam) {
return null;
}
@Override
public Connection getConnection(ConnectionParam connectionParam) {
return null;
}
@Override
public DbType getDbType() {
return null;
}
@Override
public DataSourceProcessor create() {
return null;
}
};
String other = "{\"useSSL\": \"true\"}";
Map<String, String> stringStringMap = abstractDataSourceProcessor.transformOtherParamToMap(other);
assertEquals("true", stringStringMap.get("useSSL"));
}
}

7
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/db2/Db2DataSourceProcessorTest.java
View File

@ -36,8 +36,6 @@ import org.powermock.api.mockito.PowerMockito;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
import com.google.common.collect.ImmutableMap;
@RunWith(PowerMockRunner.class)
@PrepareForTest({Class.class, DriverManager.class, DataSourceUtils.class, CommonUtils.class, DataSourceClientProvider.class, PasswordUtils.class})
public class Db2DataSourceProcessorTest {
@ -85,10 +83,7 @@ public class Db2DataSourceProcessorTest {
public void testGetJdbcUrl() {
Db2ConnectionParam db2ConnectionParam = new Db2ConnectionParam();
db2ConnectionParam.setJdbcUrl("jdbc:db2://localhost:5142/default");
ImmutableMap<String, String> map = new ImmutableMap.Builder<String, String>()
.put("other", "other")
.build();
db2ConnectionParam.setOther(map);
db2ConnectionParam.setOther("other=other");
String jdbcUrl = db2DatasourceProcessor.getJdbcUrl(db2ConnectionParam);
Assert.assertEquals("jdbc:db2://localhost:5142/default;other=other", jdbcUrl);
}

9
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/oracle/OracleDataSourceProcessorTest.java
View File

@ -37,8 +37,6 @@ import org.powermock.api.mockito.PowerMockito;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
import com.google.common.collect.ImmutableMap;
@RunWith(PowerMockRunner.class)
@PrepareForTest({Class.class, DriverManager.class, DataSourceUtils.class, CommonUtils.class, DataSourceClientProvider.class, PasswordUtils.class})
public class OracleDataSourceProcessorTest {
@ -85,11 +83,8 @@ public class OracleDataSourceProcessorTest {
public void testGetJdbcUrl() {
OracleConnectionParam oracleConnectionParam = new OracleConnectionParam();
oracleConnectionParam.setJdbcUrl("jdbc:oracle:thin:@localhost:3308:default");
ImmutableMap<String, String> map = new ImmutableMap.Builder<String, String>()
.put("other", "other")
.build();
oracleConnectionParam.setOther(map);
Assertions.assertEquals("jdbc:oracle:thin:@localhost:3308:default?other=other",
oracleConnectionParam.setOther("other=other");
Assert.assertEquals("jdbc:oracle:thin:@localhost:3308:default?other=other",
oracleDatasourceProcessor.getJdbcUrl(oracleConnectionParam));
}

9
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/postgresql/PostgreSQLDataSourceProcessorTest.java
View File

@ -36,8 +36,6 @@ import org.powermock.api.mockito.PowerMockito;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
import com.google.common.collect.ImmutableMap;
@RunWith(PowerMockRunner.class)
@PrepareForTest({Class.class, DriverManager.class, DataSourceUtils.class, CommonUtils.class, DataSourceClientProvider.class, PasswordUtils.class})
public class PostgreSQLDataSourceProcessorTest {
@ -83,13 +81,10 @@ public class PostgreSQLDataSourceProcessorTest {
public void testGetJdbcUrl() {
PostgreSQLConnectionParam postgreSqlConnectionParam = new PostgreSQLConnectionParam();
postgreSqlConnectionParam.setJdbcUrl("jdbc:postgresql://localhost:3308/default");
ImmutableMap<String, String> map = new ImmutableMap.Builder<String, String>()
.put("other", "other")
.build();
postgreSqlConnectionParam.setOther(map);
postgreSqlConnectionParam.setOther("other");
String jdbcUrl = postgreSqlDatasourceProcessor.getJdbcUrl(postgreSqlConnectionParam);
Assertions.assertEquals("jdbc:postgresql://localhost:3308/default?other=other", jdbcUrl);
Assert.assertEquals("jdbc:postgresql://localhost:3308/default?other", jdbcUrl);
}

3
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/presto/PrestoDataSourceProcessorTest.java
View File

@ -80,7 +80,8 @@ public class PrestoDataSourceProcessorTest {
public void testGetJdbcUrl() {
PrestoConnectionParam prestoConnectionParam = new PrestoConnectionParam();
prestoConnectionParam.setJdbcUrl("jdbc:postgresql://localhost:1234/default");
Assertions.assertEquals("jdbc:postgresql://localhost:1234/default",
prestoConnectionParam.setOther("other");
Assert.assertEquals("jdbc:postgresql://localhost:1234/default?other",
prestoDatasourceProcessor.getJdbcUrl(prestoConnectionParam));
}

12
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/redshift/RedshiftDataSourceProcessorTest.java
View File

@ -36,8 +36,6 @@ import org.powermock.api.mockito.PowerMockito;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
import com.google.common.collect.ImmutableMap;
@RunWith(PowerMockRunner.class)
@PrepareForTest({Class.class, DriverManager.class, DataSourceUtils.class, CommonUtils.class, DataSourceClientProvider.class, PasswordUtils.class})
public class RedshiftDataSourceProcessorTest {
@ -82,13 +80,9 @@ public class RedshiftDataSourceProcessorTest {
public void testGetJdbcUrl() {
RedshiftConnectionParam redshiftConnectionParam = new RedshiftConnectionParam();
redshiftConnectionParam.setJdbcUrl("jdbc:redshift://localhost:5439/default");
ImmutableMap<String, String> map = new ImmutableMap.Builder<String, String>()
.put("DSILogLevel", "6")
.put("defaultRowFetchSize", "100")
.build();
redshiftConnectionParam.setOther(map);
Assertions.assertEquals("jdbc:redshift://localhost:5439/default?DSILogLevel=6;defaultRowFetchSize=100",
redshiftDatasourceProcessor.getJdbcUrl(redshiftConnectionParam));
redshiftConnectionParam.setOther("DSILogLevel=6;defaultRowFetchSize=100");
Assert.assertEquals("jdbc:redshift://localhost:5439/default?DSILogLevel=6;defaultRowFetchSize=100",
redshiftDatasourceProcessor.getJdbcUrl(redshiftConnectionParam));
}

3
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/spark/SparkDataSourceProcessorTest.java
View File

@ -82,7 +82,8 @@ public class SparkDataSourceProcessorTest {
public void testGetJdbcUrl() {
SparkConnectionParam sparkConnectionParam = new SparkConnectionParam();
sparkConnectionParam.setJdbcUrl("jdbc:hive2://localhost1:1234,localhost2:1234/default");
Assertions.assertEquals("jdbc:hive2://localhost1:1234,localhost2:1234/default",
sparkConnectionParam.setOther("other");
Assert.assertEquals("jdbc:hive2://localhost1:1234,localhost2:1234/default;other",
sparkDatasourceProcessor.getJdbcUrl(sparkConnectionParam));
}

87
dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/datasource/BaseConnectionParam.java
View File

@ -17,14 +17,12 @@
package org.apache.dolphinscheduler.spi.datasource;
import java.util.HashMap;
import java.util.Map;
import lombok.Data;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonInclude.Include;
@Data
@JsonInclude(Include.NON_NULL)
public abstract class BaseConnectionParam implements ConnectionParam {
@ -44,6 +42,87 @@ public abstract class BaseConnectionParam implements ConnectionParam {
protected String validationQuery;
protected Map<String, String> other;
protected String other;
private Map<String, String> props = new HashMap<>();
public String getUser() {
return user;
}
public void setUser(String user) {
this.user = user;
}
public String getPassword() {
return password;
}
public void setPassword(String password) {
this.password = password;
}
public String getAddress() {
return address;
}
public void setAddress(String address) {
this.address = address;
}
public String getDatabase() {
return database;
}
public void setDatabase(String database) {
this.database = database;
}
public String getJdbcUrl() {
return jdbcUrl;
}
public void setJdbcUrl(String jdbcUrl) {
this.jdbcUrl = jdbcUrl;
}
public String getDriverLocation() {
return driverLocation;
}
public void setDriverLocation(String driverLocation) {
this.driverLocation = driverLocation;
}
public String getDriverClassName() {
return driverClassName;
}
public void setDriverClassName(String driverClassName) {
this.driverClassName = driverClassName;
}
public String getValidationQuery() {
return validationQuery;
}
public void setValidationQuery(String validationQuery) {
this.validationQuery = validationQuery;
}
public String getOther() {
return other;
}
public void setOther(String other) {
this.other = other;
}
public Map<String, String> getProps() {
return props;
}
public void setProps(Map<String, String> props) {
this.props = props;
}
}

10
dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/test/java/org/apache/dolphinscheduler/plugin/task/dq/DataQualityTaskTest.java
View File

@ -494,7 +494,7 @@ public class DataQualityTaskTest {
+ "\"jdbcUrl\":\"jdbc:postgresql://localhost:5432/dolphinscheduler\","
+ "\"user\":\"test\","
+ "\"password\":\"test\","
+ "\"other\":{\"stringtype\": \"unspecified\", \"characterEncoding\" : \"UTF-8\", \"allowMultiQueries\": true}}");
+ "\"other\":\"stringtype=unspecified&characterEncoding=UTF-8&allowMultiQueries=true\"}");
dataQualityTaskExecutionContext.setStatisticsValueConnectorType("JDBC");
dataQualityTaskExecutionContext.setStatisticsValueType(1);
@ -505,7 +505,7 @@ public class DataQualityTaskTest {
+ "\"jdbcUrl\":\"jdbc:postgresql://localhost:5432/dolphinscheduler\","
+ "\"user\":\"test\","
+ "\"password\":\"test\","
+ "\"other\":{\"stringtype\": \"unspecified\", \"characterEncoding\" : \"UTF-8\", \"allowMultiQueries\": true}}");
+ "\"other\":\"stringtype=unspecified&characterEncoding=UTF-8&allowMultiQueries=true\"}");
dataQualityTaskExecutionContext.setCompareWithFixedValue(true);
@ -791,7 +791,7 @@ public class DataQualityTaskTest {
+ "\"jdbcUrl\":\"jdbc:postgresql://localhost:5432/dolphinscheduler\","
+ "\"user\":\"test\","
+ "\"password\":\"test\","
+ "\"other\":{\"stringtype\": \"unspecified\", \"characterEncoding\": \"UTF-8\", \"allowMultiQueries\": true}}");
+ "\"other\":\"stringtype=unspecified&characterEncoding=UTF-8&allowMultiQueries=true\"}");
String expect = "{\"name\":\"跨表值比对\",\"env\":{\"type\":\"batch\",\"config\":null},\"readers\""
+ ":[{\"type\":\"JDBC\",\"config\":{\"database\":\"test\",\"password\":\"test\",\"driver\":"
@ -1115,7 +1115,7 @@ public class DataQualityTaskTest {
+ "\"jdbcUrl\":\"jdbc:postgresql://localhost:5432/dolphinscheduler\","
+ "\"user\":\"test\","
+ "\"password\":\"test\","
+ "\"other\":{\"stringtype\": \"unspecified\", \"characterEncoding\": \"UTF-8\", \"allowMultiQueries\": true}}");
+ "\"other\":\"stringtype=unspecified&characterEncoding=UTF-8&allowMultiQueries=true\"}");
dataQualityTaskExecutionContext.setStatisticsValueConnectorType("JDBC");
dataQualityTaskExecutionContext.setStatisticsValueType(1);
@ -1126,7 +1126,7 @@ public class DataQualityTaskTest {
+ "\"jdbcUrl\":\"jdbc:postgresql://localhost:5432/dolphinscheduler\","
+ "\"user\":\"test\","
+ "\"password\":\"test\","
+ "\"other\":{\"stringtype\": \"unspecified\", \"characterEncoding\": \"UTF-8\", \"allowMultiQueries\": true }}");
+ "\"other\":\"stringtype=unspecified&characterEncoding=UTF-8&allowMultiQueries=true\"}");
dataQualityTaskExecutionContext.setRuleName("跨表准确性");
dataQualityTaskExecutionContext.setRuleType(RuleType.MULTI_TABLE_ACCURACY.getCode());