natas_hw
/
dolphinscheduler
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[Bug][Worker] Global and local parameters are not replaced in SQL components (#14926) 

Signed-off-by: Gallardot <gallardot@apache.org>
This commit is contained in:
Gallardot 2023-09-15 10:31:14 +08:00 committed by GitHub
parent 73e846d03e
commit f6a6794289
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 53 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/AbstractTask.java
View File

@ -38,7 +38,9 @@ public abstract class AbstractTask {
protected final Logger log = LoggerFactory.getLogger(AbstractTask.class);
public String rgex = "['\"]\\$\\{(.*?)}['\"]|\\$\\{(.*?)}";
private static String groupName1 = "paramName1";
private static String groupName2 = "paramName2";
public String rgex = String.format("['\"]\\$\\{(?<%s>.*?)}['\"]|\\$\\{(?<%s>.*?)}", groupName1, groupName2);
/**
* varPool string
@ -198,7 +200,11 @@ public abstract class AbstractTask {
int index = 1;
while (m.find()) {
String paramName = m.group(1);
String paramName = m.group(groupName1);
if (paramName == null) {
paramName = m.group(groupName2);
}
Property prop = paramsPropsMap.get(paramName);
if (prop == null) {

45
dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/test/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTaskTest.java
View File

@ -18,11 +18,17 @@
package org.apache.dolphinscheduler.plugin.task.sql;
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
import org.apache.dolphinscheduler.plugin.task.api.enums.DataType;
import org.apache.dolphinscheduler.plugin.task.api.enums.Direct;
import org.apache.dolphinscheduler.plugin.task.api.enums.ResourceType;
import org.apache.dolphinscheduler.plugin.task.api.model.Property;
import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.DataSourceParameters;
import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper;
import org.apache.dolphinscheduler.spi.enums.DbType;
import java.util.HashMap;
import java.util.Map;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@ -66,6 +72,15 @@ class SqlTaskTest {
String hiveLoadSql = "load inpath '/tmp/test_table/dt=${dt}' into table test_table partition(dt=${dt})";
String expected = "load inpath '/tmp/test_table/dt=?' into table test_table partition(dt=?)";
Assertions.assertEquals(expected, hiveLoadSql.replaceAll(sqlTask.rgex, "?"));
Map<Integer, Property> sqlParamsMap = new HashMap<>();
Map<Integer, Property> expectedSQLParamsMap = new HashMap<>();
expectedSQLParamsMap.put(1, new Property("dt", Direct.IN, DataType.VARCHAR, "1970"));
expectedSQLParamsMap.put(2, new Property("dt", Direct.IN, DataType.VARCHAR, "1970"));
Map<String, Property> paramsMap = new HashMap<>();
paramsMap.put("dt", new Property("dt", Direct.IN, DataType.VARCHAR, "1970"));
sqlTask.setSqlParamsMap(hiveLoadSql, sqlTask.rgex, sqlParamsMap, paramsMap, 1);
Assertions.assertEquals(sqlParamsMap, expectedSQLParamsMap);
}
@Test
@ -74,17 +89,37 @@ class SqlTaskTest {
String expected = "select id from student where dt=?";
Assertions.assertEquals(expected, querySql.replaceAll(sqlTask.rgex, "?"));
Map<Integer, Property> sqlParamsMap = new HashMap<>();
Map<Integer, Property> expectedSQLParamsMap = new HashMap<>();
expectedSQLParamsMap.put(1, new Property("dt", Direct.IN, DataType.VARCHAR, "1970"));
Map<String, Property> paramsMap = new HashMap<>();
paramsMap.put("dt", new Property("dt", Direct.IN, DataType.VARCHAR, "1970"));
sqlTask.setSqlParamsMap(querySql, sqlTask.rgex, sqlParamsMap, paramsMap, 1);
Assertions.assertEquals(sqlParamsMap, expectedSQLParamsMap);
querySql = "select id from student where dt=\"${dt}\"";
expected = "select id from student where dt=?";
Assertions.assertEquals(expected, querySql.replaceAll(sqlTask.rgex, "?"));
sqlParamsMap.clear();
sqlTask.setSqlParamsMap(querySql, sqlTask.rgex, sqlParamsMap, paramsMap, 1);
Assertions.assertEquals(sqlParamsMap, expectedSQLParamsMap);
querySql = "select id from student where dt=${dt}";
expected = "select id from student where dt=?";
Assertions.assertEquals(expected, querySql.replaceAll(sqlTask.rgex, "?"));
sqlParamsMap.clear();
sqlTask.setSqlParamsMap(querySql, sqlTask.rgex, sqlParamsMap, paramsMap, 1);
Assertions.assertEquals(sqlParamsMap, expectedSQLParamsMap);
querySql = "select id from student where dt=${dt} and gender=1";
expected = "select id from student where dt=? and gender=1";
Assertions.assertEquals(expected, querySql.replaceAll(sqlTask.rgex, "?"));
sqlParamsMap.clear();
sqlTask.setSqlParamsMap(querySql, sqlTask.rgex, sqlParamsMap, paramsMap, 1);
Assertions.assertEquals(sqlParamsMap, expectedSQLParamsMap);
}
@Test
@ -92,5 +127,15 @@ class SqlTaskTest {
String querySql = "select id from student where year=${year} and month=${month} and gender=1";
String expected = "select id from student where year=? and month=? and gender=1";
Assertions.assertEquals(expected, querySql.replaceAll(sqlTask.rgex, "?"));
Map<Integer, Property> sqlParamsMap = new HashMap<>();
Map<Integer, Property> expectedSQLParamsMap = new HashMap<>();
expectedSQLParamsMap.put(1, new Property("year", Direct.IN, DataType.VARCHAR, "1970"));
expectedSQLParamsMap.put(2, new Property("month", Direct.IN, DataType.VARCHAR, "12"));
Map<String, Property> paramsMap = new HashMap<>();
paramsMap.put("year", new Property("year", Direct.IN, DataType.VARCHAR, "1970"));
paramsMap.put("month", new Property("month", Direct.IN, DataType.VARCHAR, "12"));
sqlTask.setSqlParamsMap(querySql, sqlTask.rgex, sqlParamsMap, paramsMap, 1);
Assertions.assertEquals(sqlParamsMap, expectedSQLParamsMap);
}
}