From 40cb4e1b1ced602b1905d6bc757e0842689e8c24 Mon Sep 17 00:00:00 2001 From: xxzuo <1293378490@qq.com> Date: Wed, 17 Apr 2024 19:09:37 +0800 Subject: [PATCH 1/6] [Fix-15828] Replace the original question mark before regex replacement --- .../plugin/task/api/utils/ParameterUtils.java | 19 +++++++++++++ .../task/api/utils/ParameterUtilsTest.java | 11 ++++++++ .../plugin/task/sql/SqlTask.java | 5 ++-- .../plugin/task/sql/SqlTaskTest.java | 27 +++++++++++++++++++ 4 files changed, 60 insertions(+), 2 deletions(-) diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/ParameterUtils.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/ParameterUtils.java index ea334125d5..a5578b886c 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/ParameterUtils.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/ParameterUtils.java @@ -52,6 +52,8 @@ public class ParameterUtils { private static final char PARAM_REPLACE_CHAR = '?'; + private static final String UNIQUE_QUESTION_MARK = "@@QUESTION_UNIQUE_MARK@@"; + private ParameterUtils() { throw new UnsupportedOperationException("Construct ParameterUtils"); } @@ -342,4 +344,21 @@ public class ParameterUtils { return userDefParamsMaps; } + /** + * use unique mark replace "?" before process + * @param sql sql + * @return sql without "?" + */ + public static String replaceSqlQuestionMark(String sql){ + return sql.replaceAll("\\?", UNIQUE_QUESTION_MARK); + } + + /** + * use "?" replace unique mark after process + * @param sql sql + * @return recovered sql + */ + public static String recoverSqlQuestionMark(String sql){ + return sql.replaceAll(UNIQUE_QUESTION_MARK, "?"); + } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/utils/ParameterUtilsTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/utils/ParameterUtilsTest.java index 4297e36fcc..27250c68c6 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/utils/ParameterUtilsTest.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/utils/ParameterUtilsTest.java @@ -113,4 +113,15 @@ public class ParameterUtilsTest { Assertions.assertEquals("test Parameter", ParameterUtils.handleEscapes("test Parameter")); Assertions.assertEquals("////%test////%Parameter", ParameterUtils.handleEscapes("%test%Parameter")); } + + @Test + public void testProcessSqlQuestionMark(){ + String querySql = "select id, concat('?', name) from test"; + String expected = "select id, concat('@@QUESTION_UNIQUE_MARK@@', name) from test"; + Assertions.assertEquals(ParameterUtils.replaceSqlQuestionMark(querySql), expected); + + querySql = "select id, concat('@@QUESTION_UNIQUE_MARK@@', name) from test"; + expected = "select id, concat('?', name) from test"; + Assertions.assertEquals(ParameterUtils.recoverSqlQuestionMark(querySql), expected); + } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTask.java index 1c8fd4519f..6ee3d80e82 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTask.java @@ -454,9 +454,10 @@ public class SqlTask extends AbstractTask { String rgexo = "['\"]*\\!\\{(.*?)\\}['\"]*"; sql = replaceOriginalValue(sql, rgexo, paramsMap); // replace the ${} of the SQL statement with the Placeholder - String formatSql = sql.replaceAll(rgex, "?"); + // Convert the original question mark to another symbol in advance, and restore the question mark later + String formatSql = ParameterUtils.replaceSqlQuestionMark(sql).replaceAll(rgex, "?"); // Convert the list parameter - formatSql = ParameterUtils.expandListParameter(sqlParamsMap, formatSql); + formatSql = ParameterUtils.recoverSqlQuestionMark(ParameterUtils.expandListParameter(sqlParamsMap, formatSql)); sqlBuilder.append(formatSql); // print replace sql printReplacedSql(sql, formatSql, rgex, sqlParamsMap); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/test/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTaskTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/test/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTaskTest.java index 268e28db76..34d7ac3319 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/test/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTaskTest.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/test/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTaskTest.java @@ -24,6 +24,7 @@ import org.apache.dolphinscheduler.plugin.task.api.enums.ResourceType; import org.apache.dolphinscheduler.plugin.task.api.model.Property; import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.DataSourceParameters; import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.dolphinscheduler.spi.enums.DbType; import java.util.HashMap; @@ -142,4 +143,30 @@ class SqlTaskTest { @Test void splitSql() { } + + @Test + void testReplacingSqlHasQuestionMarkAndParams(){ + String querySql = "select id, concat('?', year) from student where year=${year} and month=${month} and gender=1"; + String expected = "select id, concat('@@QUESTION_UNIQUE_MARK@@', year) from student where year=? and month=? and gender=1"; + Assertions.assertEquals(expected, ParameterUtils.replaceSqlQuestionMark(querySql).replaceAll(sqlTask.rgex, "?")); + + Map sqlParamsMap = new HashMap<>(); + Map expectedSQLParamsMap = new HashMap<>(); + expectedSQLParamsMap.put(1, new Property("year", Direct.IN, DataType.VARCHAR, "1970")); + expectedSQLParamsMap.put(2, new Property("month", Direct.IN, DataType.VARCHAR, "12")); + Map paramsMap = new HashMap<>(); + paramsMap.put("year", new Property("year", Direct.IN, DataType.VARCHAR, "1970")); + paramsMap.put("month", new Property("month", Direct.IN, DataType.VARCHAR, "12")); + sqlTask.setSqlParamsMap(ParameterUtils.replaceSqlQuestionMark(querySql), sqlTask.rgex, sqlParamsMap, paramsMap, 1); + Assertions.assertEquals(sqlParamsMap, expectedSQLParamsMap); + + String formatSql = ParameterUtils.replaceSqlQuestionMark(querySql).replaceAll(sqlTask.rgex, "?"); + formatSql = ParameterUtils.expandListParameter(sqlParamsMap, formatSql); + expected = "select id, concat('@@QUESTION_UNIQUE_MARK@@', year) from student where year=? and month=? and gender=1"; + Assertions.assertEquals(expected, formatSql); + + formatSql = ParameterUtils.recoverSqlQuestionMark(formatSql); + expected = "select id, concat('?', year) from student where year=? and month=? and gender=1"; + Assertions.assertEquals(expected, formatSql); + } } From 20e731c56044b03a0d8495382fdb8ddec95ad6de Mon Sep 17 00:00:00 2001 From: xxzuo <1293378490@qq.com> Date: Thu, 18 Apr 2024 09:30:27 +0800 Subject: [PATCH 2/6] fix style --- .../plugin/task/api/utils/ParameterUtils.java | 4 ++-- .../task/api/utils/ParameterUtilsTest.java | 2 +- .../plugin/task/sql/SqlTaskTest.java | 17 +++++++++++------ 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/ParameterUtils.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/ParameterUtils.java index a5578b886c..a0471757e6 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/ParameterUtils.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/ParameterUtils.java @@ -349,7 +349,7 @@ public class ParameterUtils { * @param sql sql * @return sql without "?" */ - public static String replaceSqlQuestionMark(String sql){ + public static String replaceSqlQuestionMark(String sql) { return sql.replaceAll("\\?", UNIQUE_QUESTION_MARK); } @@ -358,7 +358,7 @@ public class ParameterUtils { * @param sql sql * @return recovered sql */ - public static String recoverSqlQuestionMark(String sql){ + public static String recoverSqlQuestionMark(String sql) { return sql.replaceAll(UNIQUE_QUESTION_MARK, "?"); } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/utils/ParameterUtilsTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/utils/ParameterUtilsTest.java index 27250c68c6..83671a3500 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/utils/ParameterUtilsTest.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/utils/ParameterUtilsTest.java @@ -115,7 +115,7 @@ public class ParameterUtilsTest { } @Test - public void testProcessSqlQuestionMark(){ + public void testProcessSqlQuestionMark() { String querySql = "select id, concat('?', name) from test"; String expected = "select id, concat('@@QUESTION_UNIQUE_MARK@@', name) from test"; Assertions.assertEquals(ParameterUtils.replaceSqlQuestionMark(querySql), expected); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/test/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTaskTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/test/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTaskTest.java index 34d7ac3319..57d166eb1a 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/test/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTaskTest.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/test/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTaskTest.java @@ -145,10 +145,13 @@ class SqlTaskTest { } @Test - void testReplacingSqlHasQuestionMarkAndParams(){ - String querySql = "select id, concat('?', year) from student where year=${year} and month=${month} and gender=1"; - String expected = "select id, concat('@@QUESTION_UNIQUE_MARK@@', year) from student where year=? and month=? and gender=1"; - Assertions.assertEquals(expected, ParameterUtils.replaceSqlQuestionMark(querySql).replaceAll(sqlTask.rgex, "?")); + void testReplacingSqlHasQuestionMarkAndParams() { + String querySql = + "select id, concat('?', year) from student where year=${year} and month=${month} and gender=1"; + String expected = + "select id, concat('@@QUESTION_UNIQUE_MARK@@', year) from student where year=? and month=? and gender=1"; + Assertions.assertEquals(expected, + ParameterUtils.replaceSqlQuestionMark(querySql).replaceAll(sqlTask.rgex, "?")); Map sqlParamsMap = new HashMap<>(); Map expectedSQLParamsMap = new HashMap<>(); @@ -157,12 +160,14 @@ class SqlTaskTest { Map paramsMap = new HashMap<>(); paramsMap.put("year", new Property("year", Direct.IN, DataType.VARCHAR, "1970")); paramsMap.put("month", new Property("month", Direct.IN, DataType.VARCHAR, "12")); - sqlTask.setSqlParamsMap(ParameterUtils.replaceSqlQuestionMark(querySql), sqlTask.rgex, sqlParamsMap, paramsMap, 1); + sqlTask.setSqlParamsMap(ParameterUtils.replaceSqlQuestionMark(querySql), sqlTask.rgex, sqlParamsMap, paramsMap, + 1); Assertions.assertEquals(sqlParamsMap, expectedSQLParamsMap); String formatSql = ParameterUtils.replaceSqlQuestionMark(querySql).replaceAll(sqlTask.rgex, "?"); formatSql = ParameterUtils.expandListParameter(sqlParamsMap, formatSql); - expected = "select id, concat('@@QUESTION_UNIQUE_MARK@@', year) from student where year=? and month=? and gender=1"; + expected = + "select id, concat('@@QUESTION_UNIQUE_MARK@@', year) from student where year=? and month=? and gender=1"; Assertions.assertEquals(expected, formatSql); formatSql = ParameterUtils.recoverSqlQuestionMark(formatSql); From 9503d6ca2e32f3455ae6bae6342f618d56b87d00 Mon Sep 17 00:00:00 2001 From: xxzuo <1293378490@qq.com> Date: Thu, 18 Apr 2024 12:34:17 +0800 Subject: [PATCH 3/6] This unique mark uses a constant --- .../plugin/task/api/utils/ParameterUtils.java | 2 +- .../plugin/task/api/utils/ParameterUtilsTest.java | 4 ++-- .../dolphinscheduler/plugin/task/sql/SqlTaskTest.java | 6 ++++-- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/ParameterUtils.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/ParameterUtils.java index a0471757e6..e2de3b120c 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/ParameterUtils.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/ParameterUtils.java @@ -52,7 +52,7 @@ public class ParameterUtils { private static final char PARAM_REPLACE_CHAR = '?'; - private static final String UNIQUE_QUESTION_MARK = "@@QUESTION_UNIQUE_MARK@@"; + public static final String UNIQUE_QUESTION_MARK = "@@QUESTION_UNIQUE_MARK@@"; private ParameterUtils() { throw new UnsupportedOperationException("Construct ParameterUtils"); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/utils/ParameterUtilsTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/utils/ParameterUtilsTest.java index 83671a3500..5123c09194 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/utils/ParameterUtilsTest.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/utils/ParameterUtilsTest.java @@ -117,10 +117,10 @@ public class ParameterUtilsTest { @Test public void testProcessSqlQuestionMark() { String querySql = "select id, concat('?', name) from test"; - String expected = "select id, concat('@@QUESTION_UNIQUE_MARK@@', name) from test"; + String expected = "select id, concat('" + ParameterUtils.UNIQUE_QUESTION_MARK + "', name) from test"; Assertions.assertEquals(ParameterUtils.replaceSqlQuestionMark(querySql), expected); - querySql = "select id, concat('@@QUESTION_UNIQUE_MARK@@', name) from test"; + querySql = "select id, concat('" + ParameterUtils.UNIQUE_QUESTION_MARK + "', name) from test"; expected = "select id, concat('?', name) from test"; Assertions.assertEquals(ParameterUtils.recoverSqlQuestionMark(querySql), expected); } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/test/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTaskTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/test/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTaskTest.java index 57d166eb1a..9f639dd0bc 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/test/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTaskTest.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/test/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTaskTest.java @@ -149,7 +149,8 @@ class SqlTaskTest { String querySql = "select id, concat('?', year) from student where year=${year} and month=${month} and gender=1"; String expected = - "select id, concat('@@QUESTION_UNIQUE_MARK@@', year) from student where year=? and month=? and gender=1"; + "select id, concat('" + ParameterUtils.UNIQUE_QUESTION_MARK + + "', year) from student where year=? and month=? and gender=1"; Assertions.assertEquals(expected, ParameterUtils.replaceSqlQuestionMark(querySql).replaceAll(sqlTask.rgex, "?")); @@ -167,7 +168,8 @@ class SqlTaskTest { String formatSql = ParameterUtils.replaceSqlQuestionMark(querySql).replaceAll(sqlTask.rgex, "?"); formatSql = ParameterUtils.expandListParameter(sqlParamsMap, formatSql); expected = - "select id, concat('@@QUESTION_UNIQUE_MARK@@', year) from student where year=? and month=? and gender=1"; + "select id, concat('" + ParameterUtils.UNIQUE_QUESTION_MARK + + "', year) from student where year=? and month=? and gender=1"; Assertions.assertEquals(expected, formatSql); formatSql = ParameterUtils.recoverSqlQuestionMark(formatSql); From f36afa06e327010efbc6efd21f0a69bf8345824d Mon Sep 17 00:00:00 2001 From: xxzuo <1293378490@qq.com> Date: Thu, 18 Apr 2024 23:17:20 +0800 Subject: [PATCH 4/6] [Fix-15828] fix expandListParameter --- .../plugin/task/api/utils/ParameterUtils.java | 54 +++++++------------ .../task/api/utils/ParameterUtilsTest.java | 33 ++++++------ .../plugin/task/sql/SqlTask.java | 5 +- .../plugin/task/sql/SqlTaskTest.java | 28 +++++----- 4 files changed, 48 insertions(+), 72 deletions(-) diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/ParameterUtils.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/ParameterUtils.java index e2de3b120c..283bd566a6 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/ParameterUtils.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/ParameterUtils.java @@ -52,8 +52,6 @@ public class ParameterUtils { private static final char PARAM_REPLACE_CHAR = '?'; - public static final String UNIQUE_QUESTION_MARK = "@@QUESTION_UNIQUE_MARK@@"; - private ParameterUtils() { throw new UnsupportedOperationException("Construct ParameterUtils"); } @@ -189,29 +187,33 @@ public class ParameterUtils { return property != null && DataType.BOOLEAN.equals(property.getType()); } - public static String expandListParameter(Map params, String sql) { + public static String expandListParameter(Map params, String sql, String regex) { Map expandMap = new HashMap<>(); if (params == null || params.isEmpty()) { return sql; } - String[] split = sql.split("\\?"); - if (split.length == 0) { - return sql; - } - StringBuilder ret = new StringBuilder(split[0]); + StringBuilder ret = new StringBuilder(sql); + Pattern pattern = Pattern.compile(regex); + Matcher m = pattern.matcher(sql); int index = 1; - for (int i = 1; i < split.length; i++) { - Property property = params.get(i); + int paramsIndex = 1; + // When matching with a regex, determine whether the corresponding property is a list. + while (m.find()) { + Property property = params.get(paramsIndex++); + if (property == null) { + continue; + } String value = property.getValue(); + StringBuilder tempReplace = new StringBuilder(); if (DataType.LIST.equals(property.getType())) { List valueList = JSONUtils.toList(value, Object.class); if (valueList.isEmpty() && StringUtils.isNotBlank(value)) { valueList.add(value); } for (int j = 0; j < valueList.size(); j++) { - ret.append(PARAM_REPLACE_CHAR); + tempReplace.append(PARAM_REPLACE_CHAR); if (j != valueList.size() - 1) { - ret.append(","); + tempReplace.append(","); } } for (Object v : valueList) { @@ -233,14 +235,12 @@ public class ParameterUtils { expandMap.put(index++, newProperty); } } else { - ret.append(PARAM_REPLACE_CHAR); + tempReplace.append(PARAM_REPLACE_CHAR); expandMap.put(index++, property); } - ret.append(split[i]); - } - if (PARAM_REPLACE_CHAR == sql.charAt(sql.length() - 1)) { - ret.append(PARAM_REPLACE_CHAR); - expandMap.put(index, params.get(split.length)); + ret.replace(m.start(), m.end(), tempReplace.toString()); + // After replacement, the string length will change, so a reset is required + m.reset(ret.toString()); } params.clear(); params.putAll(expandMap); @@ -343,22 +343,4 @@ public class ParameterUtils { } return userDefParamsMaps; } - - /** - * use unique mark replace "?" before process - * @param sql sql - * @return sql without "?" - */ - public static String replaceSqlQuestionMark(String sql) { - return sql.replaceAll("\\?", UNIQUE_QUESTION_MARK); - } - - /** - * use "?" replace unique mark after process - * @param sql sql - * @return recovered sql - */ - public static String recoverSqlQuestionMark(String sql) { - return sql.replaceAll(UNIQUE_QUESTION_MARK, "?"); - } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/utils/ParameterUtilsTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/utils/ParameterUtilsTest.java index 5123c09194..3f26f95959 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/utils/ParameterUtilsTest.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/utils/ParameterUtilsTest.java @@ -23,6 +23,7 @@ import org.apache.dolphinscheduler.common.constants.DateConstants; import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.plugin.task.api.enums.DataType; +import org.apache.dolphinscheduler.plugin.task.api.enums.Direct; import org.apache.dolphinscheduler.plugin.task.api.model.Property; import org.apache.dolphinscheduler.plugin.task.api.parser.PlaceholderUtils; @@ -38,23 +39,31 @@ import com.google.common.collect.Lists; public class ParameterUtilsTest { + private String groupName1 = "paramName1"; + private String groupName2 = "paramName2"; + private String rgex = String.format("['\"]\\$\\{(?<%s>.*?)}['\"]|\\$\\{(?<%s>.*?)}", groupName1, groupName2); + @Test public void expandListParameter() { + Map params = new HashMap<>(); params.put(1, - new Property(null, null, DataType.LIST, JSONUtils.toJsonString(Lists.newArrayList("c1", "c2", "c3")))); - params.put(2, new Property(null, null, DataType.DATE, "2020-06-30")); - params.put(3, new Property(null, null, DataType.LIST, + new Property("col1", Direct.IN, DataType.LIST, + JSONUtils.toJsonString(Lists.newArrayList("c1", "c2", "c3")))); + params.put(2, new Property("date", Direct.IN, DataType.DATE, "2020-06-30")); + params.put(3, new Property("col2", Direct.IN, DataType.LIST, JSONUtils.toJsonString(Lists.newArrayList(3.1415, 2.44, 3.44)))); String sql = ParameterUtils.expandListParameter(params, - "select * from test where col1 in (?) and date=? and col2 in (?)"); + "select * from test where col1 in ('${col1}') and date='${date}' and col2 in ('${col2}')", rgex); Assertions.assertEquals("select * from test where col1 in (?,?,?) and date=? and col2 in (?,?,?)", sql); Assertions.assertEquals(7, params.size()); Map params2 = new HashMap<>(); - params2.put(1, new Property(null, null, DataType.LIST, JSONUtils.toJsonString(Lists.newArrayList("c1")))); - params2.put(2, new Property(null, null, DataType.DATE, "2020-06-30")); - String sql2 = ParameterUtils.expandListParameter(params2, "select * from test where col1 in (?) and date=?"); + params2.put(1, + new Property("col1", Direct.IN, DataType.LIST, JSONUtils.toJsonString(Lists.newArrayList("c1")))); + params2.put(2, new Property("date", Direct.IN, DataType.DATE, "2020-06-30")); + String sql2 = ParameterUtils.expandListParameter(params2, + "select * from test where col1 in ('${col}') and date='${date}'", rgex); Assertions.assertEquals("select * from test where col1 in (?) and date=?", sql2); Assertions.assertEquals(2, params2.size()); @@ -114,14 +123,4 @@ public class ParameterUtilsTest { Assertions.assertEquals("////%test////%Parameter", ParameterUtils.handleEscapes("%test%Parameter")); } - @Test - public void testProcessSqlQuestionMark() { - String querySql = "select id, concat('?', name) from test"; - String expected = "select id, concat('" + ParameterUtils.UNIQUE_QUESTION_MARK + "', name) from test"; - Assertions.assertEquals(ParameterUtils.replaceSqlQuestionMark(querySql), expected); - - querySql = "select id, concat('" + ParameterUtils.UNIQUE_QUESTION_MARK + "', name) from test"; - expected = "select id, concat('?', name) from test"; - Assertions.assertEquals(ParameterUtils.recoverSqlQuestionMark(querySql), expected); - } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTask.java index 6ee3d80e82..cd904cafa5 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTask.java @@ -454,10 +454,9 @@ public class SqlTask extends AbstractTask { String rgexo = "['\"]*\\!\\{(.*?)\\}['\"]*"; sql = replaceOriginalValue(sql, rgexo, paramsMap); // replace the ${} of the SQL statement with the Placeholder - // Convert the original question mark to another symbol in advance, and restore the question mark later - String formatSql = ParameterUtils.replaceSqlQuestionMark(sql).replaceAll(rgex, "?"); + // String formatSql = sql.replaceAll(rgex, "?"); // Convert the list parameter - formatSql = ParameterUtils.recoverSqlQuestionMark(ParameterUtils.expandListParameter(sqlParamsMap, formatSql)); + String formatSql = ParameterUtils.expandListParameter(sqlParamsMap, sql, rgex); sqlBuilder.append(formatSql); // print replace sql printReplacedSql(sql, formatSql, rgex, sqlParamsMap); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/test/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTaskTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/test/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTaskTest.java index 9f639dd0bc..c672f0175d 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/test/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTaskTest.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/test/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTaskTest.java @@ -17,6 +17,7 @@ package org.apache.dolphinscheduler.plugin.task.sql; +import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.enums.DataType; import org.apache.dolphinscheduler.plugin.task.api.enums.Direct; @@ -34,6 +35,8 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import com.google.common.collect.Lists; + class SqlTaskTest { private SqlTask sqlTask; @@ -147,33 +150,26 @@ class SqlTaskTest { @Test void testReplacingSqlHasQuestionMarkAndParams() { String querySql = - "select id, concat('?', year) from student where year=${year} and month=${month} and gender=1"; + "select id, concat('?', year) from student where year=${year} and month=${month} and gender in ('${gender}')"; String expected = - "select id, concat('" + ParameterUtils.UNIQUE_QUESTION_MARK - + "', year) from student where year=? and month=? and gender=1"; - Assertions.assertEquals(expected, - ParameterUtils.replaceSqlQuestionMark(querySql).replaceAll(sqlTask.rgex, "?")); + "select id, concat('?', year) from student where year=? and month=? and gender in (?,?)"; Map sqlParamsMap = new HashMap<>(); Map expectedSQLParamsMap = new HashMap<>(); expectedSQLParamsMap.put(1, new Property("year", Direct.IN, DataType.VARCHAR, "1970")); expectedSQLParamsMap.put(2, new Property("month", Direct.IN, DataType.VARCHAR, "12")); + expectedSQLParamsMap.put(3, + new Property("gender", Direct.IN, DataType.LIST, JSONUtils.toJsonString(Lists.newArrayList(1, 2)))); Map paramsMap = new HashMap<>(); paramsMap.put("year", new Property("year", Direct.IN, DataType.VARCHAR, "1970")); paramsMap.put("month", new Property("month", Direct.IN, DataType.VARCHAR, "12")); - sqlTask.setSqlParamsMap(ParameterUtils.replaceSqlQuestionMark(querySql), sqlTask.rgex, sqlParamsMap, paramsMap, - 1); + paramsMap.put("gender", + new Property("gender", Direct.IN, DataType.LIST, JSONUtils.toJsonString(Lists.newArrayList(1, 2)))); + sqlTask.setSqlParamsMap(querySql, sqlTask.rgex, sqlParamsMap, paramsMap, 1); Assertions.assertEquals(sqlParamsMap, expectedSQLParamsMap); - String formatSql = ParameterUtils.replaceSqlQuestionMark(querySql).replaceAll(sqlTask.rgex, "?"); - formatSql = ParameterUtils.expandListParameter(sqlParamsMap, formatSql); - expected = - "select id, concat('" + ParameterUtils.UNIQUE_QUESTION_MARK - + "', year) from student where year=? and month=? and gender=1"; - Assertions.assertEquals(expected, formatSql); - - formatSql = ParameterUtils.recoverSqlQuestionMark(formatSql); - expected = "select id, concat('?', year) from student where year=? and month=? and gender=1"; + String formatSql = ParameterUtils.expandListParameter(sqlParamsMap, querySql, sqlTask.rgex); + Assertions.assertEquals(4, sqlParamsMap.size()); Assertions.assertEquals(expected, formatSql); } } From a05390117aa15a1d28c1425583c2611ce08e2aaa Mon Sep 17 00:00:00 2001 From: xxzuo <1293378490@qq.com> Date: Fri, 19 Apr 2024 20:37:18 +0800 Subject: [PATCH 5/6] [Fix-15828] extract this rgex into TaskConstants --- .../plugin/task/api/AbstractTask.java | 15 +++------ .../plugin/task/api/TaskConstants.java | 10 ++++++ .../plugin/task/api/utils/ParameterUtils.java | 10 ++++-- .../task/api/utils/ParameterUtilsTest.java | 8 ++--- .../plugin/task/procedure/ProcedureTask.java | 5 +-- .../plugin/task/sql/SqlTask.java | 7 ++-- .../plugin/task/sql/SqlTaskTest.java | 33 ++++++++++--------- 7 files changed, 46 insertions(+), 42 deletions(-) diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/AbstractTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/AbstractTask.java index 4437df763b..f9f7dc21c3 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/AbstractTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/AbstractTask.java @@ -26,7 +26,6 @@ import java.util.Map; import java.util.StringJoiner; import java.util.concurrent.LinkedBlockingQueue; import java.util.regex.Matcher; -import java.util.regex.Pattern; import lombok.Getter; import lombok.Setter; @@ -35,10 +34,6 @@ import lombok.extern.slf4j.Slf4j; @Slf4j public abstract class AbstractTask { - private static String groupName1 = "paramName1"; - private static String groupName2 = "paramName2"; - public String rgex = String.format("['\"]\\$\\{(?<%s>.*?)}['\"]|\\$\\{(?<%s>.*?)}", groupName1, groupName2); - @Getter @Setter protected Map taskOutputParams; @@ -173,24 +168,22 @@ public abstract class AbstractTask { * regular expressions match the contents between two specified strings * * @param content content - * @param rgex rgex * @param sqlParamsMap sql params map * @param paramsPropsMap params props map */ - public void setSqlParamsMap(String content, String rgex, Map sqlParamsMap, + public void setSqlParamsMap(String content, Map sqlParamsMap, Map paramsPropsMap, int taskInstanceId) { if (paramsPropsMap == null) { return; } - Pattern pattern = Pattern.compile(rgex); - Matcher m = pattern.matcher(content); + Matcher m = TaskConstants.SQL_PARAMS_PATTERN.matcher(content); int index = 1; while (m.find()) { - String paramName = m.group(groupName1); + String paramName = m.group(TaskConstants.GROUP_NAME1); if (paramName == null) { - paramName = m.group(groupName2); + paramName = m.group(TaskConstants.GROUP_NAME2); } Property prop = paramsPropsMap.get(paramName); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/TaskConstants.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/TaskConstants.java index 43734416e7..e0e3ea0c5d 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/TaskConstants.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/TaskConstants.java @@ -21,6 +21,7 @@ import org.apache.dolphinscheduler.common.constants.DateConstants; import java.time.Duration; import java.util.Set; +import java.util.regex.Pattern; import com.google.common.collect.Sets; @@ -439,4 +440,13 @@ public class TaskConstants { // Loop task constants public static final Duration DEFAULT_LOOP_STATUS_INTERVAL = Duration.ofSeconds(5L); + /** + * sql params regex + */ + public static final String GROUP_NAME1 = "paramName1"; + public static final String GROUP_NAME2 = "paramName2"; + public static final String SQL_PARAMS_REGEX = + String.format("['\"]\\$\\{(?<%s>.*?)}['\"]|\\$\\{(?<%s>.*?)}", GROUP_NAME1, GROUP_NAME2); + public static final Pattern SQL_PARAMS_PATTERN = Pattern.compile(SQL_PARAMS_REGEX); + } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/ParameterUtils.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/ParameterUtils.java index 283bd566a6..13e0ef4575 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/ParameterUtils.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/ParameterUtils.java @@ -52,6 +52,11 @@ public class ParameterUtils { private static final char PARAM_REPLACE_CHAR = '?'; + private static final String PARAM_REGEX = + String.format("['\"]\\$\\{(?<%s>.*?)}['\"]|\\$\\{(?<%s>.*?)}", "paramName1", "paramName2"); + + private static final Pattern PARAM_REGEX_PATTERN = Pattern.compile(PARAM_REGEX); + private ParameterUtils() { throw new UnsupportedOperationException("Construct ParameterUtils"); } @@ -187,14 +192,13 @@ public class ParameterUtils { return property != null && DataType.BOOLEAN.equals(property.getType()); } - public static String expandListParameter(Map params, String sql, String regex) { + public static String expandListParameter(Map params, String sql) { Map expandMap = new HashMap<>(); if (params == null || params.isEmpty()) { return sql; } StringBuilder ret = new StringBuilder(sql); - Pattern pattern = Pattern.compile(regex); - Matcher m = pattern.matcher(sql); + Matcher m = PARAM_REGEX_PATTERN.matcher(sql); int index = 1; int paramsIndex = 1; // When matching with a regex, determine whether the corresponding property is a list. diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/utils/ParameterUtilsTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/utils/ParameterUtilsTest.java index 3f26f95959..1c0dbc7c06 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/utils/ParameterUtilsTest.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/utils/ParameterUtilsTest.java @@ -39,10 +39,6 @@ import com.google.common.collect.Lists; public class ParameterUtilsTest { - private String groupName1 = "paramName1"; - private String groupName2 = "paramName2"; - private String rgex = String.format("['\"]\\$\\{(?<%s>.*?)}['\"]|\\$\\{(?<%s>.*?)}", groupName1, groupName2); - @Test public void expandListParameter() { @@ -54,7 +50,7 @@ public class ParameterUtilsTest { params.put(3, new Property("col2", Direct.IN, DataType.LIST, JSONUtils.toJsonString(Lists.newArrayList(3.1415, 2.44, 3.44)))); String sql = ParameterUtils.expandListParameter(params, - "select * from test where col1 in ('${col1}') and date='${date}' and col2 in ('${col2}')", rgex); + "select * from test where col1 in ('${col1}') and date='${date}' and col2 in ('${col2}')"); Assertions.assertEquals("select * from test where col1 in (?,?,?) and date=? and col2 in (?,?,?)", sql); Assertions.assertEquals(7, params.size()); @@ -63,7 +59,7 @@ public class ParameterUtilsTest { new Property("col1", Direct.IN, DataType.LIST, JSONUtils.toJsonString(Lists.newArrayList("c1")))); params2.put(2, new Property("date", Direct.IN, DataType.DATE, "2020-06-30")); String sql2 = ParameterUtils.expandListParameter(params2, - "select * from test where col1 in ('${col}') and date='${date}'", rgex); + "select * from test where col1 in ('${col}') and date='${date}'"); Assertions.assertEquals("select * from test where col1 in (?) and date=?", sql2); Assertions.assertEquals(2, params2.size()); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-procedure/src/main/java/org/apache/dolphinscheduler/plugin/task/procedure/ProcedureTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-procedure/src/main/java/org/apache/dolphinscheduler/plugin/task/procedure/ProcedureTask.java index 6a66ba1965..90b41ee9c2 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-procedure/src/main/java/org/apache/dolphinscheduler/plugin/task/procedure/ProcedureTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-procedure/src/main/java/org/apache/dolphinscheduler/plugin/task/procedure/ProcedureTask.java @@ -26,6 +26,7 @@ import org.apache.dolphinscheduler.plugin.datasource.api.plugin.DataSourceClient import org.apache.dolphinscheduler.plugin.datasource.api.plugin.DataSourceProcessorProvider; import org.apache.dolphinscheduler.plugin.task.api.AbstractTask; import org.apache.dolphinscheduler.plugin.task.api.TaskCallBack; +import org.apache.dolphinscheduler.plugin.task.api.TaskConstants; import org.apache.dolphinscheduler.plugin.task.api.TaskException; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.enums.DataType; @@ -131,9 +132,9 @@ public class ProcedureTask extends AbstractTask { } private String formatSql(Map sqlParamsMap, Map paramsMap) { - setSqlParamsMap(procedureParameters.getMethod(), rgex, sqlParamsMap, paramsMap, + setSqlParamsMap(procedureParameters.getMethod(), sqlParamsMap, paramsMap, taskExecutionContext.getTaskInstanceId()); - return procedureParameters.getMethod().replaceAll(rgex, "?"); + return procedureParameters.getMethod().replaceAll(TaskConstants.SQL_PARAMS_REGEX, "?"); } /** diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTask.java index cd904cafa5..ef09a5459b 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTask.java @@ -449,17 +449,16 @@ public class SqlTask extends AbstractTask { } // special characters need to be escaped, ${} needs to be escaped - setSqlParamsMap(sql, rgex, sqlParamsMap, paramsMap, taskExecutionContext.getTaskInstanceId()); + setSqlParamsMap(sql, sqlParamsMap, paramsMap, taskExecutionContext.getTaskInstanceId()); // Replace the original value in sql ！{...} ，Does not participate in precompilation String rgexo = "['\"]*\\!\\{(.*?)\\}['\"]*"; sql = replaceOriginalValue(sql, rgexo, paramsMap); // replace the ${} of the SQL statement with the Placeholder - // String formatSql = sql.replaceAll(rgex, "?"); // Convert the list parameter - String formatSql = ParameterUtils.expandListParameter(sqlParamsMap, sql, rgex); + String formatSql = ParameterUtils.expandListParameter(sqlParamsMap, sql); sqlBuilder.append(formatSql); // print replace sql - printReplacedSql(sql, formatSql, rgex, sqlParamsMap); + printReplacedSql(sql, formatSql, TaskConstants.SQL_PARAMS_REGEX, sqlParamsMap); return new SqlBinds(sqlBuilder.toString(), sqlParamsMap); } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/test/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTaskTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/test/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTaskTest.java index c672f0175d..3e1b25a4be 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/test/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTaskTest.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/test/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTaskTest.java @@ -18,6 +18,7 @@ package org.apache.dolphinscheduler.plugin.task.sql; import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.plugin.task.api.TaskConstants; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.enums.DataType; import org.apache.dolphinscheduler.plugin.task.api.enums.Direct; @@ -61,21 +62,21 @@ class SqlTaskTest { void testReplacingSqlWithoutParams() { String querySql = "select 1"; String expected = "select 1"; - Assertions.assertEquals(expected, querySql.replaceAll(sqlTask.rgex, "?")); + Assertions.assertEquals(expected, querySql.replaceAll(TaskConstants.SQL_PARAMS_REGEX, "?")); } @Test void testReplacingSqlWithDollarSymbol() { String querySql = "select concat(amount, '$') as price from product"; String expected = "select concat(amount, '$') as price from product"; - Assertions.assertEquals(expected, querySql.replaceAll(sqlTask.rgex, "?")); + Assertions.assertEquals(expected, querySql.replaceAll(TaskConstants.SQL_PARAMS_REGEX, "?")); } @Test void testReplacingHiveLoadSql() { String hiveLoadSql = "load inpath '/tmp/test_table/dt=${dt}' into table test_table partition(dt=${dt})"; String expected = "load inpath '/tmp/test_table/dt=?' into table test_table partition(dt=?)"; - Assertions.assertEquals(expected, hiveLoadSql.replaceAll(sqlTask.rgex, "?")); + Assertions.assertEquals(expected, hiveLoadSql.replaceAll(TaskConstants.SQL_PARAMS_REGEX, "?")); Map sqlParamsMap = new HashMap<>(); Map expectedSQLParamsMap = new HashMap<>(); @@ -83,7 +84,7 @@ class SqlTaskTest { expectedSQLParamsMap.put(2, new Property("dt", Direct.IN, DataType.VARCHAR, "1970")); Map paramsMap = new HashMap<>(); paramsMap.put("dt", new Property("dt", Direct.IN, DataType.VARCHAR, "1970")); - sqlTask.setSqlParamsMap(hiveLoadSql, sqlTask.rgex, sqlParamsMap, paramsMap, 1); + sqlTask.setSqlParamsMap(hiveLoadSql, sqlParamsMap, paramsMap, 1); Assertions.assertEquals(sqlParamsMap, expectedSQLParamsMap); } @@ -91,38 +92,38 @@ class SqlTaskTest { void testReplacingSelectSql() { String querySql = "select id from student where dt='${dt}'"; String expected = "select id from student where dt=?"; - Assertions.assertEquals(expected, querySql.replaceAll(sqlTask.rgex, "?")); + Assertions.assertEquals(expected, querySql.replaceAll(TaskConstants.SQL_PARAMS_REGEX, "?")); Map sqlParamsMap = new HashMap<>(); Map expectedSQLParamsMap = new HashMap<>(); expectedSQLParamsMap.put(1, new Property("dt", Direct.IN, DataType.VARCHAR, "1970")); Map paramsMap = new HashMap<>(); paramsMap.put("dt", new Property("dt", Direct.IN, DataType.VARCHAR, "1970")); - sqlTask.setSqlParamsMap(querySql, sqlTask.rgex, sqlParamsMap, paramsMap, 1); + sqlTask.setSqlParamsMap(querySql, sqlParamsMap, paramsMap, 1); Assertions.assertEquals(sqlParamsMap, expectedSQLParamsMap); querySql = "select id from student where dt=\"${dt}\""; expected = "select id from student where dt=?"; - Assertions.assertEquals(expected, querySql.replaceAll(sqlTask.rgex, "?")); + Assertions.assertEquals(expected, querySql.replaceAll(TaskConstants.SQL_PARAMS_REGEX, "?")); sqlParamsMap.clear(); - sqlTask.setSqlParamsMap(querySql, sqlTask.rgex, sqlParamsMap, paramsMap, 1); + sqlTask.setSqlParamsMap(querySql, sqlParamsMap, paramsMap, 1); Assertions.assertEquals(sqlParamsMap, expectedSQLParamsMap); querySql = "select id from student where dt=${dt}"; expected = "select id from student where dt=?"; - Assertions.assertEquals(expected, querySql.replaceAll(sqlTask.rgex, "?")); + Assertions.assertEquals(expected, querySql.replaceAll(TaskConstants.SQL_PARAMS_REGEX, "?")); sqlParamsMap.clear(); - sqlTask.setSqlParamsMap(querySql, sqlTask.rgex, sqlParamsMap, paramsMap, 1); + sqlTask.setSqlParamsMap(querySql, sqlParamsMap, paramsMap, 1); Assertions.assertEquals(sqlParamsMap, expectedSQLParamsMap); querySql = "select id from student where dt=${dt} and gender=1"; expected = "select id from student where dt=? and gender=1"; - Assertions.assertEquals(expected, querySql.replaceAll(sqlTask.rgex, "?")); + Assertions.assertEquals(expected, querySql.replaceAll(TaskConstants.SQL_PARAMS_REGEX, "?")); sqlParamsMap.clear(); - sqlTask.setSqlParamsMap(querySql, sqlTask.rgex, sqlParamsMap, paramsMap, 1); + sqlTask.setSqlParamsMap(querySql, sqlParamsMap, paramsMap, 1); Assertions.assertEquals(sqlParamsMap, expectedSQLParamsMap); } @@ -130,7 +131,7 @@ class SqlTaskTest { void testReplacingSqlNonGreedy() { String querySql = "select id from student where year=${year} and month=${month} and gender=1"; String expected = "select id from student where year=? and month=? and gender=1"; - Assertions.assertEquals(expected, querySql.replaceAll(sqlTask.rgex, "?")); + Assertions.assertEquals(expected, querySql.replaceAll(TaskConstants.SQL_PARAMS_REGEX, "?")); Map sqlParamsMap = new HashMap<>(); Map expectedSQLParamsMap = new HashMap<>(); @@ -139,7 +140,7 @@ class SqlTaskTest { Map paramsMap = new HashMap<>(); paramsMap.put("year", new Property("year", Direct.IN, DataType.VARCHAR, "1970")); paramsMap.put("month", new Property("month", Direct.IN, DataType.VARCHAR, "12")); - sqlTask.setSqlParamsMap(querySql, sqlTask.rgex, sqlParamsMap, paramsMap, 1); + sqlTask.setSqlParamsMap(querySql, sqlParamsMap, paramsMap, 1); Assertions.assertEquals(sqlParamsMap, expectedSQLParamsMap); } @@ -165,10 +166,10 @@ class SqlTaskTest { paramsMap.put("month", new Property("month", Direct.IN, DataType.VARCHAR, "12")); paramsMap.put("gender", new Property("gender", Direct.IN, DataType.LIST, JSONUtils.toJsonString(Lists.newArrayList(1, 2)))); - sqlTask.setSqlParamsMap(querySql, sqlTask.rgex, sqlParamsMap, paramsMap, 1); + sqlTask.setSqlParamsMap(querySql, sqlParamsMap, paramsMap, 1); Assertions.assertEquals(sqlParamsMap, expectedSQLParamsMap); - String formatSql = ParameterUtils.expandListParameter(sqlParamsMap, querySql, sqlTask.rgex); + String formatSql = ParameterUtils.expandListParameter(sqlParamsMap, querySql); Assertions.assertEquals(4, sqlParamsMap.size()); Assertions.assertEquals(expected, formatSql); } From 0d1d1200f34e4c0f537b3e1e5fa4f94cd38280da Mon Sep 17 00:00:00 2001 From: xxzuo <1293378490@qq.com> Date: Thu, 16 May 2024 09:34:25 +0800 Subject: [PATCH 6/6] remove excess variables --- .../plugin/task/api/utils/ParameterUtils.java | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/ParameterUtils.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/ParameterUtils.java index 13e0ef4575..b93528a049 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/ParameterUtils.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/ParameterUtils.java @@ -23,6 +23,7 @@ import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.PARAMETE import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.plugin.task.api.TaskConstants; import org.apache.dolphinscheduler.plugin.task.api.enums.DataType; import org.apache.dolphinscheduler.plugin.task.api.enums.Direct; import org.apache.dolphinscheduler.plugin.task.api.model.Property; @@ -52,11 +53,6 @@ public class ParameterUtils { private static final char PARAM_REPLACE_CHAR = '?'; - private static final String PARAM_REGEX = - String.format("['\"]\\$\\{(?<%s>.*?)}['\"]|\\$\\{(?<%s>.*?)}", "paramName1", "paramName2"); - - private static final Pattern PARAM_REGEX_PATTERN = Pattern.compile(PARAM_REGEX); - private ParameterUtils() { throw new UnsupportedOperationException("Construct ParameterUtils"); } @@ -198,7 +194,7 @@ public class ParameterUtils { return sql; } StringBuilder ret = new StringBuilder(sql); - Matcher m = PARAM_REGEX_PATTERN.matcher(sql); + Matcher m = TaskConstants.SQL_PARAMS_PATTERN.matcher(sql); int index = 1; int paramsIndex = 1; // When matching with a regex, determine whether the corresponding property is a list.