From 3b1de41acbee827b320370b14c6c73c7158f1aec Mon Sep 17 00:00:00 2001 From: Wenjun Ruan Date: Thu, 4 Apr 2024 23:44:12 +0800 Subject: [PATCH] Remove dolphinscheduler-data-quality from dolphinscheduler-task-dataquality (#15791) --- dolphinscheduler-api/pom.xml | 5 +++ .../quality/flow/batch/reader/JdbcReader.java | 8 +++- .../quality/flow/batch/writer/JdbcWriter.java | 8 +++- .../data/quality/utils/ParserUtilsTest.java | 39 ------------------- .../dolphinscheduler-task-dataquality/pom.xml | 5 --- .../plugin/task/dq/utils/RuleParserUtils.java | 23 +++++++---- 6 files changed, 32 insertions(+), 56 deletions(-) delete mode 100644 dolphinscheduler-data-quality/src/test/java/org/apache/dolphinscheduler/data/quality/utils/ParserUtilsTest.java diff --git a/dolphinscheduler-api/pom.xml b/dolphinscheduler-api/pom.xml index 681a85318a..40d556a17e 100644 --- a/dolphinscheduler-api/pom.xml +++ b/dolphinscheduler-api/pom.xml @@ -61,6 +61,11 @@ dolphinscheduler-meter + + org.apache.dolphinscheduler + dolphinscheduler-data-quality + + org.apache.dolphinscheduler dolphinscheduler-datasource-all diff --git a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/batch/reader/JdbcReader.java b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/batch/reader/JdbcReader.java index 274d4f793a..97ae414051 100644 --- a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/batch/reader/JdbcReader.java +++ b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/batch/reader/JdbcReader.java @@ -17,6 +17,7 @@ package org.apache.dolphinscheduler.data.quality.flow.batch.reader; +import static java.nio.charset.StandardCharsets.UTF_8; import static org.apache.dolphinscheduler.data.quality.Constants.DATABASE; import static org.apache.dolphinscheduler.data.quality.Constants.DB_TABLE; import static org.apache.dolphinscheduler.data.quality.Constants.DOTS; @@ -32,17 +33,19 @@ import org.apache.dolphinscheduler.data.quality.config.ValidateResult; import org.apache.dolphinscheduler.data.quality.execution.SparkRuntimeEnvironment; import org.apache.dolphinscheduler.data.quality.flow.batch.BatchReader; import org.apache.dolphinscheduler.data.quality.utils.ConfigUtils; -import org.apache.dolphinscheduler.data.quality.utils.ParserUtils; import org.apache.spark.sql.DataFrameReader; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; import org.apache.spark.sql.SparkSession; +import java.net.URLDecoder; import java.util.Arrays; import java.util.HashMap; import java.util.Map; +import lombok.SneakyThrows; + /** * AbstractJdbcSource */ @@ -74,6 +77,7 @@ public class JdbcReader implements BatchReader { return jdbcReader(env.sparkSession()).load(); } + @SneakyThrows private DataFrameReader jdbcReader(SparkSession sparkSession) { DataFrameReader reader = sparkSession.read() @@ -81,7 +85,7 @@ public class JdbcReader implements BatchReader { .option(URL, config.getString(URL)) .option(DB_TABLE, config.getString(DATABASE) + "." + config.getString(TABLE)) .option(USER, config.getString(USER)) - .option(PASSWORD, ParserUtils.decode(config.getString(PASSWORD))) + .option(PASSWORD, URLDecoder.decode(config.getString(PASSWORD), UTF_8.name())) .option(DRIVER, config.getString(DRIVER)); Config jdbcConfig = ConfigUtils.extractSubConfig(config, JDBC + DOTS, false); diff --git a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/batch/writer/JdbcWriter.java b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/batch/writer/JdbcWriter.java index 07b2bd60d5..b737567f21 100644 --- a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/batch/writer/JdbcWriter.java +++ b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/batch/writer/JdbcWriter.java @@ -33,13 +33,16 @@ import org.apache.dolphinscheduler.data.quality.config.Config; import org.apache.dolphinscheduler.data.quality.config.ValidateResult; import org.apache.dolphinscheduler.data.quality.execution.SparkRuntimeEnvironment; import org.apache.dolphinscheduler.data.quality.flow.batch.BatchWriter; -import org.apache.dolphinscheduler.data.quality.utils.ParserUtils; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; +import java.net.URLDecoder; +import java.nio.charset.StandardCharsets; import java.util.Arrays; +import lombok.SneakyThrows; + import com.google.common.base.Strings; /** @@ -70,6 +73,7 @@ public class JdbcWriter implements BatchWriter { } } + @SneakyThrows @Override public void write(Dataset data, SparkRuntimeEnvironment env) { if (!Strings.isNullOrEmpty(config.getString(SQL))) { @@ -82,7 +86,7 @@ public class JdbcWriter implements BatchWriter { .option(URL, config.getString(URL)) .option(DB_TABLE, config.getString(DATABASE) + "." + config.getString(TABLE)) .option(USER, config.getString(USER)) - .option(PASSWORD, ParserUtils.decode(config.getString(PASSWORD))) + .option(PASSWORD, URLDecoder.decode(config.getString(PASSWORD), StandardCharsets.UTF_8.name())) .mode(config.getString(SAVE_MODE)) .save(); } diff --git a/dolphinscheduler-data-quality/src/test/java/org/apache/dolphinscheduler/data/quality/utils/ParserUtilsTest.java b/dolphinscheduler-data-quality/src/test/java/org/apache/dolphinscheduler/data/quality/utils/ParserUtilsTest.java deleted file mode 100644 index 328316cc39..0000000000 --- a/dolphinscheduler-data-quality/src/test/java/org/apache/dolphinscheduler/data/quality/utils/ParserUtilsTest.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.data.quality.utils; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -public class ParserUtilsTest { - - @Test - public void testParserUtils() { - String testStr = "aaa$bbb$ccc%ddd^eee#fff"; - String encode = ParserUtils.encode(testStr); - String decode = ParserUtils.decode(encode); - Assertions.assertEquals(testStr, decode); - - String blank = ""; - Assertions.assertEquals(ParserUtils.encode(blank), blank); - Assertions.assertEquals(ParserUtils.decode(blank), blank); - - Assertions.assertNull(ParserUtils.encode(null)); - Assertions.assertNull(ParserUtils.decode(null)); - } -} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/pom.xml index cc64b6cdcb..6626234a19 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/pom.xml +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/pom.xml @@ -43,11 +43,6 @@ dolphinscheduler-datasource-all ${project.version} - - org.apache.dolphinscheduler - dolphinscheduler-data-quality - ${project.version} - org.apache.commons commons-collections4 diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/utils/RuleParserUtils.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/utils/RuleParserUtils.java index e99bb3dfaf..185573f66e 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/utils/RuleParserUtils.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/utils/RuleParserUtils.java @@ -17,6 +17,7 @@ package org.apache.dolphinscheduler.plugin.task.dq.utils; +import static java.nio.charset.StandardCharsets.UTF_8; import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.PARAMETER_BUSINESS_DATE; import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.PARAMETER_CURRENT_DATE; import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.PARAMETER_DATETIME; @@ -62,7 +63,6 @@ import static org.apache.dolphinscheduler.plugin.task.api.utils.DataQualityConst import static org.apache.dolphinscheduler.plugin.task.api.utils.DataQualityConstants.USER; import org.apache.dolphinscheduler.common.utils.JSONUtils; -import org.apache.dolphinscheduler.data.quality.utils.ParserUtils; import org.apache.dolphinscheduler.plugin.datasource.api.utils.DataSourceUtils; import org.apache.dolphinscheduler.plugin.task.api.DataQualityTaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.enums.dp.ExecuteSqlType; @@ -80,12 +80,15 @@ import org.apache.dolphinscheduler.spi.enums.DbType; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; +import java.net.URLEncoder; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; +import lombok.SneakyThrows; + import com.fasterxml.jackson.databind.node.ArrayNode; /** @@ -102,9 +105,10 @@ public class RuleParserUtils { private static final String AND_TARGET_FILTER = "AND (${target_filter})"; private static final String WHERE_TARGET_FILTER = "WHERE (${target_filter})"; + @SneakyThrows public static List getReaderConfigList( Map inputParameterValue, - DataQualityTaskExecutionContext dataQualityTaskExecutionContext) throws DataQualityException { + DataQualityTaskExecutionContext dataQualityTaskExecutionContext) { List readerConfigList = new ArrayList<>(); @@ -123,7 +127,7 @@ public class RuleParserUtils { config.put(URL, DataSourceUtils.getJdbcUrl(DbType.of(dataQualityTaskExecutionContext.getSourceType()), sourceDataSource)); config.put(USER, sourceDataSource.getUser()); - config.put(PASSWORD, ParserUtils.encode(sourceDataSource.getPassword())); + config.put(PASSWORD, URLEncoder.encode(sourceDataSource.getPassword(), UTF_8.name())); config.put(DRIVER, DataSourceUtils .getDatasourceDriver(DbType.of(dataQualityTaskExecutionContext.getSourceType()))); String outputTable = inputParameterValue.get(SRC_DATABASE) + "_" + inputParameterValue.get(SRC_TABLE); @@ -150,7 +154,7 @@ public class RuleParserUtils { config.put(URL, DataSourceUtils.getJdbcUrl(DbType.of(dataQualityTaskExecutionContext.getTargetType()), targetDataSource)); config.put(USER, targetDataSource.getUser()); - config.put(PASSWORD, ParserUtils.encode(targetDataSource.getPassword())); + config.put(PASSWORD, URLEncoder.encode(targetDataSource.getPassword(), UTF_8.name())); config.put(DRIVER, DataSourceUtils .getDatasourceDriver(DbType.of(dataQualityTaskExecutionContext.getTargetType()))); String outputTable = @@ -264,9 +268,10 @@ public class RuleParserUtils { return defaultInputParameterValue; } + @SneakyThrows public static List getWriterConfigList( String sql, - DataQualityTaskExecutionContext dataQualityTaskExecutionContext) throws DataQualityException { + DataQualityTaskExecutionContext dataQualityTaskExecutionContext) { List writerConfigList = new ArrayList<>(); if (StringUtils.isNotEmpty(dataQualityTaskExecutionContext.getWriterConnectorType())) { @@ -284,7 +289,7 @@ public class RuleParserUtils { config.put(URL, DataSourceUtils.getJdbcUrl(DbType.of(dataQualityTaskExecutionContext.getWriterType()), writerDataSource)); config.put(USER, writerDataSource.getUser()); - config.put(PASSWORD, ParserUtils.encode(writerDataSource.getPassword())); + config.put(PASSWORD, URLEncoder.encode(writerDataSource.getPassword(), UTF_8.name())); config.put(DRIVER, DataSourceUtils .getDatasourceDriver(DbType.of(dataQualityTaskExecutionContext.getWriterType()))); config.put(SQL, sql); @@ -336,8 +341,9 @@ public class RuleParserUtils { return readerConfigList; } + @SneakyThrows public static BaseConfig getStatisticsValueConfig( - DataQualityTaskExecutionContext dataQualityTaskExecutionContext) throws DataQualityException { + DataQualityTaskExecutionContext dataQualityTaskExecutionContext) { BaseConfig baseConfig = null; if (StringUtils.isNotEmpty(dataQualityTaskExecutionContext.getStatisticsValueConnectorType())) { BaseConnectionParam writerDataSource = @@ -354,7 +360,7 @@ public class RuleParserUtils { config.put(URL, DataSourceUtils.getJdbcUrl( DbType.of(dataQualityTaskExecutionContext.getStatisticsValueType()), writerDataSource)); config.put(USER, writerDataSource.getUser()); - config.put(PASSWORD, ParserUtils.encode(writerDataSource.getPassword())); + config.put(PASSWORD, URLEncoder.encode(writerDataSource.getPassword(), UTF_8.name())); config.put(DRIVER, DataSourceUtils .getDatasourceDriver(DbType.of(dataQualityTaskExecutionContext.getWriterType()))); } @@ -544,6 +550,7 @@ public class RuleParserUtils { /** * the unique code use to get the same type and condition task statistics value + * * @param inputParameterValue * @return */