From 38b876733c0ac30ebeeeb9c679687834bb2d6009 Mon Sep 17 00:00:00 2001 From: rickchengx <38122586+rickchengx@users.noreply.github.com> Date: Thu, 24 Nov 2022 14:54:54 +0800 Subject: [PATCH] [Feature-10498] Mask the password in the log of sqoop task (#11589) --- docs/docs/en/architecture/design.md | 2 +- docs/docs/zh/architecture/design.md | 2 +- .../common}/log/SensitiveDataConverter.java | 47 +++++++------------ .../log/SensitiveDataConverterTest.java | 27 +++-------- .../src/main/resources/logback-spring.xml | 2 +- .../src/main/resources/logback-spring.xml | 2 +- .../plugin/task/sqoop/SqoopConstants.java | 1 + .../plugin/task/sqoop/SqoopTask.java | 3 ++ .../plugin/task/sqoop/SqoopTaskTest.java | 38 +++++++++++++++ .../src/main/resources/logback-spring.xml | 2 +- 10 files changed, 70 insertions(+), 56 deletions(-) rename {dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service => dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common}/log/SensitiveDataConverter.java (70%) rename {dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service => dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common}/log/SensitiveDataConverterTest.java (68%) create mode 100644 dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/test/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopTaskTest.java diff --git a/docs/docs/en/architecture/design.md b/docs/docs/en/architecture/design.md index c174f79867..26f60d9761 100644 --- a/docs/docs/en/architecture/design.md +++ b/docs/docs/en/architecture/design.md @@ -197,7 +197,7 @@ In the early schedule design, if there is no priority design and use the fair sc - For details, please refer to the logback configuration of Master and Worker, as shown in the following example: ```xml - + diff --git a/docs/docs/zh/architecture/design.md b/docs/docs/zh/architecture/design.md index 14a7b91142..dbfb1c1613 100644 --- a/docs/docs/zh/architecture/design.md +++ b/docs/docs/zh/architecture/design.md @@ -195,7 +195,7 @@ - 详情可参考Master和Worker的logback配置，如下示例： ```xml - + diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/log/SensitiveDataConverter.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/SensitiveDataConverter.java similarity index 70% rename from dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/log/SensitiveDataConverter.java rename to dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/SensitiveDataConverter.java index 4bb486735c..57b2bcde59 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/log/SensitiveDataConverter.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/SensitiveDataConverter.java @@ -15,11 +15,15 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.service.log; +package org.apache.dolphinscheduler.common.log; import org.apache.dolphinscheduler.common.constants.Constants; import org.apache.dolphinscheduler.common.constants.DataSourceConstants; +import org.apache.commons.lang3.StringUtils; + +import java.util.Arrays; +import java.util.HashSet; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -33,10 +37,9 @@ import com.google.common.base.Strings; */ public class SensitiveDataConverter extends MessageConverter { - /** - * password pattern - */ - private final Pattern pwdPattern = Pattern.compile(DataSourceConstants.DATASOURCE_PASSWORD_REGEX); + private static Pattern multilinePattern; + private static HashSet maskPatterns = + new HashSet<>(Arrays.asList(DataSourceConstants.DATASOURCE_PASSWORD_REGEX)); @Override public String convert(ILoggingEvent event) { @@ -45,41 +48,25 @@ public class SensitiveDataConverter extends MessageConverter { String requestLogMsg = event.getFormattedMessage(); // desensitization log - return convertMsg(requestLogMsg); + return maskSensitiveData(requestLogMsg); } - /** - * deal with sensitive log - * - * @param oriLogMsg original log - */ - private String convertMsg(final String oriLogMsg) { + public static void addMaskPattern(String maskPattern) { + maskPatterns.add(maskPattern); + } - String tempLogMsg = oriLogMsg; - - if (!Strings.isNullOrEmpty(tempLogMsg)) { - tempLogMsg = passwordHandler(pwdPattern, tempLogMsg); + public static String maskSensitiveData(final String logMsg) { + if (StringUtils.isEmpty(logMsg)) { + return logMsg; } - return tempLogMsg; - } - - /** - * password regex - * - * @param logMsg original log - */ - static String passwordHandler(Pattern pwdPattern, String logMsg) { - - Matcher matcher = pwdPattern.matcher(logMsg); + multilinePattern = Pattern.compile(String.join("|", maskPatterns), Pattern.MULTILINE); StringBuffer sb = new StringBuffer(logMsg.length()); + Matcher matcher = multilinePattern.matcher(logMsg); while (matcher.find()) { - String password = matcher.group(); - String maskPassword = Strings.repeat(Constants.STAR, password.length()); - matcher.appendReplacement(sb, maskPassword); } matcher.appendTail(sb); diff --git a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/log/SensitiveDataConverterTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/SensitiveDataConverterTest.java similarity index 68% rename from dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/log/SensitiveDataConverterTest.java rename to dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/SensitiveDataConverterTest.java index 9f64b6ef95..8d7b4af2c4 100644 --- a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/log/SensitiveDataConverterTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/SensitiveDataConverterTest.java @@ -15,13 +15,7 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.service.log; - -import static org.apache.dolphinscheduler.service.log.SensitiveDataConverter.passwordHandler; - -import org.apache.dolphinscheduler.common.constants.DataSourceConstants; - -import java.util.regex.Pattern; +package org.apache.dolphinscheduler.common.log; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -32,11 +26,6 @@ public class SensitiveDataConverterTest { private final Logger logger = LoggerFactory.getLogger(SensitiveDataConverterTest.class); - /** - * password pattern - */ - private final Pattern pwdPattern = Pattern.compile(DataSourceConstants.DATASOURCE_PASSWORD_REGEX); - private final String logMsg = "{\"address\":\"jdbc:mysql://192.168.xx.xx:3306\"," + "\"database\":\"carbond\"," + "\"jdbcUrl\":\"jdbc:mysql://192.168.xx.xx:3306/ods\"," @@ -49,21 +38,17 @@ public class SensitiveDataConverterTest { + "\"user\":\"view\"," + "\"password\":\"*****\"}"; - @Test - public void convert() { - Assertions.assertEquals(maskLogMsg, passwordHandler(pwdPattern, logMsg)); - } - /** * mask sensitive logMsg - sql task datasource password */ @Test public void testPwdLogMsgConverter() { - logger.info("parameter : {}", logMsg); - logger.info("parameter : {}", passwordHandler(pwdPattern, logMsg)); + final String maskedLog = SensitiveDataConverter.maskSensitiveData(logMsg); - Assertions.assertNotEquals(logMsg, passwordHandler(pwdPattern, logMsg)); - Assertions.assertEquals(maskLogMsg, passwordHandler(pwdPattern, logMsg)); + logger.info("original parameter : {}", logMsg); + logger.info("masked parameter : {}", maskedLog); + + Assertions.assertEquals(maskLogMsg, maskedLog); } diff --git a/dolphinscheduler-master/src/main/resources/logback-spring.xml b/dolphinscheduler-master/src/main/resources/logback-spring.xml index 6b9f3a7b79..9926eff21d 100644 --- a/dolphinscheduler-master/src/main/resources/logback-spring.xml +++ b/dolphinscheduler-master/src/main/resources/logback-spring.xml @@ -28,7 +28,7 @@ + converterClass="org.apache.dolphinscheduler.common.log.SensitiveDataConverter"/> diff --git a/dolphinscheduler-standalone-server/src/main/resources/logback-spring.xml b/dolphinscheduler-standalone-server/src/main/resources/logback-spring.xml index 59da2de6d6..ee7c5414f4 100644 --- a/dolphinscheduler-standalone-server/src/main/resources/logback-spring.xml +++ b/dolphinscheduler-standalone-server/src/main/resources/logback-spring.xml @@ -48,7 +48,7 @@ + converterClass="org.apache.dolphinscheduler.common.log.SensitiveDataConverter"/> diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopConstants.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopConstants.java index f4861eb4fb..1a2ce19b9a 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopConstants.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopConstants.java @@ -72,4 +72,5 @@ public final class SqoopConstants { public static final String UPDATE_KEY = "--update-key"; public static final String UPDATE_MODE = "--update-mode"; + public static final String SQOOP_PASSWORD_REGEX = "(?<=(--password \")).+?(?=\")"; } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopTask.java index 4df286aeec..c2eb473ce7 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopTask.java @@ -17,6 +17,7 @@ package org.apache.dolphinscheduler.plugin.task.sqoop; +import org.apache.dolphinscheduler.common.log.SensitiveDataConverter; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.plugin.task.api.AbstractYarnTask; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; @@ -67,6 +68,8 @@ public class SqoopTask extends AbstractYarnTask { sqoopTaskExecutionContext = sqoopParameters.generateExtendedContext(taskExecutionContext.getResourceParametersHelper()); + + SensitiveDataConverter.addMaskPattern(SqoopConstants.SQOOP_PASSWORD_REGEX); } @Override diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/test/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopTaskTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/test/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopTaskTest.java new file mode 100644 index 0000000000..f978c2284f --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/test/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopTaskTest.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.task.sqoop; + +import org.apache.dolphinscheduler.common.log.SensitiveDataConverter; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class SqoopTaskTest { + + @Test + public void testSqoopPasswordMask() { + final String originalScript = + "sqoop import -D mapred.job.name=sqoop_task -m 1 --connect \"jdbc:mysql://localhost:3306/defuault\" --username root --password \"mypassword\" --table student --target-dir /sqoop_test --as-textfile"; + + final String maskScript = + "sqoop import -D mapred.job.name=sqoop_task -m 1 --connect \"jdbc:mysql://localhost:3306/defuault\" --username root --password \"**********\" --table student --target-dir /sqoop_test --as-textfile"; + + SensitiveDataConverter.addMaskPattern(SqoopConstants.SQOOP_PASSWORD_REGEX); + Assertions.assertEquals(maskScript, SensitiveDataConverter.maskSensitiveData(originalScript)); + } +} diff --git a/dolphinscheduler-worker/src/main/resources/logback-spring.xml b/dolphinscheduler-worker/src/main/resources/logback-spring.xml index 7d0d9cd827..bd01f46502 100644 --- a/dolphinscheduler-worker/src/main/resources/logback-spring.xml +++ b/dolphinscheduler-worker/src/main/resources/logback-spring.xml @@ -29,7 +29,7 @@ + converterClass="org.apache.dolphinscheduler.common.log.SensitiveDataConverter"/>