[Feature-10498] Mask the password in the log of sqoop task (#11589)
This commit is contained in:
rickchengx
GitHub
parent
db615ba284
commit
38b876733c
|
|
@ -197,7 +197,7 @@ In the early schedule design, if there is no priority design and use the fair sc
|
|||
- For details, please refer to the logback configuration of Master and Worker, as shown in the following example:
|
||||
|
||||
```xml
|
||||
<conversionRule conversionWord="message" converterClass="org.apache.dolphinscheduler.service.log.SensitiveDataConverter"/>
|
||||
<conversionRule conversionWord="message" converterClass="org.apache.dolphinscheduler.common.log.SensitiveDataConverter"/>
|
||||
<appender name="TASKLOGFILE" class="ch.qos.logback.classic.sift.SiftingAppender">
|
||||
<filter class="org.apache.dolphinscheduler.service.log.TaskLogFilter"/>
|
||||
<Discriminator class="org.apache.dolphinscheduler.service.log.TaskLogDiscriminator">
|
||||
|
|
|
|||
|
|
@ -195,7 +195,7 @@
|
|||
- 详情可参考Master和Worker的logback配置，如下示例：
|
||||
|
||||
```xml
|
||||
<conversionRule conversionWord="message" converterClass="org.apache.dolphinscheduler.service.log.SensitiveDataConverter"/>
|
||||
<conversionRule conversionWord="message" converterClass="org.apache.dolphinscheduler.common.log.SensitiveDataConverter"/>
|
||||
<appender name="TASKLOGFILE" class="ch.qos.logback.classic.sift.SiftingAppender">
|
||||
<filter class="org.apache.dolphinscheduler.service.log.TaskLogFilter"/>
|
||||
<Discriminator class="org.apache.dolphinscheduler.service.log.TaskLogDiscriminator">
|
||||
|
|
|
|||
|
|
@ -15,11 +15,15 @@
|
|||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.dolphinscheduler.service.log;
|
||||
package org.apache.dolphinscheduler.common.log;
|
||||
|
||||
import org.apache.dolphinscheduler.common.constants.Constants;
|
||||
import org.apache.dolphinscheduler.common.constants.DataSourceConstants;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.HashSet;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
|
|
@ -33,10 +37,9 @@ import com.google.common.base.Strings;
|
|||
*/
|
||||
public class SensitiveDataConverter extends MessageConverter {
|
||||
|
||||
/**
|
||||
* password pattern
|
||||
*/
|
||||
private final Pattern pwdPattern = Pattern.compile(DataSourceConstants.DATASOURCE_PASSWORD_REGEX);
|
||||
private static Pattern multilinePattern;
|
||||
private static HashSet<String> maskPatterns =
|
||||
new HashSet<>(Arrays.asList(DataSourceConstants.DATASOURCE_PASSWORD_REGEX));
|
||||
|
||||
@Override
|
||||
public String convert(ILoggingEvent event) {
|
||||
|
|
@ -45,41 +48,25 @@ public class SensitiveDataConverter extends MessageConverter {
|
|||
String requestLogMsg = event.getFormattedMessage();
|
||||
|
||||
// desensitization log
|
||||
return convertMsg(requestLogMsg);
|
||||
return maskSensitiveData(requestLogMsg);
|
||||
}
|
||||
|
||||
/**
|
||||
* deal with sensitive log
|
||||
*
|
||||
* @param oriLogMsg original log
|
||||
*/
|
||||
private String convertMsg(final String oriLogMsg) {
|
||||
public static void addMaskPattern(String maskPattern) {
|
||||
maskPatterns.add(maskPattern);
|
||||
}
|
||||
|
||||
String tempLogMsg = oriLogMsg;
|
||||
|
||||
if (!Strings.isNullOrEmpty(tempLogMsg)) {
|
||||
tempLogMsg = passwordHandler(pwdPattern, tempLogMsg);
|
||||
public static String maskSensitiveData(final String logMsg) {
|
||||
if (StringUtils.isEmpty(logMsg)) {
|
||||
return logMsg;
|
||||
}
|
||||
return tempLogMsg;
|
||||
}
|
||||
|
||||
/**
|
||||
* password regex
|
||||
*
|
||||
* @param logMsg original log
|
||||
*/
|
||||
static String passwordHandler(Pattern pwdPattern, String logMsg) {
|
||||
|
||||
Matcher matcher = pwdPattern.matcher(logMsg);
|
||||
multilinePattern = Pattern.compile(String.join("|", maskPatterns), Pattern.MULTILINE);
|
||||
|
||||
StringBuffer sb = new StringBuffer(logMsg.length());
|
||||
Matcher matcher = multilinePattern.matcher(logMsg);
|
||||
|
||||
while (matcher.find()) {
|
||||
|
||||
String password = matcher.group();
|
||||
|
||||
String maskPassword = Strings.repeat(Constants.STAR, password.length());
|
||||
|
||||
matcher.appendReplacement(sb, maskPassword);
|
||||
}
|
||||
matcher.appendTail(sb);
|
||||
|
|
@ -15,13 +15,7 @@
|
|||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.dolphinscheduler.service.log;
|
||||
|
||||
import static org.apache.dolphinscheduler.service.log.SensitiveDataConverter.passwordHandler;
|
||||
|
||||
import org.apache.dolphinscheduler.common.constants.DataSourceConstants;
|
||||
|
||||
import java.util.regex.Pattern;
|
||||
package org.apache.dolphinscheduler.common.log;
|
||||
|
||||
import org.junit.jupiter.api.Assertions;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
|
@ -32,11 +26,6 @@ public class SensitiveDataConverterTest {
|
|||
|
||||
private final Logger logger = LoggerFactory.getLogger(SensitiveDataConverterTest.class);
|
||||
|
||||
/**
|
||||
* password pattern
|
||||
*/
|
||||
private final Pattern pwdPattern = Pattern.compile(DataSourceConstants.DATASOURCE_PASSWORD_REGEX);
|
||||
|
||||
private final String logMsg = "{\"address\":\"jdbc:mysql://192.168.xx.xx:3306\","
|
||||
+ "\"database\":\"carbond\","
|
||||
+ "\"jdbcUrl\":\"jdbc:mysql://192.168.xx.xx:3306/ods\","
|
||||
|
|
@ -49,21 +38,17 @@ public class SensitiveDataConverterTest {
|
|||
+ "\"user\":\"view\","
|
||||
+ "\"password\":\"*****\"}";
|
||||
|
||||
@Test
|
||||
public void convert() {
|
||||
Assertions.assertEquals(maskLogMsg, passwordHandler(pwdPattern, logMsg));
|
||||
}
|
||||
|
||||
/**
|
||||
* mask sensitive logMsg - sql task datasource password
|
||||
*/
|
||||
@Test
|
||||
public void testPwdLogMsgConverter() {
|
||||
logger.info("parameter : {}", logMsg);
|
||||
logger.info("parameter : {}", passwordHandler(pwdPattern, logMsg));
|
||||
final String maskedLog = SensitiveDataConverter.maskSensitiveData(logMsg);
|
||||
|
||||
Assertions.assertNotEquals(logMsg, passwordHandler(pwdPattern, logMsg));
|
||||
Assertions.assertEquals(maskLogMsg, passwordHandler(pwdPattern, logMsg));
|
||||
logger.info("original parameter : {}", logMsg);
|
||||
logger.info("masked parameter : {}", maskedLog);
|
||||
|
||||
Assertions.assertEquals(maskLogMsg, maskedLog);
|
||||
|
||||
}
|
||||
|
||||
|
|
@ -28,7 +28,7 @@
|
|||
</appender>
|
||||
|
||||
<conversionRule conversionWord="message"
|
||||
converterClass="org.apache.dolphinscheduler.service.log.SensitiveDataConverter"/>
|
||||
converterClass="org.apache.dolphinscheduler.common.log.SensitiveDataConverter"/>
|
||||
<appender name="TASKLOGFILE" class="ch.qos.logback.classic.sift.SiftingAppender">
|
||||
<filter class="org.apache.dolphinscheduler.service.log.TaskLogFilter"/>
|
||||
<Discriminator class="org.apache.dolphinscheduler.service.log.TaskLogDiscriminator">
|
||||
|
|
|
|||
|
|
@ -48,7 +48,7 @@
|
|||
<logger name="org.apache.hadoop" level="WARN"/>
|
||||
|
||||
<conversionRule conversionWord="message"
|
||||
converterClass="org.apache.dolphinscheduler.service.log.SensitiveDataConverter"/>
|
||||
converterClass="org.apache.dolphinscheduler.common.log.SensitiveDataConverter"/>
|
||||
<appender name="TASKLOGFILE" class="ch.qos.logback.classic.sift.SiftingAppender">
|
||||
<filter class="org.apache.dolphinscheduler.service.log.TaskLogFilter"/>
|
||||
<Discriminator class="org.apache.dolphinscheduler.service.log.TaskLogDiscriminator">
|
||||
|
|
|
|||
|
|
@ -72,4 +72,5 @@ public final class SqoopConstants {
|
|||
public static final String UPDATE_KEY = "--update-key";
|
||||
public static final String UPDATE_MODE = "--update-mode";
|
||||
|
||||
public static final String SQOOP_PASSWORD_REGEX = "(?<=(--password \")).+?(?=\")";
|
||||
}
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@
|
|||
|
||||
package org.apache.dolphinscheduler.plugin.task.sqoop;
|
||||
|
||||
import org.apache.dolphinscheduler.common.log.SensitiveDataConverter;
|
||||
import org.apache.dolphinscheduler.common.utils.JSONUtils;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.AbstractYarnTask;
|
||||
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
|
||||
|
|
@ -67,6 +68,8 @@ public class SqoopTask extends AbstractYarnTask {
|
|||
|
||||
sqoopTaskExecutionContext =
|
||||
sqoopParameters.generateExtendedContext(taskExecutionContext.getResourceParametersHelper());
|
||||
|
||||
SensitiveDataConverter.addMaskPattern(SqoopConstants.SQOOP_PASSWORD_REGEX);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
|
|||
|
|
@ -0,0 +1,38 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.dolphinscheduler.plugin.task.sqoop;
|
||||
|
||||
import org.apache.dolphinscheduler.common.log.SensitiveDataConverter;
|
||||
|
||||
import org.junit.jupiter.api.Assertions;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
public class SqoopTaskTest {
|
||||
|
||||
@Test
|
||||
public void testSqoopPasswordMask() {
|
||||
final String originalScript =
|
||||
"sqoop import -D mapred.job.name=sqoop_task -m 1 --connect \"jdbc:mysql://localhost:3306/defuault\" --username root --password \"mypassword\" --table student --target-dir /sqoop_test --as-textfile";
|
||||
|
||||
final String maskScript =
|
||||
"sqoop import -D mapred.job.name=sqoop_task -m 1 --connect \"jdbc:mysql://localhost:3306/defuault\" --username root --password \"**********\" --table student --target-dir /sqoop_test --as-textfile";
|
||||
|
||||
SensitiveDataConverter.addMaskPattern(SqoopConstants.SQOOP_PASSWORD_REGEX);
|
||||
Assertions.assertEquals(maskScript, SensitiveDataConverter.maskSensitiveData(originalScript));
|
||||
}
|
||||
}
|
||||
|
|
@ -29,7 +29,7 @@
|
|||
</appender>
|
||||
|
||||
<conversionRule conversionWord="message"
|
||||
converterClass="org.apache.dolphinscheduler.service.log.SensitiveDataConverter"/>
|
||||
converterClass="org.apache.dolphinscheduler.common.log.SensitiveDataConverter"/>
|
||||
<appender name="TASKLOGFILE" class="ch.qos.logback.classic.sift.SiftingAppender">
|
||||
<filter class="org.apache.dolphinscheduler.service.log.TaskLogFilter"/>
|
||||
<Discriminator class="org.apache.dolphinscheduler.service.log.TaskLogDiscriminator">
|
||||
|
|
|
|||
Loading…
Reference in New Issue