natas_hw
/
dolphinscheduler0701
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[Fix-Test][LoggerServer] fix LoggerServer UT (#8973) 

* [Fix-Test][LoggerServer] fix LoggerServer UT

* fix checkstyle

* fix code style

* fix code style

* fix code style

* fix code style
This commit is contained in:
gaojun2048 2022-03-18 08:16:01 +08:00 committed by GitHub
parent 4149a9128f
commit d74c80e2a1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 67 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/LoggerRequestProcessor.java
View File

@ -17,6 +17,7 @@
package org.apache.dolphinscheduler.server.log;
import io.netty.channel.Channel;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.LoggerUtils;
import org.apache.dolphinscheduler.remote.command.Command;
@ -31,6 +32,9 @@ import org.apache.dolphinscheduler.remote.command.log.ViewLogRequestCommand;
import org.apache.dolphinscheduler.remote.command.log.ViewLogResponseCommand;
import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor;
import org.apache.dolphinscheduler.remote.utils.Constants;
import org.apache.dolphinscheduler.spi.utils.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.ByteArrayOutputStream;
import java.io.File;
@ -46,16 +50,11 @@ import java.util.concurrent.Executors;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.apache.dolphinscheduler.spi.utils.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import io.netty.channel.Channel;
/**
* logger request process logic
*/
public class LoggerRequestProcessor implements NettyRequestProcessor {
public class LoggerRequestProcessor
implements NettyRequestProcessor {
private final Logger logger = LoggerFactory.getLogger(LoggerRequestProcessor.class);
@ -128,7 +127,8 @@ public class LoggerRequestProcessor implements NettyRequestProcessor {
if (taskLogFile.exists()) {
status = taskLogFile.delete();
}
} catch (Exception e) {
}
catch (Exception e) {
status = false;
}
@ -142,6 +142,7 @@ public class LoggerRequestProcessor implements NettyRequestProcessor {
private boolean checkPathSecurity(String path) {
String dsHome = System.getProperty("DOLPHINSCHEDULER_HOME");
// if we run server in IDE, user.dir is the DS Home.
if (StringUtils.isBlank(dsHome)) {
dsHome = System.getProperty("user.dir");
}
@ -149,7 +150,8 @@ public class LoggerRequestProcessor implements NettyRequestProcessor {
if (path.startsWith(dsHome) && !path.contains("../") && path.endsWith(".log")) {
return true;
}
} else {
}
else {
logger.warn("path is null");
}
return false;
@ -168,14 +170,15 @@ public class LoggerRequestProcessor implements NettyRequestProcessor {
*/
private byte[] getFileContentBytes(String filePath) {
try (InputStream in = new FileInputStream(filePath);
ByteArrayOutputStream bos = new ByteArrayOutputStream()) {
ByteArrayOutputStream bos = new ByteArrayOutputStream()) {
byte[] buf = new byte[1024];
int len;
while ((len = in.read(buf)) != -1) {
bos.write(buf, 0, len);
}
return bos.toByteArray();
} catch (IOException e) {
}
catch (IOException e) {
logger.error("get file bytes error", e);
}
return new byte[0];
@ -190,19 +193,20 @@ public class LoggerRequestProcessor implements NettyRequestProcessor {
* @return part file content
*/
private List<String> readPartFileContent(String filePath,
int skipLine,
int limit) {
int skipLine,
int limit) {
File file = new File(filePath);
if (file.exists() && file.isFile()) {
try (Stream<String> stream = Files.lines(Paths.get(filePath))) {
return stream.skip(skipLine).limit(limit).collect(Collectors.toList());
} catch (IOException e) {
}
catch (IOException e) {
logger.error("read file error", e);
}
} else {
}
else {
logger.info("file path: {} not exists", filePath);
}
return Collections.emptyList();
}
}

8
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/MasterLogFilter.java
View File

@ -24,7 +24,8 @@ import ch.qos.logback.core.spi.FilterReply;
/**
* master log filter
*/
public class MasterLogFilter extends Filter<ILoggingEvent> {
public class MasterLogFilter
extends Filter<ILoggingEvent> {
/**
* log level
*/
@ -32,12 +33,13 @@ public class MasterLogFilter extends Filter<ILoggingEvent> {
/**
* Accept or reject based on thread name
*
* @param event event
* @return FilterReply
*/
@Override
public FilterReply decide(ILoggingEvent event) {
if (event.getThreadName().startsWith("Master-") ){
if (event.getThreadName().startsWith("Master-")) {
return FilterReply.ACCEPT;
}
return FilterReply.DENY;
@ -46,4 +48,4 @@ public class MasterLogFilter extends Filter<ILoggingEvent> {
public void setLevel(String level) {
this.level = Level.toLevel(level);
}
}
}

21
dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/log/LoggerRequestProcessorTest.java
View File

@ -17,26 +17,37 @@
package org.apache.dolphinscheduler.server.log;
import io.netty.channel.Channel;
import org.apache.commons.lang.StringUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.LoggerUtils;
import org.apache.dolphinscheduler.remote.command.Command;
import org.apache.dolphinscheduler.remote.command.CommandType;
import org.apache.dolphinscheduler.remote.command.log.ViewLogRequestCommand;
import org.junit.Before;
import org.junit.Test;
import org.junit.Test.None;
import org.junit.runner.RunWith;
import org.mockito.Mockito;
import org.powermock.api.mockito.PowerMockito;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
import io.netty.channel.Channel;
@RunWith(PowerMockRunner.class)
@PrepareForTest({LoggerUtils.class})
public class LoggerRequestProcessorTest {
private String dsHome;
@Before
public void initDsHome() {
// DOLPHINSCHEDULER_HOME is be set in start.sh. if we run test in IDE user.dir is DS Home.
dsHome = System.getProperty("DOLPHINSCHEDULER_HOME");
if (StringUtils.isBlank(dsHome)) {
dsHome = System.getProperty("user.dir");
System.setProperty("DOLPHINSCHEDULER_HOME", dsHome);
}
}
@Test
public void testProcessViewWholeLogRequest() {
System.setProperty("DOLPHINSCHEDULER_HOME", System.getProperty("user.dir"));
@ -107,4 +118,4 @@ public class LoggerRequestProcessorTest {
LoggerRequestProcessor loggerRequestProcessor = new LoggerRequestProcessor();
loggerRequestProcessor.process(channel, command);
}
}
}

41
dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/log/LoggerServerTest.java
View File

@ -17,57 +17,68 @@
package org.apache.dolphinscheduler.server.log;
import org.apache.commons.lang.StringUtils;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.utils.FileUtils;
import org.apache.dolphinscheduler.service.log.LogClientService;
import org.apache.commons.lang.StringUtils;
import java.io.File;
import java.io.IOException;
import java.nio.charset.Charset;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import java.io.File;
import java.io.IOException;
import java.nio.charset.Charset;
public class LoggerServerTest {
private LoggerServer loggerServer;
private LogClientService logClientService;
private String dsHome;
@Before
public void startServerAndClient() {
this.loggerServer = new LoggerServer();
this.loggerServer.start();
this.logClientService = new LogClientService();
// DOLPHINSCHEDULER_HOME is be set in start.sh. if we run test in IDE user.dir is DS Home.
dsHome = System.getProperty("DOLPHINSCHEDULER_HOME");
if (StringUtils.isBlank(dsHome)) {
dsHome = System.getProperty("user.dir");
System.setProperty("DOLPHINSCHEDULER_HOME", dsHome);
}
}
@Test
public void testRollViewLog() throws IOException {
public void testRollViewLog()
throws IOException {
String expectedTmpDemoString = "testRolloViewLog";
org.apache.commons.io.FileUtils.writeStringToFile(new File("/tmp/demo.txt"), expectedTmpDemoString, Charset.defaultCharset());
String testFile = dsHome + "/tmp/demo.log";
org.apache.commons.io.FileUtils.writeStringToFile(new File(testFile), expectedTmpDemoString, Charset.defaultCharset());
String resultTmpDemoString = this.logClientService.rollViewLog(
"localhost", Constants.RPC_PORT,"/tmp/demo.txt", 0, 1000);
"localhost", Constants.RPC_PORT, testFile, 0, 1000);
Assert.assertEquals(expectedTmpDemoString, resultTmpDemoString.replaceAll("[\r|\n|\t]", StringUtils.EMPTY));
FileUtils.deleteFile("/tmp/demo.txt");
FileUtils.deleteFile(testFile);
}
@Test
public void testRemoveTaskLog() throws IOException {
public void testRemoveTaskLog()
throws IOException {
String expectedTmpRemoveString = "testRemoveTaskLog";
org.apache.commons.io.FileUtils.writeStringToFile(new File("/tmp/remove.txt"), expectedTmpRemoveString, Charset.defaultCharset());
String testFile = dsHome + "/tmp/remove.log";
org.apache.commons.io.FileUtils.writeStringToFile(new File(testFile), expectedTmpRemoveString, Charset.defaultCharset());
Boolean b = this.logClientService.removeTaskLog("localhost", Constants.RPC_PORT,"/tmp/remove.txt");
Boolean b = this.logClientService.removeTaskLog("localhost", Constants.RPC_PORT, testFile);
Assert.assertTrue(b);
String result = this.logClientService.viewLog("localhost", Constants.RPC_PORT,"/tmp/demo.txt");
String result = this.logClientService.viewLog("localhost", Constants.RPC_PORT, testFile);
Assert.assertEquals(StringUtils.EMPTY, result);
}