natas_hw
/
dolphinscheduler
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Adapting partial code(file name start with F) to the sonar cloud rule (#2045) 

* Adapting partial code(file name start with F) to the sonar cloud rule

* add more unit test

* add License

* add includes configuration to maven-surefire-plugin

* fix getResourceFilesList incorrect logic
This commit is contained in:
gabry.wu 2020-03-10 18:59:45 +08:00 committed by GitHub
parent 9224b49b58
commit ebf7a980d2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 97 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/FuncUtils.java
View File

@ -20,7 +20,7 @@ import org.apache.dolphinscheduler.common.utils.StringUtils;
public class FuncUtils {
static public String mkString(Iterable<String> list, String split) {
public static String mkString(Iterable<String> list, String split) {
if (null == list || StringUtils.isEmpty(split)){
return null;

2
dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/FuncUtilsTest.java
View File

@ -46,7 +46,7 @@ public class FuncUtilsTest {
logger.info(result);
//Expected result string
assertEquals(result, "user1|user2|user3");
assertEquals("user1|user2|user3", result);
//Null list expected return null
result = FuncUtils.mkString(null, split);

14
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/flink/FlinkParameters.java
View File

@ -20,6 +20,7 @@ import org.apache.dolphinscheduler.common.enums.ProgramType;
import org.apache.dolphinscheduler.common.process.ResourceInfo;
import org.apache.dolphinscheduler.common.task.AbstractParameters;
import java.util.Collections;
import java.util.List;
import java.util.stream.Collectors;
@ -207,12 +208,15 @@ public class FlinkParameters extends AbstractParameters {
@Override
public List<String> getResourceFilesList() {
if(resourceList !=null ) {
this.resourceList.add(mainJar);
return resourceList.stream()
.map(p -> p.getRes()).collect(Collectors.toList());
if(resourceList != null ) {
List<String> resourceFiles = resourceList.stream()
.map(ResourceInfo::getRes).collect(Collectors.toList());
if(mainJar != null) {
resourceFiles.add(mainJar.getRes());
}
return resourceFiles;
}
return null;
return Collections.emptyList();
}

6
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java
View File

@ -44,7 +44,7 @@ public class FileUtils {
String fileSuffix = "";
if (StringUtils.isNotEmpty(filename)) {
int lastIndex = filename.lastIndexOf(".");
int lastIndex = filename.lastIndexOf('.');
if (lastIndex > 0) {
fileSuffix = filename.substring(lastIndex + 1);
}
@ -325,10 +325,8 @@ public class FileUtils {
}
} else {
File parent = file.getParentFile();
if (parent != null) {
if (!parent.mkdirs() && !parent.isDirectory()) {
if (parent != null && !parent.mkdirs() && !parent.isDirectory()) {
throw new IOException("Directory '" + parent + "' could not be created");
}
}
}
return new FileOutputStream(file, append);

55
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/task/FlinkParametersTest.java Normal file
View File

@ -0,0 +1,55 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.task;
import org.apache.dolphinscheduler.common.process.ResourceInfo;
import org.apache.dolphinscheduler.common.task.flink.FlinkParameters;
import org.junit.Assert;
import org.junit.Test;
import java.util.LinkedList;
import java.util.List;
public class FlinkParametersTest {
@Test
public void getResourceFilesList() {
FlinkParameters flinkParameters = new FlinkParameters();
Assert.assertNotNull(flinkParameters.getResourceFilesList());
Assert.assertTrue(flinkParameters.getResourceFilesList().isEmpty());
ResourceInfo mainResource = new ResourceInfo();
mainResource.setRes("testFlinkMain-1.0.0-SNAPSHOT.jar");
flinkParameters.setMainJar(mainResource);
List<ResourceInfo> resourceInfos = new LinkedList<>();
ResourceInfo resourceInfo1 = new ResourceInfo();
resourceInfo1.setRes("testFlinkParameters1.jar");
resourceInfos.add(resourceInfo1);
flinkParameters.setResourceList(resourceInfos);
Assert.assertNotNull(flinkParameters.getResourceFilesList());
Assert.assertEquals(2, flinkParameters.getResourceFilesList().size());
ResourceInfo resourceInfo2 = new ResourceInfo();
resourceInfo2.setRes("testFlinkParameters2.jar");
resourceInfos.add(resourceInfo2);
flinkParameters.setResourceList(resourceInfos);
Assert.assertNotNull(flinkParameters.getResourceFilesList());
Assert.assertEquals(3, flinkParameters.getResourceFilesList().size());
}
}

17
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/FileUtilsTest.java
View File

@ -30,29 +30,32 @@ public class FileUtilsTest {
@Test
public void suffix() {
Assert.assertEquals(FileUtils.suffix("ninfor.java"),"java");
Assert.assertEquals("java", FileUtils.suffix("ninfor.java"));
Assert.assertEquals("", FileUtils.suffix(null));
Assert.assertEquals("", FileUtils.suffix(""));
Assert.assertEquals("", FileUtils.suffix("ninfor-java"));
}
@Test
public void testGetDownloadFilename() {
PowerMockito.mockStatic(DateUtils.class);
PowerMockito.when(DateUtils.getCurrentTime(YYYYMMDDHHMMSS)).thenReturn("20190101101059");
Assert.assertEquals(FileUtils.getDownloadFilename("test"),
"/tmp/dolphinscheduler/download/20190101101059/test");
Assert.assertEquals("/tmp/dolphinscheduler/download/20190101101059/test",
FileUtils.getDownloadFilename("test"));
}
@Test
public void testGetUploadFilename() {
Assert.assertEquals(FileUtils.getUploadFilename("aaa","bbb"),
"/tmp/dolphinscheduler/aaa/resources/bbb");
Assert.assertEquals("/tmp/dolphinscheduler/aaa/resources/bbb",
FileUtils.getUploadFilename("aaa","bbb"));
}
@Test
public void testGetProcessExecDir() {
String dir = FileUtils.getProcessExecDir(1,2,3, 4);
Assert.assertEquals(dir, "/tmp/dolphinscheduler/exec/process/1/2/3/4");
Assert.assertEquals("/tmp/dolphinscheduler/exec/process/1/2/3/4", dir);
dir = FileUtils.getProcessExecDir(1,2,3);
Assert.assertEquals(dir, "/tmp/dolphinscheduler/exec/process/1/2/3");
Assert.assertEquals("/tmp/dolphinscheduler/exec/process/1/2/3", dir);
}
@Test

16
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/FlinkArgsUtils.java
View File

@ -17,12 +17,11 @@
package org.apache.dolphinscheduler.server.utils;
import org.apache.commons.lang.StringUtils;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.ProgramType;
import org.apache.dolphinscheduler.common.process.ResourceInfo;
import org.apache.dolphinscheduler.common.task.flink.FlinkParameters;
import org.apache.commons.lang.StringUtils;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.List;
@ -32,12 +31,7 @@ import java.util.List;
* spark args utils
*/
public class FlinkArgsUtils {
/**
* logger of FlinkArgsUtils
*/
private static final org.slf4j.Logger logger = LoggerFactory.getLogger(FlinkArgsUtils.class);
private static final String LOCAL_DEPLOY_MODE = "local";
/**
* build args
* @param param flink parameters
@ -52,7 +46,7 @@ public class FlinkArgsUtils {
deployMode = tmpDeployMode;
}
if (!"local".equals(deployMode)) {
if (!LOCAL_DEPLOY_MODE.equals(deployMode)) {
args.add(Constants.FLINK_RUN_MODE); //-m
args.add(Constants.FLINK_YARN_CLUSTER); //yarn-cluster
@ -113,12 +107,12 @@ public class FlinkArgsUtils {
String queue = param.getQueue();
if (StringUtils.isNotEmpty(others)) {
if (!others.contains(Constants.FLINK_QUEUE) && StringUtils.isNotEmpty(queue) && !deployMode.equals("local")) {
if (!others.contains(Constants.FLINK_QUEUE) && StringUtils.isNotEmpty(queue) && !deployMode.equals(LOCAL_DEPLOY_MODE)) {
args.add(Constants.FLINK_QUEUE);
args.add(param.getQueue());
}
args.add(others);
} else if (StringUtils.isNotEmpty(queue) && !deployMode.equals("local")) {
} else if (StringUtils.isNotEmpty(queue) && !deployMode.equals(LOCAL_DEPLOY_MODE)) {
args.add(Constants.FLINK_QUEUE);
args.add(param.getQueue());
}

4
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskScheduleThread.java
View File

@ -296,9 +296,7 @@ public class TaskScheduleThread implements Runnable {
if (baseParam != null) {
List<String> projectResourceFiles = baseParam.getResourceFilesList();
if (projectResourceFiles != null) {
projectFiles.addAll(projectResourceFiles);
}
projectFiles.addAll(projectResourceFiles);
}
return new ArrayList<>(projectFiles);

24
dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/FlinkArgsUtilsTest.java
View File

@ -87,35 +87,35 @@ public class FlinkArgsUtilsTest {
}
//Expected values and order
assertEquals(result.size(),20);
assertEquals(20, result.size());
assertEquals(result.get(0),"-m");
assertEquals(result.get(1),"yarn-cluster");
assertEquals("-m", result.get(0));
assertEquals("yarn-cluster", result.get(1));
assertEquals(result.get(2),"-ys");
assertEquals("-ys", result.get(2));
assertSame(Integer.valueOf(result.get(3)),slot);
assertEquals(result.get(4),"-ynm");
assertEquals("-ynm",result.get(4));
assertEquals(result.get(5),appName);
assertEquals(result.get(6),"-yn");
assertEquals("-yn", result.get(6));
assertSame(Integer.valueOf(result.get(7)),taskManager);
assertEquals(result.get(8),"-yjm");
assertEquals("-yjm", result.get(8));
assertEquals(result.get(9),jobManagerMemory);
assertEquals(result.get(10),"-ytm");
assertEquals("-ytm", result.get(10));
assertEquals(result.get(11),taskManagerMemory);
assertEquals(result.get(12),"-d");
assertEquals("-d", result.get(12));
assertEquals(result.get(13),"-c");
assertEquals("-c", result.get(13));
assertEquals(result.get(14),mainClass);
assertEquals(result.get(15),mainJar.getRes());
assertEquals(result.get(16),mainArgs);
assertEquals(result.get(17),"--qu");
assertEquals("--qu", result.get(17));
assertEquals(result.get(18),queue);
assertEquals(result.get(19),others);
@ -125,7 +125,7 @@ public class FlinkArgsUtilsTest {
param1.setQueue(queue);
param1.setDeployMode(mode);
result = FlinkArgsUtils.buildArgs(param1);
assertEquals(result.size(),5);
assertEquals(5, result.size());
}
}

1
pom.xml
View File

@ -691,6 +691,7 @@
<include>**/common/threadutils/*.java</include>
<include>**/common/graph/*.java</include>
<include>**/common/queue/*.java</include>
<include>**/common/task/FlinkParametersTest.java</include>
<include>**/common/task/SqoopParameterEntityTest.java</include>
<include>**/api/utils/CheckUtilsTest.java</include>
<include>**/api/utils/FileUtilsTest.java</include>