natas_hw
/
dolphinscheduler0701
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[Fix-5309][Common] Fix the wrong memory (#5312) 

* [Fix-5309][Common] Fix the wrong memory

* [Fix-5309][Common] Fix unit test and remove unused code
This commit is contained in:
Shiwen Cheng 2021-04-19 10:24:54 +08:00 committed by GitHub
parent 5ef1f731b7
commit e92e29ef9a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 32 additions and 106 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java
View File

@ -380,21 +380,11 @@ public final class Constants {
*/
public static final int DEFAULT_MASTER_CPU_LOAD = Runtime.getRuntime().availableProcessors() * 2;
/**
* master reserved memory
*/
public static final double DEFAULT_MASTER_RESERVED_MEMORY = OSUtils.totalMemorySize() / 10;
/**
* worker cpu load
*/
public static final int DEFAULT_WORKER_CPU_LOAD = Runtime.getRuntime().availableProcessors() * 2;
/**
* worker reserved memory
*/
public static final double DEFAULT_WORKER_RESERVED_MEMORY = OSUtils.totalMemorySize() / 10;
/**
* worker host weight
*/

35
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java
View File

@ -17,11 +17,8 @@
package org.apache.dolphinscheduler.common.utils;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.shell.ShellExecutor;
import org.apache.commons.configuration.Configuration;
import java.io.BufferedReader;
import java.io.FileInputStream;
import java.io.IOException;
@ -85,7 +82,7 @@ public class OSUtils {
*/
public static double memoryUsage() {
GlobalMemory memory = hal.getMemory();
double memoryUsage = (memory.getTotal() - memory.getAvailable() - memory.getSwapUsed()) * 0.1 / memory.getTotal() * 10;
double memoryUsage = (memory.getTotal() - memory.getAvailable()) * 1.0 / memory.getTotal();
DecimalFormat df = new DecimalFormat(TWO_DECIMAL);
df.setRoundingMode(RoundingMode.HALF_UP);
@ -101,12 +98,11 @@ public class OSUtils {
*/
public static double availablePhysicalMemorySize() {
GlobalMemory memory = hal.getMemory();
double availablePhysicalMemorySize = (memory.getAvailable() + memory.getSwapUsed()) / 1024.0 / 1024 / 1024;
double availablePhysicalMemorySize = memory.getAvailable() / 1024.0 / 1024 / 1024;
DecimalFormat df = new DecimalFormat(TWO_DECIMAL);
df.setRoundingMode(RoundingMode.HALF_UP);
return Double.parseDouble(df.format(availablePhysicalMemorySize));
}
/**
@ -116,13 +112,13 @@ public class OSUtils {
*
* @return available Physical Memory Size, unit: G
*/
public static double totalMemorySize() {
public static double totalPhysicalMemorySize() {
GlobalMemory memory = hal.getMemory();
double availablePhysicalMemorySize = memory.getTotal() / 1024.0 / 1024 / 1024;
double totalPhysicalMemorySize = memory.getTotal() / 1024.0 / 1024 / 1024;
DecimalFormat df = new DecimalFormat(TWO_DECIMAL);
df.setRoundingMode(RoundingMode.HALF_UP);
return Double.parseDouble(df.format(availablePhysicalMemorySize));
return Double.parseDouble(df.format(totalPhysicalMemorySize));
}
/**
@ -504,25 +500,4 @@ public class OSUtils {
}
}
/**
* check memory and cpu usage
*
* @param conf conf
* @param isMaster is master
* @return check memory and cpu usage
*/
public static Boolean checkResource(Configuration conf, Boolean isMaster) {
double systemCpuLoad;
double systemReservedMemory;
if (Boolean.TRUE.equals(isMaster)) {
systemCpuLoad = conf.getDouble(Constants.MASTER_MAX_CPULOAD_AVG, Constants.DEFAULT_MASTER_CPU_LOAD);
systemReservedMemory = conf.getDouble(Constants.MASTER_RESERVED_MEMORY, Constants.DEFAULT_MASTER_RESERVED_MEMORY);
} else {
systemCpuLoad = conf.getDouble(Constants.WORKER_MAX_CPULOAD_AVG, Constants.DEFAULT_WORKER_CPU_LOAD);
systemReservedMemory = conf.getDouble(Constants.WORKER_RESERVED_MEMORY, Constants.DEFAULT_WORKER_RESERVED_MEMORY);
}
return checkResource(systemCpuLoad, systemReservedMemory);
}
}

46
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/os/OSUtilsTest.java
View File

@ -18,13 +18,11 @@ package org.apache.dolphinscheduler.common.os;
import org.apache.dolphinscheduler.common.utils.OSUtils;
import org.junit.Assert;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import oshi.hardware.GlobalMemory;
import java.math.RoundingMode;
import java.text.DecimalFormat;
/**
@ -34,42 +32,34 @@ public class OSUtilsTest {
private static Logger logger = LoggerFactory.getLogger(OSUtilsTest.class);
@Test
public void memoryUsage() {
logger.info("memoryUsage : {}", OSUtils.memoryUsage());// 0.3361799418926239
double memoryUsage = OSUtils.memoryUsage();
logger.info("memoryUsage : {}", memoryUsage);
Assert.assertTrue(memoryUsage >= 0.0);
}
@Test
public void availablePhysicalMemorySize() {
logger.info("availablePhysicalMemorySize : {}", OSUtils.availablePhysicalMemorySize());
logger.info("availablePhysicalMemorySize : {}", OSUtils.totalMemorySize() / 10);
public void physicalMemorySize() {
double availablePhysicalMemorySize = OSUtils.availablePhysicalMemorySize();
double totalPhysicalMemorySize = OSUtils.totalPhysicalMemorySize();
logger.info("availablePhysicalMemorySize : {}", availablePhysicalMemorySize);
logger.info("totalPhysicalMemorySize : {}", totalPhysicalMemorySize);
Assert.assertTrue(availablePhysicalMemorySize >= 0.0);
Assert.assertTrue(totalPhysicalMemorySize >= 0.0);
}
@Test
public void loadAverage() {
logger.info("memoryUsage : {}", OSUtils.loadAverage());
double loadAverage = OSUtils.loadAverage();
logger.info("loadAverage : {}", loadAverage);
Assert.assertTrue(loadAverage >= 0.0);
}
private void printMemory(GlobalMemory memory) {
logger.info("memoryUsage : {} %" , (memory.getTotal() - memory.getAvailable()) * 100 / memory.getTotal() );
}
@Test
public void cpuUsage() throws Exception {
logger.info("cpuUsage : {}", OSUtils.cpuUsage());
Thread.sleep(1000L);
logger.info("cpuUsage : {}", OSUtils.cpuUsage());
public void cpuUsage() {
double cpuUsage = OSUtils.cpuUsage();
DecimalFormat df = new DecimalFormat("0.00");
df.setRoundingMode(RoundingMode.HALF_UP);
logger.info("cpuUsage1 : {}", df.format(cpuUsage));
logger.info("cpuUsage : {}", cpuUsage);
Assert.assertTrue(cpuUsage >= 0.0);
}
}

40
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/OSUtilsTest.java
View File

@ -16,11 +16,6 @@
*/
package org.apache.dolphinscheduler.common.utils;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.commons.configuration.Configuration;
import org.apache.commons.configuration.PropertiesConfiguration;
import java.io.IOException;
import java.util.List;
@ -40,12 +35,12 @@ public class OSUtilsTest {
}
@Test
public void testOSMetric(){
public void testOSMetric() {
if (!OSUtils.isWindows()) {
double availablePhysicalMemorySize = OSUtils.availablePhysicalMemorySize();
Assert.assertTrue(availablePhysicalMemorySize >= 0.0d);
double totalMemorySize = OSUtils.totalMemorySize();
Assert.assertTrue(totalMemorySize >= 0.0d);
double totalPhysicalMemorySize = OSUtils.totalPhysicalMemorySize();
Assert.assertTrue(totalPhysicalMemorySize >= 0.0d);
double loadAverage = OSUtils.loadAverage();
logger.info("loadAverage {}", loadAverage);
double memoryUsage = OSUtils.memoryUsage();
@ -92,7 +87,7 @@ public class OSUtilsTest {
@Test
public void exeCmd() {
if(OSUtils.isMacOS() || !OSUtils.isWindows()){
if (OSUtils.isMacOS() || !OSUtils.isWindows()) {
try {
String result = OSUtils.exeCmd("echo helloWorld");
Assert.assertEquals("helloWorld\n",result);
@ -102,39 +97,16 @@ public class OSUtilsTest {
}
}
@Test
public void getProcessID(){
public void getProcessID() {
int processId = OSUtils.getProcessID();
Assert.assertNotEquals(0, processId);
}
@Test
public void checkResource(){
public void checkResource() {
boolean resource = OSUtils.checkResource(100,0);
Assert.assertTrue(resource);
resource = OSUtils.checkResource(0,Double.MAX_VALUE);
Assert.assertFalse(resource);
Configuration configuration = new PropertiesConfiguration();
configuration.setProperty(Constants.MASTER_MAX_CPULOAD_AVG,100);
configuration.setProperty(Constants.MASTER_RESERVED_MEMORY,0);
resource = OSUtils.checkResource(configuration,true);
Assert.assertTrue(resource);
configuration.setProperty(Constants.MASTER_MAX_CPULOAD_AVG,0);
configuration.setProperty(Constants.MASTER_RESERVED_MEMORY,Double.MAX_VALUE);
resource = OSUtils.checkResource(configuration,true);
Assert.assertFalse(resource);
configuration.setProperty(Constants.WORKER_MAX_CPULOAD_AVG,100);
configuration.setProperty(Constants.WORKER_RESERVED_MEMORY,0);
resource = OSUtils.checkResource(configuration,false);
Assert.assertTrue(resource);
configuration.setProperty(Constants.WORKER_MAX_CPULOAD_AVG,0);
configuration.setProperty(Constants.WORKER_RESERVED_MEMORY,Double.MAX_VALUE);
resource = OSUtils.checkResource(configuration,false);
Assert.assertFalse(resource);
}
}

2
dolphinscheduler-dist/release-docs/LICENSE
View File

@ -471,7 +471,7 @@ The text of each license is also included at licenses/LICENSE-[project].txt.
aspectjweaver 1.9.2:https://mvnrepository.com/artifact/org.aspectj/aspectjweaver/1.9.2, EPL 1.0
logback-classic 1.2.3: https://mvnrepository.com/artifact/ch.qos.logback/logback-classic/1.2.3, EPL 1.0 and LGPL 2.1
logback-core 1.2.3: https://mvnrepository.com/artifact/ch.qos.logback/logback-core/1.2.3, EPL 1.0 and LGPL 2.1
oshi-core 3.5.0: https://mvnrepository.com/artifact/com.github.oshi/oshi-core/3.5.0, EPL 1.0
oshi-core 3.9.1: https://mvnrepository.com/artifact/com.github.oshi/oshi-core/3.9.1, EPL 1.0
junit 4.12: https://mvnrepository.com/artifact/junit/junit/4.12, EPL 1.0
h2-1.4.200 https://github.com/h2database/h2database/blob/master/LICENSE.txt, MPL 2.0 or EPL 1.0

2
pom.xml
View File

@ -92,7 +92,7 @@
<postgresql.version>42.1.4</postgresql.version>
<hive.jdbc.version>2.1.0</hive.jdbc.version>
<commons.io.version>2.4</commons.io.version>
<oshi.core.version>3.5.0</oshi.core.version>
<oshi.core.version>3.9.1</oshi.core.version>
<clickhouse.jdbc.version>0.1.52</clickhouse.jdbc.version>
<mssql.jdbc.version>6.1.0.jre8</mssql.jdbc.version>
<presto.jdbc.version>0.238.1</presto.jdbc.version>

3
tools/dependencies/known-dependencies.txt
View File

@ -179,7 +179,7 @@ mybatis-spring-2.0.2.jar
netty-3.6.2.Final.jar
netty-all-4.1.52.Final.jar
opencsv-2.3.jar
oshi-core-3.5.0.jar
oshi-core-3.9.1.jar
paranamer-2.3.jar
parquet-hadoop-bundle-1.8.1.jar
plexus-cipher-1.7.jar
@ -237,7 +237,6 @@ swagger-annotations-1.5.20.jar
swagger-bootstrap-ui-1.9.3.jar
swagger-models-1.5.24.jar
tephra-api-0.6.0.jar
threetenbp-1.3.6.jar
transaction-api-1.1.jar
validation-api-2.0.1.Final.jar
wagon-provider-api-2.2.jar