natas_hw
/
dolphinscheduler
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix singe resource manager will NPE (#3013) 

* fix worker group config no effect

* remove codehaus janino jar
the license about janino maybe not compatiable with Apache v2

* Merge remote-tracking branch 'upstream/dev-1.3.0' into dev-1.3.0

# Conflicts:
#	dolphinscheduler-server/src/main/resources/config/install_config.conf

* datasource config

* Update datasource.properties

* fix RunConfig bug

* remove param monitor server state

* fix table T_DS_ALERT

* update h2 database

* fix #2910 master server will show exception for some time when it restart

* fix oom when no master is active

* fix worker oom when master server restart

* fix oom

* fix

* add UT

* fix worker group config no effect

* fix singe resource manager will NPE

* Update HadoopUtilsTest.java
This commit is contained in:
dailidong 2020-06-19 14:45:48 +08:00 committed by GitHub
parent 08beae8358
commit 620ecffa72
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 14 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java
View File

@ -186,7 +186,7 @@ public class HadoopUtils implements Closeable {
* @param applicationId application id
* @return url of application
*/
public String getApplicationUrl(String applicationId) {
public String getApplicationUrl(String applicationId) throws Exception {
/**
* if rmHaIds contains xx, it signs not use resourcemanager
* otherwise:
@ -194,21 +194,21 @@ public class HadoopUtils implements Closeable {
* if rmHaIds not empty: resourcemanager HA enabled
*/
String appUrl = "";
//not use resourcemanager
if (rmHaIds.contains(Constants.YARN_RESOURCEMANAGER_HA_XX)){
yarnEnabled = false;
logger.warn("should not step here");
} else if (!StringUtils.isEmpty(rmHaIds)) {
if (StringUtils.isEmpty(rmHaIds)){
//single resourcemanager enabled
appUrl = appAddress;
yarnEnabled = true;
} else {
//resourcemanager HA enabled
appUrl = getAppAddress(appAddress, rmHaIds);
yarnEnabled = true;
logger.info("application url : {}", appUrl);
} else {
//single resourcemanager enabled
yarnEnabled = true;
}
if(StringUtils.isBlank(appUrl)){
throw new Exception("application url is blank");
}
return String.format(appUrl, applicationId);
}
@ -402,7 +402,7 @@ public class HadoopUtils implements Closeable {
* @return the return may be null or there may be other parse exceptions
* @throws JSONException json exception
*/
public ExecutionStatus getApplicationStatus(String applicationId) throws JSONException {
public ExecutionStatus getApplicationStatus(String applicationId) throws Exception {
if (StringUtils.isEmpty(applicationId)) {
return null;
}

2
dolphinscheduler-common/src/main/resources/common.properties
View File

@ -54,7 +54,7 @@ fs.s3a.access.key=A3DXS30FO22544RE
# if resource.storage.type=S3，s3 secret key
fs.s3a.secret.key=OloCLq3n+8+sdPHUhJ21XrSxTC+JK
# if not use hadoop resourcemanager, please keep default value; if resourcemanager HA enable, please type the HA ips ; if resourcemanager is single, make this value empty
# if resourcemanager HA enable, please type the HA ips ; if resourcemanager is single, make this value empty
yarn.resourcemanager.ha.rm.ids=192.168.xx.xx,192.168.xx.xx
# if resourcemanager HA enable or not use resourcemanager, please keep the default value; If resourcemanager is single, you only need to replace ds1 to actual resourcemanager hostname.

4
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HadoopUtilsTest.java
View File

@ -142,7 +142,7 @@ public class HadoopUtilsTest {
@Test
public void isYarnEnabled() {
boolean result = hadoopUtils.isYarnEnabled();
Assert.assertEquals(false, result);
Assert.assertEquals(true, result);
}
@Test
@ -185,7 +185,7 @@ public class HadoopUtilsTest {
}
@Test
public void getApplicationUrl(){
public void getApplicationUrl() throws Exception {
String application_1516778421218_0042 = hadoopUtils.getApplicationUrl("application_1529051418016_0167");
logger.info(application_1516778421218_0042);
}

2
dolphinscheduler-server/src/main/resources/config/install_config.conf
View File

@ -87,7 +87,7 @@ s3Endpoint="http://192.168.xx.xx:9010"
s3AccessKey="xxxxxxxxxx"
s3SecretKey="xxxxxxxxxx"
# if not use hadoop resourcemanager, please keep default value; if resourcemanager HA enable, please type the HA ips ; if resourcemanager is single, make this value empty
# if resourcemanager HA enable, please type the HA ips ; if resourcemanager is single, make this value empty
yarnHaIps="192.168.xx.xx,192.168.xx.xx"
# if resourcemanager HA enable or not use resourcemanager, please skip this value setting; If resourcemanager is single, you only need to replace yarnIp1 to actual resourcemanager hostname.