natas_hw
/
dolphinscheduler
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[Improvement-5527][api-server] failed find any kerberos (#5533) 

* fix failed find any kerberos.

* increase code coverage.

* update common utils test.

* update common utils test.

* update common utils test.

* fix exception type.
This commit is contained in:
zhuangchong 2021-05-23 23:43:59 +08:00 committed by GitHub
parent 46660b58ed
commit 9ba4ffbe48
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 79 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CommonUtils.java
View File

@ -91,30 +91,48 @@ public class CommonUtils {
/**
* load kerberos configuration
*
* @throws Exception errors
* @param configuration
* @return load kerberos config return true
* @throws IOException errors
*/
public static void loadKerberosConf() throws Exception {
loadKerberosConf(PropertyUtils.getString(Constants.JAVA_SECURITY_KRB5_CONF_PATH),
public static boolean loadKerberosConf(Configuration configuration) throws IOException {
return loadKerberosConf(PropertyUtils.getString(Constants.JAVA_SECURITY_KRB5_CONF_PATH),
PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_USERNAME),
PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_PATH));
PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_PATH), configuration);
}
/**
* load kerberos configuration
*
* @param javaSecurityKrb5Conf javaSecurityKrb5Conf
* @param loginUserKeytabUsername loginUserKeytabUsername
* @param loginUserKeytabPath loginUserKeytabPath
* @throws IOException errors
*/
public static void loadKerberosConf(String javaSecurityKrb5Conf, String loginUserKeytabUsername, String loginUserKeytabPath) throws IOException {
loadKerberosConf(javaSecurityKrb5Conf, loginUserKeytabUsername, loginUserKeytabPath, new Configuration());
}
/**
* load kerberos configuration
*
* @param javaSecurityKrb5Conf javaSecurityKrb5Conf
* @param loginUserKeytabUsername loginUserKeytabUsername
* @param loginUserKeytabPath loginUserKeytabPath
* @param configuration configuration
* @return load kerberos config return true
* @throws IOException errors
*/
public static boolean loadKerberosConf(String javaSecurityKrb5Conf, String loginUserKeytabUsername, String loginUserKeytabPath, Configuration configuration) throws IOException {
if (CommonUtils.getKerberosStartupState()) {
System.setProperty(Constants.JAVA_SECURITY_KRB5_CONF, StringUtils.defaultIfBlank(javaSecurityKrb5Conf, PropertyUtils.getString(Constants.JAVA_SECURITY_KRB5_CONF_PATH)));
Configuration configuration = new Configuration();
configuration.set(Constants.HADOOP_SECURITY_AUTHENTICATION, Constants.KERBEROS);
UserGroupInformation.setConfiguration(configuration);
UserGroupInformation.loginUserFromKeytab(StringUtils.defaultIfBlank(loginUserKeytabUsername, PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_USERNAME)),
StringUtils.defaultIfBlank(loginUserKeytabPath, PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_PATH)));
return true;
}
return false;
}
/**

29
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java
View File

@ -127,14 +127,8 @@ public class HadoopUtils implements Closeable {
ResUploadType resUploadType = ResUploadType.valueOf(resourceStorageType);
if (resUploadType == ResUploadType.HDFS) {
if (PropertyUtils.getBoolean(Constants.HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE, false)) {
System.setProperty(Constants.JAVA_SECURITY_KRB5_CONF,
PropertyUtils.getString(Constants.JAVA_SECURITY_KRB5_CONF_PATH));
configuration.set(Constants.HADOOP_SECURITY_AUTHENTICATION, "kerberos");
if (CommonUtils.loadKerberosConf(configuration)) {
hdfsUser = "";
UserGroupInformation.setConfiguration(configuration);
UserGroupInformation.loginUserFromKeytab(PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_USERNAME),
PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_PATH));
}
String defaultFS = configuration.get(Constants.FS_DEFAULTFS);
@ -156,20 +150,15 @@ public class HadoopUtils implements Closeable {
logger.info("get property:{} -> {}, from core-site.xml hdfs-site.xml ", Constants.FS_DEFAULTFS, defaultFS);
}
if (fs == null) {
if (StringUtils.isNotEmpty(hdfsUser)) {
UserGroupInformation ugi = UserGroupInformation.createRemoteUser(hdfsUser);
ugi.doAs(new PrivilegedExceptionAction<Boolean>() {
@Override
public Boolean run() throws Exception {
fs = FileSystem.get(configuration);
return true;
}
});
} else {
logger.warn("hdfs.root.user is not set value!");
if (StringUtils.isNotEmpty(hdfsUser)) {
UserGroupInformation ugi = UserGroupInformation.createRemoteUser(hdfsUser);
ugi.doAs((PrivilegedExceptionAction<Boolean>) () -> {
fs = FileSystem.get(configuration);
}
return true;
});
} else {
logger.warn("hdfs.root.user is not set value!");
fs = FileSystem.get(configuration);
}
} else if (resUploadType == ResUploadType.S3) {
System.setProperty(Constants.AWS_S3_V4, Constants.STRING_TRUE);

61
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/CommonUtilsTest.java
View File

@ -14,20 +14,31 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.utils;
import org.apache.dolphinscheduler.common.Constants;
import org.junit.Assert;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
import java.net.InetAddress;
import java.net.UnknownHostException;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.powermock.api.mockito.PowerMockito;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* configuration test
*/
@RunWith(PowerMockRunner.class)
@PrepareForTest(value = { PropertyUtils.class, UserGroupInformation.class})
public class CommonUtilsTest {
private static final Logger logger = LoggerFactory.getLogger(CommonUtilsTest.class);
@Test
@ -35,21 +46,43 @@ public class CommonUtilsTest {
logger.info(CommonUtils.getSystemEnvPath());
Assert.assertTrue(true);
}
@Test
public void isDevelopMode() {
logger.info("develop mode: {}",CommonUtils.isDevelopMode());
Assert.assertTrue(true);
}
@Test
public void getKerberosStartupState(){
logger.info("kerberos startup state: {}",CommonUtils.getKerberosStartupState());
Assert.assertTrue(true);
public void getKerberosStartupState() {
boolean kerberosStartupState = CommonUtils.getKerberosStartupState();
logger.info("kerberos startup state: {}",kerberosStartupState);
Assert.assertFalse(kerberosStartupState);
PowerMockito.mockStatic(PropertyUtils.class);
PowerMockito.when(PropertyUtils.getUpperCaseString(Constants.RESOURCE_STORAGE_TYPE)).thenReturn("HDFS");
PowerMockito.when(PropertyUtils.getBoolean(Constants.HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE, false)).thenReturn(Boolean.TRUE);
kerberosStartupState = CommonUtils.getKerberosStartupState();
logger.info("kerberos startup state: {}",kerberosStartupState);
Assert.assertTrue(kerberosStartupState);
}
@Test
public void loadKerberosConf(){
public void loadKerberosConf() {
try {
CommonUtils.loadKerberosConf();
Assert.assertTrue(true);
PowerMockito.mockStatic(PropertyUtils.class);
PowerMockito.when(PropertyUtils.getUpperCaseString(Constants.RESOURCE_STORAGE_TYPE)).thenReturn("HDFS");
PowerMockito.when(PropertyUtils.getBoolean(Constants.HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE, false)).thenReturn(Boolean.TRUE);
PowerMockito.when(PropertyUtils.getString(Constants.JAVA_SECURITY_KRB5_CONF_PATH)).thenReturn("/opt/krb5.conf");
PowerMockito.when(PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_USERNAME)).thenReturn("hdfs-mycluster@ESZ.COM");
PowerMockito.when(PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_PATH)).thenReturn("/opt/hdfs.headless.keytab");
PowerMockito.mockStatic(UserGroupInformation.class);
boolean result = CommonUtils.loadKerberosConf(new Configuration());
Assert.assertTrue(result);
CommonUtils.loadKerberosConf(null, null, null);
} catch (Exception e) {
Assert.fail("load Kerberos Conf failed");
}
@ -80,11 +113,11 @@ public class CommonUtilsTest {
}
@Test
public void test(){
InetAddress IP = null;
public void test() {
InetAddress ip;
try {
IP = InetAddress.getLocalHost();
logger.info(IP.getHostAddress());
ip = InetAddress.getLocalHost();
logger.info(ip.getHostAddress());
} catch (UnknownHostException e) {
e.printStackTrace();
}