[Fix-3262][common] When you request the URL through applicationID to get the application status, you cannot get it if Kerberos authentication is enabled (#3264)
* fix bug #3165 get resource.storage.type value toUpperCase * fix bug #3176 optimize Gets the value of this property “resource.storage.type” * fix When you request the URL through applicationID to get the application status, you cannot get it if Kerberos authentication is enabled * fix When you request the URL through applicationID to get the application status, you cannot get it if Kerberos authentication is enabled * fix When you request the URL through applicationID to get the application status, you cannot get it if Kerberos authentication is enabled * fix miss * add KerberosHttpClient fix kerberos bug * fix map init * add juint test * Extraction of public methods * Extraction of public methods * Fix code quality * Fix code quality * Fix code quality * Fix code quality * Fix code quality * Fix code quality * Fix code quality * rebuild test * rebuild test * check style * check style * check style * revert UT POM * Kerberos judgment goes up to hadoopUtils * fix merge * Remove connection pool shutdown Co-authored-by: dailidong <dailidong66@gmail.com>
This commit is contained in:
felix.wang
GitHub
parent
232a24441b
commit
5584f0cb4d
|
|
@ -417,7 +417,12 @@ public class HadoopUtils implements Closeable {
|
|||
String applicationUrl = getApplicationUrl(applicationId);
|
||||
logger.info("applicationUrl={}", applicationUrl);
|
||||
|
||||
String responseContent = HttpUtils.get(applicationUrl);
|
||||
String responseContent ;
|
||||
if (PropertyUtils.getBoolean(Constants.HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE, false)) {
|
||||
responseContent = KerberosHttpClient.get(applicationUrl);
|
||||
} else {
|
||||
responseContent = HttpUtils.get(applicationUrl);
|
||||
}
|
||||
if (responseContent != null) {
|
||||
ObjectNode jsonObject = JSONUtils.parseObject(responseContent);
|
||||
result = jsonObject.path("app").path("finalStatus").asText();
|
||||
|
|
|
|||
|
|
@ -128,40 +128,50 @@ public class HttpUtils {
|
|||
CloseableHttpClient httpclient = HttpUtils.getInstance();
|
||||
|
||||
HttpGet httpget = new HttpGet(url);
|
||||
String responseContent = null;
|
||||
CloseableHttpResponse response = null;
|
||||
return getResponseContentString(httpget,httpclient);
|
||||
}
|
||||
|
||||
try {
|
||||
response = httpclient.execute(httpget);
|
||||
//check response status is 200
|
||||
if (response.getStatusLine().getStatusCode() == 200) {
|
||||
HttpEntity entity = response.getEntity();
|
||||
if (entity != null) {
|
||||
responseContent = EntityUtils.toString(entity, Constants.UTF_8);
|
||||
}else{
|
||||
logger.warn("http entity is null");
|
||||
}
|
||||
}else{
|
||||
logger.error("http get:{} response status code is not 200!", response.getStatusLine().getStatusCode());
|
||||
}
|
||||
}catch (Exception e){
|
||||
logger.error(e.getMessage(),e);
|
||||
}finally {
|
||||
try {
|
||||
if (response != null) {
|
||||
EntityUtils.consume(response.getEntity());
|
||||
response.close();
|
||||
}
|
||||
} catch (IOException e) {
|
||||
logger.error(e.getMessage(),e);
|
||||
}
|
||||
|
||||
if (!httpget.isAborted()) {
|
||||
httpget.releaseConnection();
|
||||
httpget.abort();
|
||||
}
|
||||
/**
|
||||
* get http response content
|
||||
*
|
||||
* @param httpget httpget
|
||||
* @param httpClient httpClient
|
||||
* @return http get request response content
|
||||
*/
|
||||
public static String getResponseContentString(HttpGet httpget, CloseableHttpClient httpClient) {
|
||||
String responseContent = null;
|
||||
CloseableHttpResponse response = null;
|
||||
try {
|
||||
response = httpClient.execute(httpget);
|
||||
// check response status is 200
|
||||
if (response.getStatusLine().getStatusCode() == 200) {
|
||||
HttpEntity entity = response.getEntity();
|
||||
if (entity != null) {
|
||||
responseContent = EntityUtils.toString(entity, Constants.UTF_8);
|
||||
} else {
|
||||
logger.warn("http entity is null");
|
||||
}
|
||||
return responseContent;
|
||||
} else {
|
||||
logger.error("http get:{} response status code is not 200!", response.getStatusLine().getStatusCode());
|
||||
}
|
||||
} catch (IOException ioe) {
|
||||
logger.error(ioe.getMessage(), ioe);
|
||||
} finally {
|
||||
try {
|
||||
if (response != null) {
|
||||
EntityUtils.consume(response.getEntity());
|
||||
response.close();
|
||||
}
|
||||
} catch (IOException e) {
|
||||
logger.error(e.getMessage(), e);
|
||||
}
|
||||
if (!httpget.isAborted()) {
|
||||
httpget.releaseConnection();
|
||||
httpget.abort();
|
||||
}
|
||||
|
||||
}
|
||||
return responseContent;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,156 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.dolphinscheduler.common.utils;
|
||||
|
||||
import org.apache.dolphinscheduler.common.Constants;
|
||||
import org.apache.http.auth.AuthSchemeProvider;
|
||||
import org.apache.http.auth.AuthScope;
|
||||
import org.apache.http.auth.Credentials;
|
||||
import org.apache.http.client.config.AuthSchemes;
|
||||
import org.apache.http.client.methods.HttpGet;
|
||||
import org.apache.http.config.Lookup;
|
||||
import org.apache.http.config.RegistryBuilder;
|
||||
import org.apache.http.impl.auth.SPNegoSchemeFactory;
|
||||
import org.apache.http.impl.client.BasicCredentialsProvider;
|
||||
import org.apache.http.impl.client.CloseableHttpClient;
|
||||
import org.apache.http.impl.client.HttpClientBuilder;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import javax.security.auth.Subject;
|
||||
import javax.security.auth.kerberos.KerberosPrincipal;
|
||||
import javax.security.auth.login.AppConfigurationEntry;
|
||||
import javax.security.auth.login.Configuration;
|
||||
import javax.security.auth.login.LoginContext;
|
||||
import javax.security.auth.login.LoginException;
|
||||
import java.security.Principal;
|
||||
import java.security.PrivilegedAction;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
/**
|
||||
* kerberos http client
|
||||
*/
|
||||
public class KerberosHttpClient {
|
||||
|
||||
public static final Logger logger = LoggerFactory.getLogger(KerberosHttpClient.class);
|
||||
|
||||
private String principal;
|
||||
private String keyTabLocation;
|
||||
|
||||
public KerberosHttpClient(String principal, String keyTabLocation) {
|
||||
super();
|
||||
this.principal = principal;
|
||||
this.keyTabLocation = keyTabLocation;
|
||||
}
|
||||
|
||||
public KerberosHttpClient(String principal, String keyTabLocation, boolean isDebug) {
|
||||
this(principal, keyTabLocation);
|
||||
if (isDebug) {
|
||||
System.setProperty("sun.security.spnego.debug", "true");
|
||||
System.setProperty("sun.security.krb5.debug", "true");
|
||||
}
|
||||
}
|
||||
|
||||
public KerberosHttpClient(String principal, String keyTabLocation, String krb5Location, boolean isDebug) {
|
||||
this(principal, keyTabLocation, isDebug);
|
||||
System.setProperty("java.security.krb5.conf", krb5Location);
|
||||
}
|
||||
|
||||
private static CloseableHttpClient buildSpengoHttpClient() {
|
||||
HttpClientBuilder builder = HttpClientBuilder.create();
|
||||
Lookup<AuthSchemeProvider> authSchemeRegistry = RegistryBuilder.<AuthSchemeProvider>create()
|
||||
.register(AuthSchemes.SPNEGO, new SPNegoSchemeFactory(true)).build();
|
||||
builder.setDefaultAuthSchemeRegistry(authSchemeRegistry);
|
||||
BasicCredentialsProvider credentialsProvider = new BasicCredentialsProvider();
|
||||
credentialsProvider.setCredentials(new AuthScope(null, -1, null), new Credentials() {
|
||||
@Override
|
||||
public Principal getUserPrincipal() {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getPassword() {
|
||||
return null;
|
||||
}
|
||||
});
|
||||
builder.setDefaultCredentialsProvider(credentialsProvider);
|
||||
return builder.build();
|
||||
}
|
||||
|
||||
public String get(final String url, final String userId) {
|
||||
logger.info("Calling KerberosHttpClient {} {} {}", this.principal, this.keyTabLocation, url);
|
||||
Configuration config = new Configuration() {
|
||||
@SuppressWarnings("serial")
|
||||
@Override
|
||||
public AppConfigurationEntry[] getAppConfigurationEntry(String name) {
|
||||
Map<String, Object> options = new HashMap<>(9);
|
||||
options.put("useTicketCache", "false");
|
||||
options.put("useKeyTab", "true");
|
||||
options.put("keyTab", keyTabLocation);
|
||||
options.put("refreshKrb5Config", "true");
|
||||
options.put("principal", principal);
|
||||
options.put("storeKey", "true");
|
||||
options.put("doNotPrompt", "true");
|
||||
options.put("isInitiator", "true");
|
||||
options.put("debug", "true");
|
||||
return new AppConfigurationEntry[] {
|
||||
new AppConfigurationEntry("com.sun.security.auth.module.Krb5LoginModule",
|
||||
AppConfigurationEntry.LoginModuleControlFlag.REQUIRED, options) };
|
||||
}
|
||||
};
|
||||
Set<Principal> princ = new HashSet<>(1);
|
||||
princ.add(new KerberosPrincipal(userId));
|
||||
Subject sub = new Subject(false, princ, new HashSet<>(), new HashSet<>());
|
||||
|
||||
LoginContext lc;
|
||||
try {
|
||||
lc = new LoginContext("", sub, null, config);
|
||||
lc.login();
|
||||
Subject serviceSubject = lc.getSubject();
|
||||
return Subject.doAs(serviceSubject, (PrivilegedAction<String>) () -> {
|
||||
CloseableHttpClient httpClient = buildSpengoHttpClient();
|
||||
HttpGet httpget = new HttpGet(url);
|
||||
return HttpUtils.getResponseContentString(httpget, httpClient);
|
||||
});
|
||||
} catch (LoginException le) {
|
||||
logger.error("Kerberos authentication failed ", le);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* get http request content by kerberosClient
|
||||
*
|
||||
* @param url url
|
||||
* @return http get request response content
|
||||
*/
|
||||
public static String get(String url) {
|
||||
|
||||
String responseContent;
|
||||
KerberosHttpClient kerberosHttpClient = new KerberosHttpClient(
|
||||
PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_USERNAME),
|
||||
PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_PATH),
|
||||
PropertyUtils.getString(Constants.JAVA_SECURITY_KRB5_CONF_PATH), true);
|
||||
responseContent = kerberosHttpClient.get(url, PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_USERNAME));
|
||||
return responseContent;
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -17,7 +17,13 @@
|
|||
package org.apache.dolphinscheduler.common.utils;
|
||||
|
||||
import com.fasterxml.jackson.databind.node.ObjectNode;
|
||||
|
||||
import org.apache.dolphinscheduler.common.Constants;
|
||||
import org.apache.http.client.config.RequestConfig;
|
||||
import org.apache.http.client.methods.HttpGet;
|
||||
import org.apache.http.impl.client.CloseableHttpClient;
|
||||
import org.apache.http.impl.client.HttpClients;
|
||||
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
import org.slf4j.Logger;
|
||||
|
|
@ -28,26 +34,53 @@ import org.slf4j.LoggerFactory;
|
|||
*/
|
||||
public class HttpUtilsTest {
|
||||
|
||||
public static final Logger logger = LoggerFactory.getLogger(HttpUtilsTest.class);
|
||||
private HadoopUtils hadoopUtils = HadoopUtils.getInstance();
|
||||
|
||||
public static final Logger logger = LoggerFactory.getLogger(HttpUtilsTest.class);
|
||||
@Test
|
||||
public void testGetTest() {
|
||||
// success
|
||||
String result = HttpUtils.get("https://github.com/manifest.json");
|
||||
Assert.assertNotNull(result);
|
||||
ObjectNode jsonObject = JSONUtils.parseObject(result);
|
||||
Assert.assertEquals("GitHub", jsonObject.path("name").asText());
|
||||
result = HttpUtils.get("https://123.333.111.33/ccc");
|
||||
Assert.assertNull(result);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetByKerberos() {
|
||||
try {
|
||||
String applicationUrl = hadoopUtils.getApplicationUrl("application_1542010131334_0029");
|
||||
String responseContent;
|
||||
responseContent = HttpUtils.get(applicationUrl);
|
||||
Assert.assertNull(responseContent);
|
||||
|
||||
@Test
|
||||
public void testGetTest(){
|
||||
//success
|
||||
String result = HttpUtils.get("https://github.com/manifest.json");
|
||||
Assert.assertNotNull(result);
|
||||
ObjectNode jsonObject = JSONUtils.parseObject(result);
|
||||
Assert.assertEquals("GitHub", jsonObject.path("name").asText());
|
||||
|
||||
result = HttpUtils.get("https://123.333.111.33/ccc");
|
||||
Assert.assertNull(result);
|
||||
} catch (Exception e) {
|
||||
logger.error(e.getMessage(), e);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetResponseContentString() {
|
||||
CloseableHttpClient httpclient = HttpClients.createDefault();
|
||||
HttpGet httpget = new HttpGet("https://github.com/manifest.json");
|
||||
/** set timeout、request time、socket timeout */
|
||||
RequestConfig requestConfig = RequestConfig.custom().setConnectTimeout(Constants.HTTP_CONNECT_TIMEOUT)
|
||||
.setConnectionRequestTimeout(Constants.HTTP_CONNECTION_REQUEST_TIMEOUT)
|
||||
.setSocketTimeout(Constants.SOCKET_TIMEOUT).setRedirectsEnabled(true).build();
|
||||
httpget.setConfig(requestConfig);
|
||||
String responseContent = HttpUtils.getResponseContentString(httpget, httpclient);
|
||||
Assert.assertNotNull(responseContent);
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testGetHttpClient() {
|
||||
CloseableHttpClient httpClient1 = HttpUtils.getInstance();
|
||||
CloseableHttpClient httpClient2 = HttpUtils.getInstance();
|
||||
Assert.assertEquals(httpClient1, httpClient2);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,46 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.dolphinscheduler.common.utils;
|
||||
|
||||
import org.apache.dolphinscheduler.common.Constants;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
/**
|
||||
* KerberosHttpClient test
|
||||
*/
|
||||
public class KerberosHttpClientTest {
|
||||
public static final Logger logger = LoggerFactory.getLogger(KerberosHttpClientTest.class);
|
||||
private HadoopUtils hadoopUtils = HadoopUtils.getInstance();
|
||||
|
||||
@Test
|
||||
public void get() {
|
||||
try {
|
||||
String applicationUrl = hadoopUtils.getApplicationUrl("application_1542010131334_0029");
|
||||
String responseContent;
|
||||
KerberosHttpClient kerberosHttpClient = new KerberosHttpClient(PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_USERNAME),
|
||||
PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_PATH), PropertyUtils.getString(Constants.JAVA_SECURITY_KRB5_CONF_PATH), true);
|
||||
responseContent = kerberosHttpClient.get(applicationUrl,
|
||||
PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_USERNAME));
|
||||
Assert.assertNull(responseContent);
|
||||
} catch (Exception e) {
|
||||
logger.error(e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
}
|
||||
1
pom.xml
1
pom.xml
|
|
@ -779,6 +779,7 @@
|
|||
<include>**/common/utils/TaskParametersUtilsTest.java</include>
|
||||
<include>**/common/utils/HadoopUtilsTest.java</include>
|
||||
<include>**/common/utils/HttpUtilsTest.java</include>
|
||||
<include>**/common/utils/KerberosHttpClientTest.java</include>
|
||||
<include>**/common/ConstantsTest.java</include>
|
||||
<include>**/common/utils/HadoopUtils.java</include>
|
||||
<include>**/common/utils/RetryerUtilsTest.java</include>
|
||||
|
|
|
|||
Loading…
Reference in New Issue