natas_hw
/
dolphinscheduler
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[Improvement][HTTP] support custom rendering of http body (#15531) 

Signed-off-by: Gallardot <gallardot@apache.org>
Co-authored-by: Eric Gao <ericgao.apache@gmail.com>
This commit is contained in:
Gallardot 2024-01-31 11:02:48 +08:00 committed by GitHub
parent 86ef9666c4
commit 3640a28348
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
9 changed files with 74 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpParameters.java
View File

@ -45,6 +45,11 @@ public class HttpParameters extends AbstractParameters {
*/
private List<HttpProperty> httpParams;
/**
* httpBody
*/
private String httpBody;
/**
* httpCheckCondition
*/
@ -132,4 +137,12 @@ public class HttpParameters extends AbstractParameters {
public void setSocketTimeout(int socketTimeout) {
this.socketTimeout = socketTimeout;
}
public String getHttpBody() {
return httpBody;
}
public void setHttpBody(String httpBody) {
this.httpBody = httpBody;
}
}

27
dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpTask.java
View File

@ -17,8 +17,6 @@
package org.apache.dolphinscheduler.plugin.task.http;
import static org.apache.dolphinscheduler.plugin.task.http.HttpTaskConstants.APPLICATION_JSON;
import org.apache.dolphinscheduler.common.utils.DateUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.plugin.task.api.AbstractTask;
@ -32,7 +30,6 @@ import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters
import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.io.Charsets;
import org.apache.commons.lang3.StringUtils;
import org.apache.http.HttpEntity;
import org.apache.http.ParseException;
@ -40,6 +37,7 @@ import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.client.methods.RequestBuilder;
import org.apache.http.entity.ContentType;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
@ -148,7 +146,9 @@ public class HttpTask extends AbstractTask {
httpPropertyList.add(JSONUtils.parseObject(params, HttpProperty.class));
}
}
addRequestParams(builder, httpPropertyList);
String httpBody = ParameterUtils.convertParameterPlaceholders(httpParameters.getHttpBody(),
ParameterUtils.convert(paramsMap));
addRequestParams(builder, httpPropertyList, httpBody);
String requestUrl =
ParameterUtils.convertParameterPlaceholders(httpParameters.getUrl(), ParameterUtils.convert(paramsMap));
HttpUriRequest request = builder.setUri(requestUrl).build();
@ -250,7 +250,14 @@ public class HttpTask extends AbstractTask {
* @param builder buidler
* @param httpPropertyList http property list
*/
protected void addRequestParams(RequestBuilder builder, List<HttpProperty> httpPropertyList) {
protected void addRequestParams(RequestBuilder builder, List<HttpProperty> httpPropertyList, String httpBody) {
if (StringUtils.isNotEmpty(httpBody)) {
builder.setEntity(new StringEntity(
httpBody,
ContentType.create(ContentType.APPLICATION_JSON.getMimeType(),
StandardCharsets.UTF_8)));
}
if (CollectionUtils.isNotEmpty(httpPropertyList)) {
ObjectNode jsonParam = JSONUtils.createObjectNode();
for (HttpProperty property : httpPropertyList) {
@ -262,10 +269,12 @@ public class HttpTask extends AbstractTask {
}
}
}
StringEntity postingString = new StringEntity(jsonParam.toString(), Charsets.UTF_8);
postingString.setContentEncoding(StandardCharsets.UTF_8.name());
postingString.setContentType(APPLICATION_JSON);
builder.setEntity(postingString);
if (builder.getEntity() == null) {
builder.setEntity(new StringEntity(
jsonParam.toString(),
ContentType.create(ContentType.APPLICATION_JSON.getMimeType(),
StandardCharsets.UTF_8)));
}
}
}

33
dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/test/java/org/apache/dolphinscheduler/plugin/task/http/HttpTaskTest.java
View File

@ -150,13 +150,28 @@ public class HttpTaskTest {
prepareParamsMap.put("day", "20220812");
// The MockWebServer will return the request body as response body directly
// So we just need to check if the response body contains string "20220812"
HttpTask httpTask = generateHttpTask(MOCK_DISPATCH_PATH_REQ_BODY_TO_RES_BODY, HttpMethod.POST,
HttpTask httpTask = generateHttpTask(MOCK_DISPATCH_PATH_REQ_BODY_TO_RES_BODY, HttpMethod.POST, null,
httpParams, prepareParamsMap, HttpCheckCondition.BODY_CONTAINS, "20220812",
HttpStatus.SC_OK, "");
httpTask.handle(null);
Assertions.assertEquals(EXIT_CODE_SUCCESS, httpTask.getExitStatusCode());
}
@Test
public void testHandleWithHttpBody() throws Exception {
String httpBody = "{\"day\": ${day}}";
Map<String, String> prepareParamsMap = new HashMap<>();
prepareParamsMap.put("day", "20220812");
// The MockWebServer will return the request body as response body directly
// So we just need to check if the response body contains string "20220812"
HttpTask httpTask = generateHttpTask(MOCK_DISPATCH_PATH_REQ_BODY_TO_RES_BODY, HttpMethod.POST, httpBody,
null, prepareParamsMap, HttpCheckCondition.BODY_CONTAINS, "20220812",
HttpStatus.SC_OK, "");
httpTask.handle(null);
Assertions.assertEquals(EXIT_CODE_SUCCESS, httpTask.getExitStatusCode());
}
@Test
public void testHandleWithHttpParameterParams() throws Exception {
List<HttpProperty> httpParams = new ArrayList<>();
@ -170,7 +185,7 @@ public class HttpTaskTest {
prepareParamsMap.put("day", "20220812");
// The MockWebServer will return the request parameter as response body directly
// So we just need to check if the response body contains string "20220812"
HttpTask httpTask = generateHttpTask(MOCK_DISPATCH_PATH_REQ_PARAMS_TO_RES_BODY, HttpMethod.POST,
HttpTask httpTask = generateHttpTask(MOCK_DISPATCH_PATH_REQ_PARAMS_TO_RES_BODY, HttpMethod.POST, null,
httpParams, prepareParamsMap, HttpCheckCondition.BODY_CONTAINS, "20220812",
HttpStatus.SC_OK, "");
httpTask.handle(null);
@ -203,22 +218,24 @@ public class HttpTaskTest {
}
private HttpTask generateHttpTask(HttpMethod httpMethod, int actualResponseCode) throws IOException {
return generateHttpTask("/test", httpMethod, null, null,
return generateHttpTask("/test", httpMethod, null, null, null,
HttpCheckCondition.STATUS_CODE_DEFAULT, "", actualResponseCode, "");
}
private HttpTask generateHttpTask(HttpMethod httpMethod, HttpCheckCondition httpCheckConditionType,
String condition, int actualResponseCode,
String actualResponseBody) throws IOException {
return generateHttpTask("/test", httpMethod, null, null,
return generateHttpTask("/test", httpMethod, null, null, null,
httpCheckConditionType, condition, actualResponseCode, actualResponseBody);
}
private HttpTask generateHttpTask(String mockPath, HttpMethod httpMethod, List<HttpProperty> httpParams,
private HttpTask generateHttpTask(String mockPath, HttpMethod httpMethod, String httpBody,
List<HttpProperty> httpParams,
Map<String, String> prepareParamsMap, HttpCheckCondition httpCheckConditionType,
String condition, int actualResponseCode,
String actualResponseBody) throws IOException {
String url = withMockWebServer(mockPath, actualResponseCode, actualResponseBody);
String paramData = generateHttpParameters(url, httpMethod, httpParams, httpCheckConditionType, condition);
String paramData =
generateHttpParameters(url, httpMethod, httpBody, httpParams, httpCheckConditionType, condition);
return generateHttpTaskFromParamData(paramData, prepareParamsMap);
}
@ -240,13 +257,15 @@ public class HttpTaskTest {
return httpTask;
}
private String generateHttpParameters(String url, HttpMethod httpMethod, List<HttpProperty> httpParams,
private String generateHttpParameters(String url, HttpMethod httpMethod, String httpBody,
List<HttpProperty> httpParams,
HttpCheckCondition httpCheckConditionType,
String condition) throws JsonProcessingException {
ObjectMapper mapper = new ObjectMapper();
HttpParameters httpParameters = new HttpParameters();
httpParameters.setUrl(url);
httpParameters.setHttpMethod(httpMethod);
httpParameters.setHttpBody(httpBody);
httpParameters.setHttpCheckCondition(httpCheckConditionType);
httpParameters.setCondition(condition);
httpParameters.setConnectTimeout(10000);

3
dolphinscheduler-ui/src/locales/en_US/project.ts
View File

@ -498,6 +498,9 @@ export default {
http_url_validator: 'The request address must contain HTTP or HTTPS',
http_method: 'Http Method',
http_parameters: 'Http Parameters',
http_body: 'Http Body',
http_body_tips:
'Please fill in the http body, if filled, http parameters in the body type will be ignored',
http_check_condition: 'Http Check Condition',
http_condition: 'Http Condition',
http_condition_tips: 'Please Enter Http Condition',

2
dolphinscheduler-ui/src/locales/zh_CN/project.ts
View File

@ -488,6 +488,8 @@ export default {
http_url_validator: '请求地址需包含http或者https',
http_method: '请求类型',
http_parameters: '请求参数',
http_body: '请求Body',
http_body_tips: '请填写http body,如若填写将忽略请求参数中的body类型参数',
http_check_condition: '校验条件',
http_condition: '校验内容',
http_condition_tips: '请填写校验内容',

9
dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-http.ts
View File

@ -127,6 +127,15 @@ export function useHttp(model: { [field: string]: any }): IJsonItem[] {
}
]
},
{
type: 'input',
field: 'httpBody',
name: t('project.node.http_body'),
props: {
type: 'textarea',
placeholder: t('project.node.http_body_tips')
}
},
{
type: 'select',
field: 'httpCheckCondition',

1
dolphinscheduler-ui/src/views/projects/task/components/node/format-data.ts
View File

@ -86,6 +86,7 @@ export function formatParams(data: INodeData): {
}
if (data.taskType === 'HTTP') {
taskParams.httpMethod = data.httpMethod
taskParams.httpBody = data.httpBody
taskParams.httpCheckCondition = data.httpCheckCondition
taskParams.httpParams = data.httpParams
taskParams.url = data.url

1
dolphinscheduler-ui/src/views/projects/task/components/node/tasks/use-http.ts
View File

@ -44,6 +44,7 @@ export function useHttp({
delayTime: 0,
timeout: 30,
httpMethod: 'GET',
httpBody: '',
httpCheckCondition: 'STATUS_CODE_DEFAULT',
httpParams: [],
url: '',

1
dolphinscheduler-ui/src/views/projects/task/components/node/types.ts
View File

@ -287,6 +287,7 @@ interface ITaskParams {
mainArgs?: string
others?: string
httpMethod?: string
httpBody?: string
httpCheckCondition?: string
httpParams?: []
url?: string