natas_hw
/
dolphinscheduler
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[Fix-5795][Improvement][Server] The starttime field in the HttpTask log is not displayed as expected. (#5796) 

* improve timestamp format

make the startime in the log of httptask to be easier to read.


* fix bad code smell and update the note.
This commit is contained in:
kyoty 2021-07-12 14:31:48 +08:00 committed by GitHub
parent 16986c3c65
commit 7bffe0ac85
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 113 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

52
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/DateUtils.java
View File

@ -26,6 +26,7 @@ import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter; import java.time.format.DateTimeFormatter;
import java.util.Calendar; import java.util.Calendar;
import java.util.Date; import java.util.Date;
import java.util.Objects;
import java.util.TimeZone; import java.util.TimeZone;
import org.slf4j.Logger; import org.slf4j.Logger;
@ -36,12 +37,44 @@ import org.slf4j.LoggerFactory;
*/ */
public class DateUtils { public class DateUtils {
static final long C0 = 1L;
static final long C1 = C0 * 1000L;
static final long C2 = C1 * 1000L;
static final long C3 = C2 * 1000L;
static final long C4 = C3 * 60L;
static final long C5 = C4 * 60L;
static final long C6 = C5 * 24L;
/**
* a default datetime formatter for the timestamp
*/
private static final DateTimeFormatter DEFAULT_DATETIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
private static final Logger logger = LoggerFactory.getLogger(DateUtils.class); private static final Logger logger = LoggerFactory.getLogger(DateUtils.class);
private DateUtils() { private DateUtils() {
throw new UnsupportedOperationException("Construct DateUtils"); throw new UnsupportedOperationException("Construct DateUtils");
} }
/**
* @param timeMillis timeMillis like System.currentTimeMillis()
* @return string formatted as yyyy-MM-dd HH:mm:ss
*/
public static String formatTimeStamp(long timeMillis) {
return formatTimeStamp(timeMillis, DEFAULT_DATETIME_FORMATTER);
}
/**
* @param timeMillis timeMillis like System.currentTimeMillis()
* @param dateTimeFormatter expect formatter, like yyyy-MM-dd HH:mm:ss
* @return formatted string
*/
public static String formatTimeStamp(long timeMillis, DateTimeFormatter dateTimeFormatter) {
Objects.requireNonNull(dateTimeFormatter);
return dateTimeFormatter.format(LocalDateTime.ofInstant(Instant.ofEpochMilli(timeMillis),
ZoneId.systemDefault()));
}
/** /**
* date to local datetime * date to local datetime
* *
@ -253,7 +286,6 @@ public class DateUtils {
} }
/** /**
*
* format time to duration * format time to duration
* *
* @param d1 d1 * @param d1 d1
@ -522,14 +554,6 @@ public class DateUtils {
return TimeZone.getTimeZone(timezoneId); return TimeZone.getTimeZone(timezoneId);
} }
static final long C0 = 1L;
static final long C1 = C0 * 1000L;
static final long C2 = C1 * 1000L;
static final long C3 = C2 * 1000L;
static final long C4 = C3 * 60L;
static final long C5 = C4 * 60L;
static final long C6 = C5 * 24L;
/** /**
* Time unit representing one thousandth of a second * Time unit representing one thousandth of a second
*/ */
@ -543,23 +567,23 @@ public class DateUtils {
return d / (C4 / C2); return d / (C4 / C2);
} }
public static long toHours(long d) { public static long toHours(long d) {
return d / (C5 / C2); return d / (C5 / C2);
} }
public static long toDays(long d) { public static long toDays(long d) {
return d / (C6 / C2); return d / (C6 / C2);
} }
public static long toDurationSeconds(long d) { public static long toDurationSeconds(long d) {
return (d % (C4 / C2)) / (C3 / C2); return (d % (C4 / C2)) / (C3 / C2);
} }
public static long toDurationMinutes(long d) { public static long toDurationMinutes(long d) {
return (d % (C5 / C2)) / (C4 / C2); return (d % (C5 / C2)) / (C4 / C2);
} }
public static long toDurationHours(long d) { public static long toDurationHours(long d) {
return (d % (C6 / C2)) / (C5 / C2); return (d % (C6 / C2)) / (C5 / C2);
} }

15
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/DateUtilsTest.java
View File

@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.common.utils;
import java.text.ParseException; import java.text.ParseException;
import java.text.SimpleDateFormat; import java.text.SimpleDateFormat;
import java.time.format.DateTimeFormatter;
import java.util.Date; import java.util.Date;
import java.util.TimeZone; import java.util.TimeZone;
@ -41,6 +42,20 @@ public class DateUtilsTest {
Assert.assertEquals("01 09:23:08", readableDate); Assert.assertEquals("01 09:23:08", readableDate);
} }
@Test
public void testConvertTimeStampsToString() {
TimeZone defaultTimeZone = TimeZone.getDefault();
final TimeZone timeZone = TimeZone.getTimeZone("Asia/Shanghai");
TimeZone.setDefault(timeZone);
long timeMillis = 1625989249021L;
Assert.assertEquals("2021-07-11 15:40:49", DateUtils.formatTimeStamp(timeMillis));
DateTimeFormatter testFormatter = DateTimeFormatter.ofPattern("yyyy/MM/dd HH:mm:ss");
Assert.assertEquals("2021/07/11 15:40:49", DateUtils.formatTimeStamp(timeMillis, testFormatter));
TimeZone.setDefault(defaultTimeZone);
}
@Test @Test
public void testWeek() { public void testWeek() {
Date curr = DateUtils.stringToDate("2019-02-01 00:00:00"); Date curr = DateUtils.stringToDate("2019-02-01 00:00:00");

105
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/http/HttpTask.java
View File

@ -14,11 +14,9 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.dolphinscheduler.server.worker.task.http; package org.apache.dolphinscheduler.server.worker.task.http;
import com.fasterxml.jackson.databind.node.ObjectNode;
import org.apache.commons.io.Charsets;
import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.common.enums.CommandType;
import org.apache.dolphinscheduler.common.enums.HttpMethod; import org.apache.dolphinscheduler.common.enums.HttpMethod;
@ -27,10 +25,16 @@ import org.apache.dolphinscheduler.common.process.HttpProperty;
import org.apache.dolphinscheduler.common.process.Property; import org.apache.dolphinscheduler.common.process.Property;
import org.apache.dolphinscheduler.common.task.AbstractParameters; import org.apache.dolphinscheduler.common.task.AbstractParameters;
import org.apache.dolphinscheduler.common.task.http.HttpParameters; import org.apache.dolphinscheduler.common.task.http.HttpParameters;
import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.common.utils.DateUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.ParameterUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import org.apache.dolphinscheduler.server.entity.TaskExecutionContext; import org.apache.dolphinscheduler.server.entity.TaskExecutionContext;
import org.apache.dolphinscheduler.server.utils.ParamUtils; import org.apache.dolphinscheduler.server.utils.ParamUtils;
import org.apache.dolphinscheduler.server.worker.task.AbstractTask; import org.apache.dolphinscheduler.server.worker.task.AbstractTask;
import org.apache.commons.io.Charsets;
import org.apache.http.HttpEntity; import org.apache.http.HttpEntity;
import org.apache.http.ParseException; import org.apache.http.ParseException;
import org.apache.http.client.config.RequestConfig; import org.apache.http.client.config.RequestConfig;
@ -42,7 +46,6 @@ import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder; import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.impl.client.HttpClients; import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils; import org.apache.http.util.EntityUtils;
import org.slf4j.Logger;
import java.io.IOException; import java.io.IOException;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
@ -50,28 +53,27 @@ import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import org.slf4j.Logger;
import com.fasterxml.jackson.databind.node.ObjectNode;
/** /**
* http task * http task
*/ */
public class HttpTask extends AbstractTask { public class HttpTask extends AbstractTask {
/**
* http parameters
*/
private HttpParameters httpParameters;
/** /**
* application json * application json
*/ */
protected static final String APPLICATION_JSON = "application/json"; protected static final String APPLICATION_JSON = "application/json";
/** /**
* output * output
*/ */
protected String output; protected String output;
/**
* http parameters
*/
private HttpParameters httpParameters;
/** /**
* taskExecutionContext * taskExecutionContext
*/ */
@ -79,8 +81,9 @@ public class HttpTask extends AbstractTask {
/** /**
* constructor * constructor
* @param taskExecutionContext taskExecutionContext *
* @param logger logger * @param taskExecutionContext taskExecutionContext
* @param logger logger
*/ */
public HttpTask(TaskExecutionContext taskExecutionContext, Logger logger) { public HttpTask(TaskExecutionContext taskExecutionContext, Logger logger) {
super(taskExecutionContext, logger); super(taskExecutionContext, logger);
@ -103,27 +106,30 @@ public class HttpTask extends AbstractTask {
Thread.currentThread().setName(threadLoggerInfoName); Thread.currentThread().setName(threadLoggerInfoName);
long startTime = System.currentTimeMillis(); long startTime = System.currentTimeMillis();
String formatTimeStamp = DateUtils.formatTimeStamp(startTime);
String statusCode = null; String statusCode = null;
String body = null; String body = null;
try(CloseableHttpClient client = createHttpClient(); try (CloseableHttpClient client = createHttpClient();
CloseableHttpResponse response = sendRequest(client)) { CloseableHttpResponse response = sendRequest(client)) {
statusCode = String.valueOf(getStatusCode(response)); statusCode = String.valueOf(getStatusCode(response));
body = getResponseBody(response); body = getResponseBody(response);
exitStatusCode = validResponse(body, statusCode); exitStatusCode = validResponse(body, statusCode);
long costTime = System.currentTimeMillis() - startTime; long costTime = System.currentTimeMillis() - startTime;
logger.info("startTime: {}, httpUrl: {}, httpMethod: {}, costTime : {}Millisecond, statusCode : {}, body : {}, log : {}", logger.info("startTime: {}, httpUrl: {}, httpMethod: {}, costTime : {} milliseconds, statusCode : {}, body : {}, log : {}",
DateUtils.format2Readable(startTime), httpParameters.getUrl(),httpParameters.getHttpMethod(), costTime, statusCode, body, output); formatTimeStamp, httpParameters.getUrl(),
}catch (Exception e){ httpParameters.getHttpMethod(), costTime, statusCode, body, output);
} catch (Exception e) {
appendMessage(e.toString()); appendMessage(e.toString());
exitStatusCode = -1; exitStatusCode = -1;
logger.error("httpUrl[" + httpParameters.getUrl() + "] connection failed："+output, e); logger.error("httpUrl[" + httpParameters.getUrl() + "] connection failed：" + output, e);
throw e; throw e;
} }
} }
/** /**
* send request * send request
*
* @param client client * @param client client
* @return CloseableHttpResponse * @return CloseableHttpResponse
* @throws IOException io exception * @throws IOException io exception
@ -139,23 +145,24 @@ public class HttpTask extends AbstractTask {
CommandType.of(taskExecutionContext.getCmdTypeIfComplement()), CommandType.of(taskExecutionContext.getCmdTypeIfComplement()),
taskExecutionContext.getScheduleTime()); taskExecutionContext.getScheduleTime());
List<HttpProperty> httpPropertyList = new ArrayList<>(); List<HttpProperty> httpPropertyList = new ArrayList<>();
if(CollectionUtils.isNotEmpty(httpParameters.getHttpParams() )){ if (CollectionUtils.isNotEmpty(httpParameters.getHttpParams())) {
for (HttpProperty httpProperty: httpParameters.getHttpParams()) { for (HttpProperty httpProperty : httpParameters.getHttpParams()) {
String jsonObject = JSONUtils.toJsonString(httpProperty); String jsonObject = JSONUtils.toJsonString(httpProperty);
String params = ParameterUtils.convertParameterPlaceholders(jsonObject,ParamUtils.convert(paramsMap)); String params = ParameterUtils.convertParameterPlaceholders(jsonObject, ParamUtils.convert(paramsMap));
logger.info("http request params：{}",params); logger.info("http request params：{}", params);
httpPropertyList.add(JSONUtils.parseObject(params,HttpProperty.class)); httpPropertyList.add(JSONUtils.parseObject(params, HttpProperty.class));
} }
} }
addRequestParams(builder,httpPropertyList); addRequestParams(builder, httpPropertyList);
String requestUrl = ParameterUtils.convertParameterPlaceholders(httpParameters.getUrl(),ParamUtils.convert(paramsMap)); String requestUrl = ParameterUtils.convertParameterPlaceholders(httpParameters.getUrl(), ParamUtils.convert(paramsMap));
HttpUriRequest request = builder.setUri(requestUrl).build(); HttpUriRequest request = builder.setUri(requestUrl).build();
setHeaders(request,httpPropertyList); setHeaders(request, httpPropertyList);
return client.execute(request); return client.execute(request);
} }
/** /**
* get response body * get response body
*
* @param httpResponse http response * @param httpResponse http response
* @return response body * @return response body
* @throws ParseException parse exception * @throws ParseException parse exception
@ -174,6 +181,7 @@ public class HttpTask extends AbstractTask {
/** /**
* get status code * get status code
*
* @param httpResponse http response * @param httpResponse http response
* @return status code * @return status code
*/ */
@ -183,11 +191,12 @@ public class HttpTask extends AbstractTask {
/** /**
* valid response * valid response
* @param body body *
* @param statusCode status code * @param body body
* @param statusCode status code
* @return exit status code * @return exit status code
*/ */
protected int validResponse(String body, String statusCode){ protected int validResponse(String body, String statusCode) {
int exitStatusCode = 0; int exitStatusCode = 0;
switch (httpParameters.getHttpCheckCondition()) { switch (httpParameters.getHttpCheckCondition()) {
case BODY_CONTAINS: case BODY_CONTAINS:
@ -226,6 +235,7 @@ public class HttpTask extends AbstractTask {
/** /**
* append message * append message
*
* @param message message * @param message message
*/ */
protected void appendMessage(String message) { protected void appendMessage(String message) {
@ -239,17 +249,18 @@ public class HttpTask extends AbstractTask {
/** /**
* add request params * add request params
* @param builder buidler *
* @param httpPropertyList http property list * @param builder buidler
* @param httpPropertyList http property list
*/ */
protected void addRequestParams(RequestBuilder builder,List<HttpProperty> httpPropertyList) { protected void addRequestParams(RequestBuilder builder, List<HttpProperty> httpPropertyList) {
if(CollectionUtils.isNotEmpty(httpPropertyList)){ if (CollectionUtils.isNotEmpty(httpPropertyList)) {
ObjectNode jsonParam = JSONUtils.createObjectNode(); ObjectNode jsonParam = JSONUtils.createObjectNode();
for (HttpProperty property: httpPropertyList){ for (HttpProperty property : httpPropertyList) {
if(property.getHttpParametersType() != null){ if (property.getHttpParametersType() != null) {
if (property.getHttpParametersType().equals(HttpParametersType.PARAMETER)){ if (property.getHttpParametersType().equals(HttpParametersType.PARAMETER)) {
builder.addParameter(property.getProp(), property.getValue()); builder.addParameter(property.getProp(), property.getValue());
}else if(property.getHttpParametersType().equals(HttpParametersType.BODY)){ } else if (property.getHttpParametersType().equals(HttpParametersType.BODY)) {
jsonParam.put(property.getProp(), property.getValue()); jsonParam.put(property.getProp(), property.getValue());
} }
} }
@ -263,12 +274,13 @@ public class HttpTask extends AbstractTask {
/** /**
* set headers * set headers
* @param request request *
* @param httpPropertyList http property list * @param request request
* @param httpPropertyList http property list
*/ */
protected void setHeaders(HttpUriRequest request,List<HttpProperty> httpPropertyList) { protected void setHeaders(HttpUriRequest request, List<HttpProperty> httpPropertyList) {
if(CollectionUtils.isNotEmpty(httpPropertyList)){ if (CollectionUtils.isNotEmpty(httpPropertyList)) {
for (HttpProperty property: httpPropertyList) { for (HttpProperty property : httpPropertyList) {
if (HttpParametersType.HEADERS.equals(property.getHttpParametersType())) { if (HttpParametersType.HEADERS.equals(property.getHttpParametersType())) {
request.addHeader(property.getProp(), property.getValue()); request.addHeader(property.getProp(), property.getValue());
} }
@ -278,6 +290,7 @@ public class HttpTask extends AbstractTask {
/** /**
* create http client * create http client
*
* @return CloseableHttpClient * @return CloseableHttpClient
*/ */
protected CloseableHttpClient createHttpClient() { protected CloseableHttpClient createHttpClient() {
@ -289,6 +302,7 @@ public class HttpTask extends AbstractTask {
/** /**
* request config * request config
*
* @return RequestConfig * @return RequestConfig
*/ */
private RequestConfig requestConfig() { private RequestConfig requestConfig() {
@ -297,6 +311,7 @@ public class HttpTask extends AbstractTask {
/** /**
* create request builder * create request builder
*
* @return RequestBuilder * @return RequestBuilder
*/ */
protected RequestBuilder createRequestBuilder() { protected RequestBuilder createRequestBuilder() {