natas_hw
/
dolphinscheduler0701
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[Feature-14138][Metrics] Add metrics for api server (#14177)

This commit is contained in:
Rick Cheng 2023-05-23 12:22:49 +08:00 committed by GitHub
parent 0ab2447d58
commit 5c4ba4105d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 762 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
docs/docs/en/guide/metrics/metrics.md
View File

@ -17,7 +17,7 @@ cd dolphinscheduler-meter/src/main/resources/grafana-demo
docker compose up
```
then access the `Grafana` by the url: `http://localhost/3001` for dashboards.
then access the `Grafana` by the url: `http://localhost:3001` for dashboards.
![image.png](../../../../img/metrics/metrics-master.png)
![image.png](../../../../img/metrics/metrics-worker.png)
@ -111,7 +111,11 @@ For example, you can get the master metrics by `curl http://localhost:5679/actua
### Api Server Metrics
- Currently, we have not embedded any metrics in Api Server.
- ds.api.request.count: (counter) the number of requests received by the api server
- ds.api.response.count: (counter) the number of responses received by the api server, sliced by tag `code`
- ds.api.response.time: (histogram) the response time distribution of the api server
- ds.api.resource.upload.size: (histogram) size distribution of resource files uploaded by the api server (bytes)
- ds.api.resource.download.size: (histogram) size distribution of resource files download by the api server (bytes)
### Alert Server Related

8
docs/docs/zh/guide/metrics/metrics.md
View File

@ -17,7 +17,7 @@ cd dolphinscheduler-meter/src/main/resources/grafana-demo
docker compose up
```
然后，您即可通过http://localhost/3001`链接访问`Grafana`面板。
然后，您即可通过`http://localhost:3001`链接访问`Grafana`面板。
![image.png](../../../../img/metrics/metrics-master.png)
![image.png](../../../../img/metrics/metrics-worker.png)
@ -111,7 +111,11 @@ metrics exporter端口`server.port`是在application.yaml里定义的: master: `
### Api Server指标
- 目前我们尚未提供任何Api Server指标
- ds.api.request.count: (counter) api请求次数
- ds.api.response.count: (counter) api响应次数，可由标签`code`切分
- ds.api.response.time: (histogram) api响应时间分布
- ds.api.resource.upload.size: (histogram) api上传资源文件大小的分布（bytes）
- ds.api.resource.download.size: (histogram) api下载资源文件大小的分布（bytes）
### Alert Server指标

5
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/aspect/AccessLogAspect.java
View File

@ -17,6 +17,7 @@
package org.apache.dolphinscheduler.api.aspect;
import org.apache.dolphinscheduler.api.metrics.ApiServerMetrics;
import org.apache.dolphinscheduler.common.constants.Constants;
import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils;
import org.apache.dolphinscheduler.dao.entity.User;
@ -110,7 +111,9 @@ public class AccessLogAspect {
Object ob = proceedingJoinPoint.proceed();
log.info("Call {}:{} success, cost: {}ms", requestMethod, URI, (System.currentTimeMillis() - startTime));
long costTime = System.currentTimeMillis() - startTime;
log.info("Call {}:{} success, cost: {}ms", requestMethod, URI, costTime);
ApiServerMetrics.recordApiResponseTime(costTime);
return ob;
}

14
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/interceptor/LoginHandlerInterceptor.java
View File

@ -18,6 +18,7 @@
package org.apache.dolphinscheduler.api.interceptor;
import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.metrics.ApiServerMetrics;
import org.apache.dolphinscheduler.api.security.Authenticator;
import org.apache.dolphinscheduler.common.constants.Constants;
import org.apache.dolphinscheduler.common.enums.Flag;
@ -61,6 +62,8 @@ public class LoginHandlerInterceptor implements HandlerInterceptor {
*/
@Override
public boolean preHandle(HttpServletRequest request, HttpServletResponse response, Object handler) {
ApiServerMetrics.incApiRequestCount();
// get token
String token = request.getHeader("token");
User user;
@ -96,5 +99,16 @@ public class LoginHandlerInterceptor implements HandlerInterceptor {
public void postHandle(HttpServletRequest request, HttpServletResponse response, Object handler,
ModelAndView modelAndView) throws Exception {
ThreadLocalContext.getTimezoneThreadLocal().remove();
int code = response.getStatus();
if (code >= 200 && code < 300) {
ApiServerMetrics.incApiResponse2xxCount();
} else if (code >= 300 && code < 400) {
ApiServerMetrics.incApiResponse3xxCount();
} else if (code >= 400 && code < 500) {
ApiServerMetrics.incApiResponse4xxCount();
} else if (code >= 500 && code < 600) {
ApiServerMetrics.incApiResponse5xxCount();
}
}
}

112
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/metrics/ApiServerMetrics.java Normal file
View File

@ -0,0 +1,112 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.metrics;
import lombok.experimental.UtilityClass;
import io.micrometer.core.instrument.Counter;
import io.micrometer.core.instrument.DistributionSummary;
import io.micrometer.core.instrument.Metrics;
@UtilityClass
public class ApiServerMetrics {
private final Counter apiRequestCounter =
Counter.builder("ds.api.request.count")
.description("Api request count")
.register(Metrics.globalRegistry);
private final Counter apiResponse2xxCounter =
Counter.builder("ds.api.response.count")
.tag("code", "2xx")
.description("Api 2xx response count")
.register(Metrics.globalRegistry);
private final Counter apiResponse3xxCounter =
Counter.builder("ds.api.response.count")
.tag("code", "3xx")
.description("Api 3xx response count")
.register(Metrics.globalRegistry);
private final Counter apiResponse4xxCounter =
Counter.builder("ds.api.response.count")
.tag("code", "4xx")
.description("Api 4xx response count")
.register(Metrics.globalRegistry);
private final Counter apiResponse5xxCounter =
Counter.builder("ds.api.response.count")
.tag("code", "5xx")
.description("Api 5xx response count")
.register(Metrics.globalRegistry);
private final DistributionSummary apiResourceUploadSizeDistribution =
DistributionSummary.builder("ds.api.resource.upload.size")
.baseUnit("bytes")
.publishPercentiles(0.5, 0.75, 0.95, 0.99)
.publishPercentileHistogram()
.description("size of upload resource files on api")
.register(Metrics.globalRegistry);
private final DistributionSummary apiResourceDownloadSizeDistribution =
DistributionSummary.builder("ds.api.resource.download.size")
.baseUnit("bytes")
.publishPercentiles(0.5, 0.75, 0.95, 0.99)
.publishPercentileHistogram()
.description("size of download resource files on api")
.register(Metrics.globalRegistry);
private final DistributionSummary apiResponseTimeDistribution =
DistributionSummary.builder("ds.api.response.time")
.baseUnit("milliseconds")
.publishPercentiles(0.5, 0.75, 0.95, 0.99)
.publishPercentileHistogram()
.description("response time on api")
.register(Metrics.globalRegistry);
public void incApiRequestCount() {
apiRequestCounter.increment();
}
public void incApiResponse2xxCount() {
apiResponse2xxCounter.increment();
}
public void incApiResponse3xxCount() {
apiResponse3xxCounter.increment();
}
public void incApiResponse4xxCount() {
apiResponse4xxCounter.increment();
}
public void incApiResponse5xxCount() {
apiResponse5xxCounter.increment();
}
public void recordApiResourceUploadSize(final long size) {
apiResourceUploadSizeDistribution.record(size);
}
public void recordApiResourceDownloadSize(final long size) {
apiResourceDownloadSizeDistribution.record(size);
}
public void recordApiResponseTime(final long milliseconds) {
apiResponseTimeDistribution.record(milliseconds);
}
}

11
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java
View File

@ -32,6 +32,7 @@ import org.apache.dolphinscheduler.api.dto.resources.visitor.ResourceTreeVisitor
import org.apache.dolphinscheduler.api.dto.resources.visitor.Visitor;
import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.exceptions.ServiceException;
import org.apache.dolphinscheduler.api.metrics.ApiServerMetrics;
import org.apache.dolphinscheduler.api.service.ResourcesService;
import org.apache.dolphinscheduler.api.utils.PageInfo;
import org.apache.dolphinscheduler.api.utils.RegexUtils;
@ -64,6 +65,7 @@ import org.apache.commons.lang3.StringUtils;
import java.io.File;
import java.io.IOException;
import java.nio.file.Paths;
import java.text.MessageFormat;
import java.time.LocalDateTime;
import java.time.temporal.ChronoUnit;
@ -277,8 +279,9 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe
throw new ServiceException(
String.format("upload resource: %s file: %s failed.", name, file.getOriginalFilename()));
} else
log.info("Upload resource file complete, resourceName:{}, fileName:{}.",
RegexUtils.escapeNRT(name), RegexUtils.escapeNRT(file.getOriginalFilename()));
ApiServerMetrics.recordApiResourceUploadSize(file.getSize());
log.info("Upload resource file complete, resourceName:{}, fileName:{}.",
RegexUtils.escapeNRT(name), RegexUtils.escapeNRT(file.getOriginalFilename()));
return result;
}
@ -467,6 +470,7 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe
}
}
ApiServerMetrics.recordApiResourceUploadSize(file.getSize());
return result;
}
@ -1143,6 +1147,8 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe
try {
if (storageOperate.exists(fullName)) {
content = storageOperate.vimFile(tenantCode, fullName, skipLineNum, limit);
long size = content.stream().mapToLong(String::length).sum();
ApiServerMetrics.recordApiResourceDownloadSize(size);
} else {
log.error("read file {} not exist in storage", fullName);
putMsg(result, Status.RESOURCE_FILE_NOT_EXIST, fullName);
@ -1463,6 +1469,7 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe
try {
storageOperate.download(tenantCode, fullName, localFileName, true);
ApiServerMetrics.recordApiResourceDownloadSize(java.nio.file.Files.size(Paths.get(localFileName)));
return org.apache.dolphinscheduler.api.utils.FileUtils.file2Resource(localFileName);
} catch (IOException e) {
log.error("Download resource error, the path is {}, and local filename is {}, the error message is {}",

13
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ResourcesServiceTest.java
View File

@ -48,6 +48,8 @@ import org.apache.dolphinscheduler.spi.enums.ResourceType;
import org.apache.commons.collections4.CollectionUtils;
import java.io.IOException;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.Arrays;
@ -127,6 +129,10 @@ public class ResourcesServiceTest {
private MockedStatic<PropertyUtils> mockedStaticPropertyUtils;
private MockedStatic<Paths> mockedStaticPaths;
private MockedStatic<java.nio.file.Files> filesMockedStatic;
private Throwable exception;
@BeforeEach
@ -137,6 +143,8 @@ public class ResourcesServiceTest {
Mockito.mockStatic(org.apache.dolphinscheduler.api.utils.FileUtils.class);
mockedStaticPropertyUtils = Mockito.mockStatic(PropertyUtils.class);
mockedStaticPaths = Mockito.mockStatic(Paths.class);
filesMockedStatic = Mockito.mockStatic(java.nio.file.Files.class);
}
@AfterEach
@ -145,6 +153,8 @@ public class ResourcesServiceTest {
mockedStaticFiles.close();
mockedStaticDolphinschedulerFileUtils.close();
mockedStaticPropertyUtils.close();
mockedStaticPaths.close();
filesMockedStatic.close();
}
@Test
@ -668,7 +678,10 @@ public class ResourcesServiceTest {
Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant());
Mockito.when(userMapper.selectById(1)).thenReturn(getUser());
org.springframework.core.io.Resource resourceMock = Mockito.mock(org.springframework.core.io.Resource.class);
Path path = Mockito.mock(Path.class);
Mockito.when(Paths.get(Mockito.any())).thenReturn(path);
try {
Mockito.when(java.nio.file.Files.size(Mockito.any())).thenReturn(1L);
// resource null
org.springframework.core.io.Resource resource = resourcesService.downloadResource(getUser(), "");
Assertions.assertNull(resource);

598
dolphinscheduler-meter/src/main/resources/grafana/DolphinSchedulerApi.json Normal file
View File

@ -0,0 +1,598 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": "-- Grafana --",
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"target": {
"limit": 100,
"matchAny": false,
"tags": [],
"type": "dashboard"
},
"type": "dashboard"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": 7,
"links": [],
"liveNow": false,
"panels": [
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 0
},
"id": 2,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "single"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"exemplar": true,
"expr": "sum(increase(ds_api_request_count_total{}[1m]))",
"interval": "",
"legendFormat": "",
"refId": "A"
}
],
"title": "Api Request Count/1m",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 0
},
"id": 8,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "single"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"exemplar": true,
"expr": "increase(ds_api_response_time_milliseconds{}[5m])",
"interval": "",
"legendFormat": "",
"refId": "A"
}
],
"title": "Api Response Time/5m",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 8
},
"id": 6,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "single"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"exemplar": true,
"expr": "sum(increase(ds_api_response_count_total{}[1m]))",
"interval": "",
"legendFormat": "",
"refId": "A"
}
],
"title": "Api Response Count/1m",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 8
},
"id": 4,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "single"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"exemplar": true,
"expr": "sum(increase(ds_api_response_count_total{code=\"3xx\"}[1m]))",
"hide": false,
"interval": "",
"legendFormat": "",
"refId": "B"
},
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"exemplar": true,
"expr": "sum(increase(ds_api_response_count_total{code=\"4xx\"}[1m]))",
"hide": false,
"interval": "",
"legendFormat": "",
"refId": "C"
},
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"exemplar": true,
"expr": "sum(increase(ds_api_response_count_total{code=\"5xx\"}[1m]))",
"hide": false,
"interval": "",
"legendFormat": "",
"refId": "D"
}
],
"title": "Api Abnormal Response Count/1m",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 16
},
"id": 10,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "single"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"exemplar": true,
"expr": "increase(ds_api_resource_upload_size_bytes{}[5m])",
"interval": "",
"legendFormat": "",
"refId": "A"
}
],
"title": "Api Resource Upload Size/5m",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 16
},
"id": 12,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "single"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"exemplar": true,
"expr": "increase(ds_api_resource_download_size_bytes{}[5m])",
"interval": "",
"legendFormat": "",
"refId": "A"
}
],
"title": "Api Resource Download Size/5m",
"type": "timeseries"
}
],
"schemaVersion": 34,
"style": "dark",
"tags": [],
"templating": {
"list": []
},
"time": {
"from": "now-6h",
"to": "now"
},
"timepicker": {},
"timezone": "",
"title": "Api",
"uid": "437wjZQ4k",
"version": 6,
"weekStart": ""
}