natas_hw
/
dolphinscheduler0701
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[Fix]Recovery Traffic Control Config (#10505) 

* recovery traffic config

* run ci

* update
This commit is contained in:
旺阳 2022-06-20 18:43:48 +08:00 committed by GitHub
parent 64cee03fed
commit 6e4b2e6992
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 61 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
docs/docs/en/architecture/configuration.md
View File

@ -228,6 +228,11 @@ security.authentication.ldap.password|password|LDAP password
security.authentication.ldap.user.identity-attribute|uid|LDAP user identity attribute
security.authentication.ldap.user.email-attribute|mail|LDAP user email attribute
security.authentication.ldap.user.not-exist-action|CREATE|action when LDAP user is not exist. Default CREATE: automatically create user when user not exist, DENY: deny log-in when user not exist
traffic.control.global.switch|false|traffic control global switch
traffic.control.max-global-qps-rate|300|global max request number per second
traffic.control.tenant-switch|false|traffic control tenant switch
traffic.control.default-tenant-qps-rate|10|default tenant max request number per second
traffic.control.customize-tenant-qps-rate||customize tenant max request number per second
### master.properties [master-service log config]

5
docs/docs/zh/architecture/configuration.md
View File

@ -219,6 +219,11 @@ security.authentication.ldap.password|password|LDAP密码
security.authentication.ldap.user.identity-attribute|uid|LDAP用户身份标识字段名
security.authentication.ldap.user.email-attribute|mail|LDAP邮箱字段名
security.authentication.ldap.user.not-exist-action|CREATE|当LDAP用户不存在时执行的操作。CREATE：当用户不存在时自动新建用户, DENY：当用户不存在时拒绝登陆
traffic.control.global.switch|false|流量控制全局开关
traffic.control.max-global-qps-rate|300|全局最大请求数/秒
traffic.control.tenant-switch|false|流量控制租户开关
traffic.control.default-tenant-qps-rate|10|默认租户最大请求数/秒限制
traffic.control.customize-tenant-qps-rate||自定义租户最大请求数/秒限制
## 6.master.properties [Master服务配置]
|参数 |默认值| 描述|

2
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/configuration/AppConfiguration.java
View File

@ -95,7 +95,7 @@ public class AppConfiguration implements WebMvcConfigurer {
public void addInterceptors(InterceptorRegistry registry) {
// i18n
registry.addInterceptor(localeChangeInterceptor());
if (trafficConfiguration.isTrafficGlobalControlSwitch() || trafficConfiguration.isTrafficTenantControlSwitch()) {
if (trafficConfiguration.isGlobalSwitch() || trafficConfiguration.isTenantSwitch()) {
registry.addInterceptor(createRateLimitInterceptor());
}
registry.addInterceptor(loginInterceptor())

62
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/configuration/TrafficConfiguration.java
View File

@ -17,62 +17,22 @@
package org.apache.dolphinscheduler.api.configuration;
import lombok.Data;
import java.util.HashMap;
import java.util.Map;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Configuration;
@Data
@Configuration
@ConfigurationProperties(prefix = "traffic.control")
public class TrafficConfiguration {
@Value("${traffic.control.global.switch:false}")
private boolean trafficGlobalControlSwitch;
@Value("${traffic.control.max.global.qps.rate:300}")
private Integer maxGlobalQpsRate;
@Value("${traffic.control.tenant.switch:false}")
private boolean trafficTenantControlSwitch;
@Value("${traffic.control.default.tenant.qps.rate:10}")
private Integer defaultTenantQpsRate;
@Value("#{'${traffic.control.customize.tenant.qps.rate:}'.empty?null:'${traffic.control.customize.tenant.qps.rate:}'}")
private Map<String, Integer> customizeTenantQpsRate;
public boolean isTrafficGlobalControlSwitch() {
return trafficGlobalControlSwitch;
}
public void setTrafficGlobalControlSwitch(boolean trafficGlobalControlSwitch) {
this.trafficGlobalControlSwitch = trafficGlobalControlSwitch;
}
public Integer getMaxGlobalQpsRate() {
return maxGlobalQpsRate;
}
public void setMaxGlobalQpsRate(Integer maxGlobalQpsRate) {
this.maxGlobalQpsRate = maxGlobalQpsRate;
}
public boolean isTrafficTenantControlSwitch() {
return trafficTenantControlSwitch;
}
public void setTrafficTenantControlSwitch(boolean trafficTenantControlSwitch) {
this.trafficTenantControlSwitch = trafficTenantControlSwitch;
}
public Integer getDefaultTenantQpsRate() {
return defaultTenantQpsRate;
}
public void setDefaultTenantQpsRate(Integer defaultTenantQpsRate) {
this.defaultTenantQpsRate = defaultTenantQpsRate;
}
public Map<String, Integer> getCustomizeTenantQpsRate() {
return customizeTenantQpsRate;
}
public void setCustomizeTenantQpsRate(Map<String, Integer> customizeTenantQpsRate) {
this.customizeTenantQpsRate = customizeTenantQpsRate;
}
private boolean globalSwitch;
private Integer maxGlobalQpsRate = 300;
private boolean tenantSwitch;
private Integer defaultTenantQpsRate = 10;
private Map<String, Integer> customizeTenantQpsRate = new HashMap<>();
}

6
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/interceptor/RateLimitInterceptor.java
View File

@ -72,7 +72,7 @@ public class RateLimitInterceptor implements HandlerInterceptor {
@Override
public boolean preHandle(HttpServletRequest request, HttpServletResponse response, Object handler) throws ExecutionException {
// tenant-level rate limit
if (trafficConfiguration.isTrafficTenantControlSwitch()) {
if (trafficConfiguration.isTenantSwitch()) {
String token = request.getHeader("token");
if (!StringUtils.isEmpty(token)) {
RateLimiter tenantRateLimiter = tenantRateLimiterCache.get(token);
@ -84,7 +84,7 @@ public class RateLimitInterceptor implements HandlerInterceptor {
}
}
// global rate limit
if (trafficConfiguration.isTrafficGlobalControlSwitch()) {
if (trafficConfiguration.isGlobalSwitch()) {
if (!globalRateLimiter.tryAcquire()) {
response.setStatus(HttpStatus.TOO_MANY_REQUESTS.value());
logger.warn("Too many request, reach global rate limit, current qps is {}", globalRateLimiter.getRate());
@ -96,7 +96,7 @@ public class RateLimitInterceptor implements HandlerInterceptor {
public RateLimitInterceptor(TrafficConfiguration trafficConfiguration) {
this.trafficConfiguration = trafficConfiguration;
if (trafficConfiguration.isTrafficGlobalControlSwitch()) {
if (trafficConfiguration.isGlobalSwitch()) {
this.globalRateLimiter = RateLimiter.create(trafficConfiguration.getMaxGlobalQpsRate(), 1, TimeUnit.SECONDS);
}
}

15
dolphinscheduler-api/src/main/resources/application.yaml
View File

@ -147,6 +147,21 @@ security:
# action when ldap user is not exist (supported types: CREATE,DENY)
not-exist-action: CREATE
# Traffic control, if you turn on this config, the maximum number of request/s will be limited.
# global max request number per second
# default tenant-level max request number
traffic:
control:
global-switch: false
max-global-qps-rate: 300
tenant-switch: false
default-tenant-qps-rate: 10
#customize-tenant-qps-rate:
# eg.
#tenant1: 11
#tenant2: 20
# Override by profile
---

6
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/configuration/TrafficConfigurationTest.java
View File

@ -32,7 +32,7 @@ public class TrafficConfigurationTest extends AbstractControllerTest {
@Test
public void isTrafficGlobalControlSwitch() {
Assert.assertFalse(trafficConfiguration.isTrafficGlobalControlSwitch());
Assert.assertFalse(trafficConfiguration.isGlobalSwitch());
}
@Test
@ -42,7 +42,7 @@ public class TrafficConfigurationTest extends AbstractControllerTest {
@Test
public void isTrafficTenantControlSwitch() {
Assert.assertFalse(trafficConfiguration.isTrafficTenantControlSwitch());
Assert.assertFalse(trafficConfiguration.isTenantSwitch());
}
@Test
@ -54,4 +54,4 @@ public class TrafficConfigurationTest extends AbstractControllerTest {
public void getCustomizeTenantQpsRate() {
Assert.assertTrue(MapUtils.isEmpty(trafficConfiguration.getCustomizeTenantQpsRate()));
}
}
}

8
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/interceptor/RateLimitInterceptorTest.java
View File

@ -48,7 +48,7 @@ public class RateLimitInterceptorTest {
@Test
public void testPreHandleWithTenantLevenControl() throws ExecutionException {
TrafficConfiguration trafficConfiguration = new TrafficConfiguration();
trafficConfiguration.setTrafficTenantControlSwitch(true);
trafficConfiguration.setTenantSwitch(true);
Map<String, Integer> map = new HashMap<>();
map.put("tenant1", 2);
map.put("tenant2", 2);
@ -72,8 +72,8 @@ public class RateLimitInterceptorTest {
@Test
public void testPreHandleWithGlobalControl() throws ExecutionException {
TrafficConfiguration trafficConfiguration = new TrafficConfiguration();
trafficConfiguration.setTrafficTenantControlSwitch(true);
trafficConfiguration.setTrafficGlobalControlSwitch(true);
trafficConfiguration.setTenantSwitch(true);
trafficConfiguration.setGlobalSwitch(true);
trafficConfiguration.setMaxGlobalQpsRate(3);
RateLimitInterceptor rateLimitInterceptor = new RateLimitInterceptor(trafficConfiguration);
@ -86,4 +86,4 @@ public class RateLimitInterceptorTest {
Assert.assertFalse(rateLimitInterceptor.preHandle(request, response, null));
}
}
}

14
dolphinscheduler-standalone-server/src/main/resources/application.yaml
View File

@ -105,6 +105,20 @@ security:
# action when ldap user is not exist (supported types: CREATE,DENY)
not-exist-action: CREATE
# Traffic control, if you turn on this config, the maximum number of request/s will be limited.
# global max request number per second
# default tenant-level max request number
traffic:
control:
global-switch: false
max-global-qps-rate: 300
tenant-switch: false
default-tenant-qps-rate: 10
#customize-tenant-qps-rate:
# eg.
#tenant1: 11
#tenant2: 20
master:
listen-port: 5678
# master fetch command num