TencentOS
/
TencentOS-kernel
mirror of https://github.com/Tencent/TencentOS-kernel
0
0
Fork 4
Code Issues Projects Releases Wiki Activity

Make CFS bandwidth controller burstable 

Accumulate unused quota from previous periods, thus accumulated
bandwidth runtime can be used in the following periods. During
accumulation, take care of runtime overflow. Previous non-burstable
CFS bandwidth controller only assign quota to runtime, that saves a lot.

A sysctl parameter cpu_qos_cfs_bw_burst_enabled is introduced as a
switch for burst. It is disabled by default.

Signed-off-by: Huaixin Chang <changhuaixin@linux.alibaba.com>
Signed-off-by: Shanpei Chen <shanpeic@linux.alibaba.com>
This commit is contained in:
zgpeng 2021-12-27 15:45:42 +08:00 committed by Kaixu Xia
parent d175f17162
commit 9f2d227de5
5 changed files with 64 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
include/linux/sched/sysctl.h
View File

@ -64,6 +64,7 @@ extern unsigned int sysctl_sched_uclamp_util_max;
#ifdef CONFIG_CFS_BANDWIDTH
extern unsigned int sysctl_sched_cfs_bandwidth_slice;
extern unsigned int sysctl_sched_cfs_bw_burst_enabled;
#endif
#ifdef CONFIG_SCHED_AUTOGROUP

8
kernel/sched/core.c
View File

@ -7496,7 +7496,7 @@ static DEFINE_MUTEX(cfs_constraints_mutex);
const u64 max_cfs_quota_period = 1 * NSEC_PER_SEC; /* 1s */
static const u64 min_cfs_quota_period = 1 * NSEC_PER_MSEC; /* 1ms */
/* More than 203 days if BW_SHIFT equals 20. */
static const u64 max_cfs_runtime = MAX_BW * NSEC_PER_USEC;
const u64 max_cfs_runtime = MAX_BW * NSEC_PER_USEC;
static int __cfs_schedulable(struct task_group *tg, u64 period, u64 runtime);
@ -7560,12 +7560,12 @@ static int tg_set_cfs_bandwidth(struct task_group *tg, u64 period, u64 quota,
cfs_b->quota = quota;
cfs_b->burst = burst;
__refill_cfs_bandwidth_runtime(cfs_b);
/* Restart the period timer (if active) to handle new period expiry: */
if (runtime_enabled) {
cfs_b->buffer = min(max_cfs_runtime, quota + burst);
start_cfs_bandwidth(cfs_b);
cfs_b->max_overrun = DIV_ROUND_UP_ULL(max_cfs_runtime, quota);
cfs_b->runtime = cfs_b->quota;
start_cfs_bandwidth(cfs_b, 1);
}
raw_spin_unlock_irq(&cfs_b->lock);

56
kernel/sched/fair.c
View File

@ -117,6 +117,11 @@ int __weak arch_asym_cpu_priority(int cpu)
* (default: 5 msec, units: microseconds)
*/
unsigned int sysctl_sched_cfs_bandwidth_slice = 5000UL;
/*
* A switch for cfs bandwidth burst.
*/
unsigned int sysctl_sched_cfs_bw_burst_enabled;
#endif
static inline void update_load_add(struct load_weight *lw, unsigned long inc)
@ -4377,10 +4382,23 @@ static inline u64 sched_cfs_bandwidth_slice(void)
*
* requires cfs_b->lock
*/
void __refill_cfs_bandwidth_runtime(struct cfs_bandwidth *cfs_b)
static void __refill_cfs_bandwidth_runtime(struct cfs_bandwidth *cfs_b,
u64 overrun)
{
if (cfs_b->quota != RUNTIME_INF)
cfs_b->runtime = cfs_b->quota;
u64 refill;
if (cfs_b->quota != RUNTIME_INF) {
if (!sysctl_sched_cfs_bw_burst_enabled) {
cfs_b->runtime = cfs_b->quota;
return;
}
overrun = min(overrun, cfs_b->max_overrun);
refill = cfs_b->quota * overrun;
cfs_b->runtime += refill;
cfs_b->runtime = min(cfs_b->runtime, cfs_b->buffer);
}
}
static inline struct cfs_bandwidth *tg_cfs_bandwidth(struct task_group *tg)
@ -4402,7 +4420,7 @@ static int __assign_cfs_rq_runtime(struct cfs_bandwidth *cfs_b,
if (cfs_b->quota == RUNTIME_INF)
amount = min_amount;
else {
start_cfs_bandwidth(cfs_b);
start_cfs_bandwidth(cfs_b, 0);
if (cfs_b->runtime > 0) {
amount = min(cfs_b->runtime, min_amount);
@ -4729,7 +4747,7 @@ static int do_sched_cfs_period_timer(struct cfs_bandwidth *cfs_b, int overrun, u
if (cfs_b->idle && !throttled)
goto out_deactivate;
__refill_cfs_bandwidth_runtime(cfs_b);
__refill_cfs_bandwidth_runtime(cfs_b, overrun);
if (!throttled) {
/* mark as potentially idle for the upcoming period */
@ -4971,6 +4989,7 @@ static enum hrtimer_restart sched_cfs_slack_timer(struct hrtimer *timer)
}
extern const u64 max_cfs_quota_period;
extern const u64 max_cfs_runtime;
static enum hrtimer_restart sched_cfs_period_timer(struct hrtimer *timer)
{
@ -5000,7 +5019,18 @@ static enum hrtimer_restart sched_cfs_period_timer(struct hrtimer *timer)
new = old * 2;
if (new < max_cfs_quota_period) {
cfs_b->period = ns_to_ktime(new);
cfs_b->quota *= 2;
cfs_b->quota = min(cfs_b->quota * 2,
max_cfs_runtime);
cfs_b->buffer = min(cfs_b->quota + cfs_b->burst,
max_cfs_runtime);
/*
* Add 1 in case max_overrun becomes 0.
* 0 max_overrun will cause no runtime being
* refilled in __refill_cfs_bandwidth_runtime().
*/
cfs_b->max_overrun >>= 1;
cfs_b->max_overrun++;
pr_warn_ratelimited(
"cfs_period_timer[cpu%d]: period too short, scaling up (new cfs_period_us = %lld, cfs_quota_us = %lld)\n",
@ -5050,16 +5080,26 @@ static void init_cfs_rq_runtime(struct cfs_rq *cfs_rq)
INIT_LIST_HEAD(&cfs_rq->throttled_list);
}
void start_cfs_bandwidth(struct cfs_bandwidth *cfs_b)
void start_cfs_bandwidth(struct cfs_bandwidth *cfs_b, int init)
{
u64 overrun;
lockdep_assert_held(&cfs_b->lock);
if (cfs_b->period_active)
return;
cfs_b->period_active = 1;
hrtimer_forward_now(&cfs_b->period_timer, cfs_b->period);
overrun = hrtimer_forward_now(&cfs_b->period_timer, cfs_b->period);
hrtimer_start_expires(&cfs_b->period_timer, HRTIMER_MODE_ABS_PINNED);
/*
* When period timer stops, quota for the following period is not
* refilled, however period timer is already forwarded. We should
* accumulate quota once more than overrun here.
*/
if (!init)
__refill_cfs_bandwidth_runtime(cfs_b, overrun + 1);
}
static void destroy_cfs_bandwidth(struct cfs_bandwidth *cfs_b)

4
kernel/sched/sched.h
View File

@ -336,6 +336,7 @@ struct cfs_bandwidth {
u64 runtime;
u64 burst;
u64 buffer;
u64 max_overrun;
s64 hierarchical_quota;
u8 idle;
@ -449,8 +450,7 @@ extern void init_tg_cfs_entry(struct task_group *tg, struct cfs_rq *cfs_rq,
struct sched_entity *parent);
extern void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b);
extern void __refill_cfs_bandwidth_runtime(struct cfs_bandwidth *cfs_b);
extern void start_cfs_bandwidth(struct cfs_bandwidth *cfs_b);
extern void start_cfs_bandwidth(struct cfs_bandwidth *cfs_b, int init);
extern void unthrottle_cfs_rq(struct cfs_rq *cfs_rq);
extern void free_rt_sched_group(struct task_group *tg);

9
kernel/sysctl.c
View File

@ -627,6 +627,15 @@ static struct ctl_table kern_table[] = {
.proc_handler = proc_dointvec_minmax,
.extra1 = SYSCTL_ONE,
},
{
.procname = "cpu_qos_cfs_bw_burst_enabled",
.data = &sysctl_sched_cfs_bw_burst_enabled,
.maxlen = sizeof(unsigned int),
.mode = 0644,
.proc_handler = proc_dointvec_minmax,
.extra1 = SYSCTL_ZERO,
.extra2 = SYSCTL_ONE,
},
#endif
#if defined(CONFIG_ENERGY_MODEL) && defined(CONFIG_CPU_FREQ_GOV_SCHEDUTIL)
{