From de8ae6bfd23601bc5a1d40eb35efad86e30d30c6 Mon Sep 17 00:00:00 2001 From: He Chen Date: Thu, 11 Jun 2020 10:46:30 +0800 Subject: [PATCH] sched/bt: cgroup support and ignore cpu binding for offline task This commit add cpu.offline to cpu cgroup, echo 1 > cpu.offline would convert all tasks under this cgroup to offline task. Beside, a new sysctl sysctl_sched_bt_ignore_cpubind is added, which makes offline tasks ignoring CPU binding and running on any CPU. Signed-off-by: Xiaoming Gao Signed-off-by: Hua Liu Signed-off-by: Xiaogguang Chen Signed-off-by: Zhiguang Peng Signed-off-by: Bin Fan Signed-off-by: He Chen --- include/linux/cgroup-defs.h | 1 + include/linux/sched/sysctl.h | 1 + init/main.c | 2 + kernel/cgroup/cgroup.c | 9 ++ kernel/sched/autogroup.c | 10 +++ kernel/sched/batch.c | 161 +++++++++++++++++++++++++++++------ kernel/sched/core.c | 118 ++++++++++++++++++++++++- kernel/sched/cpuacct.c | 2 + kernel/sched/sched.h | 3 + kernel/sysctl.c | 9 ++ 10 files changed, 289 insertions(+), 27 deletions(-) diff --git a/include/linux/cgroup-defs.h b/include/linux/cgroup-defs.h index 973b07250..b5ebe1cf4 100644 --- a/include/linux/cgroup-defs.h +++ b/include/linux/cgroup-defs.h @@ -92,6 +92,7 @@ enum { CFTYPE_WORLD_WRITABLE = (1 << 4), /* (DON'T USE FOR NEW FILES) S_IWUGO */ #ifdef CONFIG_BT_SCHED CFTYPE_BT_SHARES = (1 << 5), /* BT-sched shares */ + CFTYPE_BT_PRIVATE = (1 << 6), #endif /* internal flags, do not use outside cgroup core proper */ diff --git a/include/linux/sched/sysctl.h b/include/linux/sched/sysctl.h index f9132921d..ef66c628f 100644 --- a/include/linux/sched/sysctl.h +++ b/include/linux/sched/sysctl.h @@ -60,6 +60,7 @@ extern int sysctl_sched_rt_runtime; #ifdef CONFIG_BT_SCHED extern unsigned int sysctl_sched_bt_period; extern int sysctl_sched_bt_runtime; +extern unsigned int sysctl_sched_bt_ignore_cpubind; extern unsigned int sysctl_idle_balance_bt_cost; extern unsigned int sysctl_sched_bt_granularity_ns; extern unsigned int sysctl_sched_bt_load_fair; diff --git a/init/main.c b/init/main.c index 2254d098f..fcfb037f7 100644 --- a/init/main.c +++ b/init/main.c @@ -100,6 +100,7 @@ static int kernel_init(void *); extern void init_IRQ(void); extern void fork_init(void); extern void radix_tree_init(void); +extern void init_offline_cpu_control(void); /* * Debug helper: via this flag we know that we are in 'early bootup code' @@ -1007,6 +1008,7 @@ static int __ref kernel_init(void *unused) int ret; kernel_init_freeable(); + init_offline_cpu_control(); /* need to finish all async __init code before freeing the memory */ async_synchronize_full(); ftrace_free_init_mem(); diff --git a/kernel/cgroup/cgroup.c b/kernel/cgroup/cgroup.c index 21bbfc09e..439cfead9 100644 --- a/kernel/cgroup/cgroup.c +++ b/kernel/cgroup/cgroup.c @@ -54,6 +54,7 @@ #include #include #include +#include #include #define CREATE_TRACE_POINTS @@ -80,6 +81,8 @@ EXPORT_SYMBOL_GPL(cgroup_mutex); EXPORT_SYMBOL_GPL(css_set_lock); #endif +extern unsigned int offlinegroup_enabled; + /* * Protects cgroup_idr and css_idr so that IDs can be released without * grabbing cgroup_mutex. @@ -3526,6 +3529,12 @@ restart: continue; if ((cft->flags & CFTYPE_ONLY_ON_ROOT) && cgroup_parent(cgrp)) continue; + if ((cft->flags & CFTYPE_BT_SHARES) && !sched_bt_on) + continue; + if ((cft->flags & CFTYPE_BT_PRIVATE) && !sched_bt_on) + continue; + if ((cft->flags & CFTYPE_BT_PRIVATE) && !offlinegroup_enabled) + continue; if (is_add) { ret = cgroup_add_file(css, cgrp, cft); diff --git a/kernel/sched/autogroup.c b/kernel/sched/autogroup.c index 1d179ab9e..8b6ccfab0 100644 --- a/kernel/sched/autogroup.c +++ b/kernel/sched/autogroup.c @@ -5,11 +5,13 @@ #include #include #include +#include #include #include #include unsigned int __read_mostly sysctl_sched_autogroup_enabled = 1; +unsigned int __read_mostly offlinegroup_enabled = 0; static struct autogroup autogroup_default; static atomic_t autogroup_seq_nr; @@ -207,6 +209,14 @@ static int __init setup_autogroup(char *str) __setup("noautogroup", setup_autogroup); +static int __init setup_offlinegroup(char *str) +{ + if (sched_bt_on) + offlinegroup_enabled = 1; + return 1; +} +__setup("offline_group", setup_offlinegroup); + #ifdef CONFIG_PROC_FS int proc_sched_autogroup_set_nice(struct task_struct *p, int nice) diff --git a/kernel/sched/batch.c b/kernel/sched/batch.c index 5a36238d2..4bf2c4afe 100644 --- a/kernel/sched/batch.c +++ b/kernel/sched/batch.c @@ -19,6 +19,10 @@ #include #include #include +#include +#include +#include +#include #include #include "sched.h" @@ -35,10 +39,13 @@ void set_bt_load_weight(struct task_struct *p) load->inv_weight = sched_prio_to_wmult[prio]; } +extern unsigned int offlinegroup_enabled; const struct sched_class bt_sched_class; unsigned int sysctl_idle_balance_bt_cost = 300000UL; unsigned int sysctl_sched_bt_granularity_ns = 4000000; unsigned int sysctl_sched_bt_load_fair = 1; +void * bt_cpu_control_set = 0; +unsigned int sysctl_sched_bt_ignore_cpubind = 0; /* * sd_lb_stats_bt - Structure to store the statistics of a sched_domain @@ -385,7 +392,8 @@ static void start_bandwidth_timer(struct hrtimer *period_timer, ktime_t period) static void start_bt_bandwidth(struct bt_bandwidth *bt_b) { - if (!bt_bandwidth_enabled() || bt_b->bt_runtime == RUNTIME_INF) + if (!offlinegroup_enabled && + (!bt_bandwidth_enabled() || bt_b->bt_runtime == RUNTIME_INF)) return; if (hrtimer_active(&bt_b->bt_period_timer)) @@ -636,7 +644,7 @@ static int balance_bt_runtime(struct bt_rq *bt_rq) { int more = 0; - if (!sched_feat(BT_RUNTIME_SHARE)) + if (offlinegroup_enabled || !sched_feat(BT_RUNTIME_SHARE)) return more; if (bt_rq->bt_time > bt_rq->bt_runtime) { @@ -797,7 +805,8 @@ static int do_sched_bt_period_timer(struct bt_bandwidth *bt_b, int overrun) raw_spin_unlock(&rq->lock); } - if (!throttled && (!bt_bandwidth_enabled() || bt_b->bt_runtime == RUNTIME_INF)) + if (!throttled && !offlinegroup_enabled && + (!bt_bandwidth_enabled() || bt_b->bt_runtime == RUNTIME_INF)) idle = 1; if (idle) @@ -829,22 +838,26 @@ static int sched_bt_runtime_exceeded(struct bt_rq *bt_rq) * Don't actually throttle groups that have no runtime assigned * but accrue some time due to boosting. */ - if (likely(bt_b->bt_runtime)) { - static bool once = false; + if (!offlinegroup_enabled) { + if (likely(bt_b->bt_runtime)) { + static bool once = false; - throttle_bt_rq(bt_rq); + throttle_bt_rq(bt_rq); - if (!once) { - once = true; - printk_deferred("sched: BT throttling activated\n"); + if (!once) { + once = true; + printk_deferred("sched: BT throttling activated\n"); + } + } else { + /* + * In case we did anyway, make it go away, + * replenishment is a joke, since it will replenish us + * with exactly 0 ns. + */ + bt_rq->bt_time = 0; } } else { - /* - * In case we did anyway, make it go away, - * replenishment is a joke, since it will replenish us - * with exactly 0 ns. - */ - bt_rq->bt_time = 0; + throttle_bt_rq(bt_rq); } if (bt_rq_throttled(bt_rq)) { @@ -861,7 +874,8 @@ static void account_bt_rq_runtime(struct bt_rq *bt_rq, unsigned long delta_exec) { - if (!bt_bandwidth_enabled() || sched_bt_runtime(bt_rq) == RUNTIME_INF) + if (!offlinegroup_enabled && + (!bt_bandwidth_enabled() || sched_bt_runtime(bt_rq) == RUNTIME_INF)) return; raw_spin_lock(&bt_rq->bt_runtime_lock); @@ -2506,12 +2520,19 @@ select_task_rq_bt(struct task_struct *p, int prev_cpu, int sd_flag, int wake_fla int cpu = smp_processor_id(); int new_cpu = prev_cpu; int want_affine = 0; + bool check_cpumask; - if (p->nr_cpus_allowed == 1) + if (offlinegroup_enabled && sysctl_sched_bt_ignore_cpubind) + check_cpumask = false; + else + check_cpumask = true; + + if (check_cpumask && p->nr_cpus_allowed == 1) return prev_cpu; if (sd_flag & SD_BALANCE_WAKE) { - if (cpumask_test_cpu(cpu, tsk_cpus_allowed(p))) + if (!check_cpumask || + (check_cpumask && cpumask_test_cpu(cpu, tsk_cpus_allowed(p)))) want_affine = 1; new_cpu = prev_cpu; } @@ -2866,6 +2887,8 @@ static bool yield_to_task_bt(struct rq *rq, struct task_struct *p, bool preempt) static int can_migrate_bt_task(struct task_struct *p, struct lb_env *env) { + bool check_cpumask; + /* * We do not migrate tasks that are: * 1) throttled_lb_pair, or @@ -2873,7 +2896,12 @@ int can_migrate_bt_task(struct task_struct *p, struct lb_env *env) * 3) running (obviously), or * 4) are cache-hot on their current CPU. */ - if (!cpumask_test_cpu(env->dst_cpu, tsk_cpus_allowed(p))) { + if (offlinegroup_enabled && sysctl_sched_bt_ignore_cpubind) + check_cpumask = false; + else + check_cpumask = true; + + if (check_cpumask && !cpumask_test_cpu(env->dst_cpu, tsk_cpus_allowed(p))) { int cpu; schedstat_inc(p->se.bt_statistics->nr_failed_migrations_affine); @@ -3949,6 +3977,13 @@ static int load_balance_bt(int this_cpu, struct rq *this_rq, .cpus = cpus, }; + bool check_cpumask; + + if (offlinegroup_enabled && sysctl_sched_bt_ignore_cpubind) + check_cpumask = true; + else + check_cpumask = false; + /* * For NEWLY_IDLE load_balancing, we don't need to consider * other cpus in our group @@ -4088,8 +4123,8 @@ more_balance: * if the curr task on busiest cpu can't be * moved to this_cpu */ - if (!cpumask_test_cpu(this_cpu, - tsk_cpus_allowed(busiest->curr))) { + if (check_cpumask && + !cpumask_test_cpu(this_cpu, tsk_cpus_allowed(busiest->curr))) { raw_spin_unlock_irqrestore(&busiest->lock, flags); env.flags |= LBF_ALL_PINNED; @@ -4901,10 +4936,9 @@ static int sched_bt_global_constraints(void) if (sysctl_sched_bt_period <= 0) return -EINVAL; - /* - * There's always some BT tasks in the root group - * -- migration, kstopmachine etc.. - */ + if (offlinegroup_enabled) + return 0; + if (sysctl_sched_bt_runtime == 0) return -EINVAL; @@ -4950,3 +4984,80 @@ int sched_bt_handler(struct ctl_table *table, int write, return ret; } + +static int offline_proc_show(struct seq_file *m, void *v) { + unsigned long * n = (unsigned long*)m->private; + + seq_printf(m, "%lu\n", *n); + return 0; +} + +static int offline_proc_open(struct inode *inode, struct file *file) +{ + return single_open(file, offline_proc_show, PDE_DATA(inode)); +} + +#define OFFLINE_NUMBUF 8 +static ssize_t offline_proc_write(struct file *file, const char __user *ubuf, + size_t cnt, loff_t *ppos) +{ + unsigned long * start = (unsigned long *)bt_cpu_control_set; + unsigned long *to = (unsigned long *)PDE_DATA(file_inode(file)); + char buffer[OFFLINE_NUMBUF]; + int cpu = to - start; + struct bt_rq *bt_rq; + unsigned long tmp; + int cpn; + + cpn = min((int)OFFLINE_NUMBUF, (int)cnt); + if (copy_from_user(buffer, ubuf, cpn)) + return -EFAULT; + + buffer[cpn - 1] = '\0'; + if (kstrtoul(buffer, 0, &tmp) || tmp > 100) + return -EINVAL; + + *to = tmp; + bt_rq = &cpu_rq(cpu)->bt; + + raw_spin_lock(&bt_rq->bt_runtime_lock); + bt_rq->bt_runtime = (u64)sysctl_sched_bt_period * NSEC_PER_USEC * tmp / 100; + raw_spin_unlock(&bt_rq->bt_runtime_lock); + + return cnt; +} + +static const struct file_operations info_fops = { + .open = offline_proc_open, + .read = seq_read, + .write = offline_proc_write, + .llseek = seq_lseek, + .release = single_release, +}; + +void init_offline_cpu_control(void) +{ + int i, nr = num_online_cpus(); + char buffer[20] = "offline"; + struct proc_dir_entry * dir; + + if(!offlinegroup_enabled) + return; + + dir= proc_mkdir(buffer, NULL); + + if (!dir) + return; + + bt_cpu_control_set = kmalloc(sizeof(unsigned long)*nr, GFP_KERNEL); + + if(!bt_cpu_control_set) + return; + + for(i=0; i @@ -1020,8 +1022,16 @@ struct migration_arg { static struct rq *__migrate_task(struct rq *rq, struct rq_flags *rf, struct task_struct *p, int dest_cpu) { + bool check_cpumask; + + if (offlinegroup_enabled && sysctl_sched_bt_ignore_cpubind && + (p->sched_class == &bt_sched_class)) + check_cpumask = false; + else + check_cpumask = true; + /* Affinity changed (again). */ - if (!is_cpu_allowed(p, dest_cpu)) + if (!check_cpumask && !is_cpu_allowed(p, dest_cpu)) return rq; update_rq_clock(rq); @@ -1605,6 +1615,10 @@ int select_task_rq(struct task_struct *p, int cpu, int sd_flags, int wake_flags) else cpu = cpumask_any(&p->cpus_allowed); + if (offlinegroup_enabled && sysctl_sched_bt_ignore_cpubind && + (p->sched_class == &bt_sched_class)) + return cpu; + /* * In order not to call set_task_cpu() on a blocking task we need * to rely on ttwu() to place the task on a valid ->cpus_allowed @@ -4675,6 +4689,7 @@ do_sched_setscheduler(pid_t pid, int policy, struct sched_param __user *param) struct sched_param lparam; struct task_struct *p; int retval; + struct task_group *tg; if (!param || pid < 0) return -EINVAL; @@ -4684,8 +4699,21 @@ do_sched_setscheduler(pid_t pid, int policy, struct sched_param __user *param) rcu_read_lock(); retval = -ESRCH; p = find_process_by_pid(pid); - if (p != NULL) + if (p != NULL) { + if (offlinegroup_enabled) { + tg = container_of(task_css_check(p, cpu_cgrp_id, false), + struct task_group, css); + if (tg->parent && tg->offline && policy != SCHED_BT && + policy != SCHED_RR && policy != SCHED_FIFO) + goto out; + + if (tg->parent && !tg->offline && policy == SCHED_BT) + goto out; + } retval = sched_setscheduler(p, policy, &lparam); + } + +out: rcu_read_unlock(); return retval; @@ -6647,6 +6675,7 @@ void sched_offline_group(struct task_group *tg) static void sched_change_group(struct task_struct *tsk, int type) { struct task_group *tg; + struct rq *rq = task_rq(tsk); /* * All callers are synchronized by task_rq_lock(); we do not use RCU @@ -6658,6 +6687,34 @@ static void sched_change_group(struct task_struct *tsk, int type) tg = autogroup_task_group(tsk, tg); tsk->sched_task_group = tg; + /* No need to re-setcheduler when fork or exit a task */ + if (offlinegroup_enabled && !rt_task(tsk) && + !(tsk->flags & PF_EXITING) && (type != TASK_SET_GROUP)) { + struct sched_attr attr = { + .sched_priority = 0, + }; + + if (tg->offline) { + attr.sched_nice = PRIO_TO_BT_NICE(tsk->static_prio); + attr.sched_policy = SCHED_BT; + } else { + attr.sched_nice = PRIO_TO_NICE(tsk->static_prio); + attr.sched_policy = SCHED_NORMAL; + } + + /* + * FIXME: __setscheduler before task_change_group would + * lead to missing prev cfs_rq/bt_rq stats updating. + * Otherwise, putting __setscheduler after task_change_group + * is not right either, which would miss next cfs_rq/bt_rq + * stats updating. + * In fact, __setscheduler should be right before set_task_rq + * in task_change_group callback, but if so, the code would be + * messed up. + */ + __setscheduler(rq, tsk, &attr, 0); + } + #if defined(CONFIG_FAIR_GROUP_SCHED) || defined (CONFIG_BT_GROUP_SCHED) if (tsk->sched_class->task_change_group) tsk->sched_class->task_change_group(tsk, type); @@ -6721,6 +6778,9 @@ cpu_cgroup_css_alloc(struct cgroup_subsys_state *parent_css) if (IS_ERR(tg)) return ERR_PTR(-ENOMEM); + if (offlinegroup_enabled) + tg->offline = parent->offline; + return &tg->css; } @@ -7093,6 +7153,53 @@ static u64 cpu_bt_shares_read_u64(struct cgroup_subsys_state *css, return (u64) scale_load_down(tg->bt_shares); } + +static int cpu_offline_write_uint(struct cgroup_subsys_state *css, + struct cftype *cftype, u64 offline_input) +{ + struct css_task_iter it; + struct task_struct *tsk; + struct task_group *tg; + struct sched_param param; + int pid, sched_class; + int err; + tg = css_tg(css); + + if (!tg->se[0]) + return -EINVAL; + + if (tg->offline == !!offline_input) + return 0; + + if (!tg->offline && offline_input) { + sched_class = SCHED_BT; + } else if (tg->offline && !offline_input) { + sched_class = SCHED_NORMAL; + } else + return 0; + + tg->offline = !!offline_input; + + param.sched_priority = 0; + css_task_iter_start(css, 0, &it); + while ((tsk = css_task_iter_next(&it))) { + pid = task_tgid_vnr(tsk); + + if (pid > 0 && !rt_task(tsk)) { + err=sched_setscheduler(tsk, sched_class, ¶m); + } + } + css_task_iter_end(&it); + return 0; +} + +static u64 cpu_offline_read_uint(struct cgroup_subsys_state *css, + struct cftype *cft) +{ + struct task_group *tg = css_tg(css); + return tg->offline; +} + #endif #ifdef CONFIG_RT_GROUP_SCHED @@ -7132,9 +7239,16 @@ static struct cftype cpu_files[] = { #ifdef CONFIG_BT_GROUP_SCHED { .name = "bt_shares", + .flags = CFTYPE_BT_SHARES, .read_u64 = cpu_bt_shares_read_u64, .write_u64 = cpu_bt_shares_write_u64, }, + { + .name = "offline", + .flags = CFTYPE_BT_PRIVATE, + .read_u64 = cpu_offline_read_uint, + .write_u64 = cpu_offline_write_uint, + }, #endif #ifdef CONFIG_CFS_BANDWIDTH { diff --git a/kernel/sched/cpuacct.c b/kernel/sched/cpuacct.c index 54d455a5e..eb24f6a23 100644 --- a/kernel/sched/cpuacct.c +++ b/kernel/sched/cpuacct.c @@ -519,6 +519,7 @@ static struct cftype files[] = { #ifdef CONFIG_BT_SCHED { .name = "bt_usage_percpu", + .flags = CFTYPE_BT_SHARES, .seq_show = bt_cpuacct_percpu_seq_show, }, #endif @@ -541,6 +542,7 @@ static struct cftype files[] = { #ifdef CONFIG_BT_SCHED { .name = "bt_stat", + .flags = CFTYPE_BT_SHARES, .seq_show = bt_cpuacct_stats_show, }, #endif diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index e781f699d..c1910ca0c 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -372,6 +373,8 @@ struct task_group { #endif struct cfs_bandwidth cfs_bandwidth; + + unsigned long offline; }; #ifdef CONFIG_FAIR_GROUP_SCHED diff --git a/kernel/sysctl.c b/kernel/sysctl.c index 72fc53748..628f58ad4 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -571,6 +571,15 @@ static struct ctl_table kern_table[] = { .mode = 0644, .proc_handler = proc_dointvec, }, + { + .procname = "sched_bt_ignore_cpubind", + .data = &sysctl_sched_bt_ignore_cpubind, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = &zero, + .extra2 = &one, + }, #ifdef CONFIG_INTEL_RDT { .procname = "sched_bt_rdt_cache_percent",