From b48a63ef2c06113aaa153f8a0e73df88f79cfeae Mon Sep 17 00:00:00 2001 From: He Chen Date: Mon, 2 Mar 2020 16:14:50 +0800 Subject: [PATCH] sched: add offline scheduler class The offline scheduler named BT sched is based on the CFS scheduler. We also use the rb-tree as the run queue to save the runnable tasks. And the vruntime concept is also used in the offline scheduler. And the priority of offline scheduler is from 140 to 179. So now the schedulers in the kernel are as follows: stop, RT, CFS, BT and idle. Signed-off-by: Xiaoming Gao Signed-off-by: Hua Liu Signed-off-by: Xiaogguang Chen Signed-off-by: Zhiguang Peng Signed-off-by: Bin Fan Signed-off-by: He Chen --- fs/proc/Makefile | 1 + fs/proc/base.c | 2 +- fs/proc/bt_stat.c | 182 ++++ fs/proc/stat.c | 8 +- include/linux/init_task.h | 15 +- include/linux/ioprio.h | 4 + include/linux/kernel_stat.h | 3 + include/linux/sched.h | 28 +- include/linux/sched/batch.h | 60 ++ include/linux/sched/prio.h | 4 + include/uapi/linux/sched.h | 3 + init/Kconfig | 6 + init/main.c | 9 + kernel/delayacct.c | 2 +- kernel/exit.c | 2 +- kernel/sched/Makefile | 1 + kernel/sched/batch.c | 1446 ++++++++++++++++++++++++++++++++ kernel/sched/batch.h | 83 ++ kernel/sched/bt_debug.c | 72 ++ kernel/sched/bt_debug.h | 36 + kernel/sched/core.c | 129 ++- kernel/sched/cpupri.c | 5 + kernel/sched/cputime.c | 11 +- kernel/sched/debug.c | 25 +- kernel/sched/fair.c | 88 +- kernel/sched/fair.h | 47 ++ kernel/sched/loadavg.c | 4 +- kernel/sched/sched.h | 40 +- kernel/time/posix-cpu-timers.c | 4 +- package/default/config.default | 1 + 30 files changed, 2234 insertions(+), 87 deletions(-) create mode 100644 fs/proc/bt_stat.c create mode 100644 include/linux/sched/batch.h create mode 100644 kernel/sched/batch.c create mode 100644 kernel/sched/batch.h create mode 100644 kernel/sched/bt_debug.c create mode 100644 kernel/sched/bt_debug.h create mode 100644 kernel/sched/fair.h diff --git a/fs/proc/Makefile b/fs/proc/Makefile index f7456c4e7..1f7b57472 100644 --- a/fs/proc/Makefile +++ b/fs/proc/Makefile @@ -26,6 +26,7 @@ proc-y += softirqs.o proc-y += namespaces.o proc-y += self.o proc-y += thread_self.o +proc-$(CONFIG_BT_SCHED) += bt_stat.o proc-$(CONFIG_PROC_SYSCTL) += proc_sysctl.o proc-$(CONFIG_NET) += proc_net.o proc-$(CONFIG_PROC_KCORE) += kcore.o diff --git a/fs/proc/base.c b/fs/proc/base.c index a628d0925..65ef7d5d8 100644 --- a/fs/proc/base.c +++ b/fs/proc/base.c @@ -481,7 +481,7 @@ static int proc_pid_schedstat(struct seq_file *m, struct pid_namespace *ns, seq_printf(m, "0 0 0\n"); else seq_printf(m, "%llu %llu %lu\n", - (unsigned long long)task->se.sum_exec_runtime, + TASK_SUM_EXEC_RUNTIME(task), (unsigned long long)task->sched_info.run_delay, task->sched_info.pcount); diff --git a/fs/proc/bt_stat.c b/fs/proc/bt_stat.c new file mode 100644 index 000000000..49b64f774 --- /dev/null +++ b/fs/proc/bt_stat.c @@ -0,0 +1,182 @@ +/* + * Copyright (C) 2019 Tencent Ltd. All rights reserved. + * + * File Name ：bt_stat.c + * Author ： + * Date ：2019-12-28 + * Descriptor： + */ + +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +extern u64 get_idle_time(int cpu); +extern u64 get_iowait_time(int cpu); + +static int show_bt_stat(struct seq_file *p, void *v) +{ + int i, j; + unsigned long jif; + u64 user, nice, system, idle, iowait, irq, softirq, steal, cputime_bt; + u64 guest, guest_nice; + u64 sum = 0; + u64 sum_softirq = 0; + unsigned int per_softirq_sums[NR_SOFTIRQS] = {0}; + struct timespec boottime; + + user = nice = system = idle = iowait = + irq = softirq = steal = cputime_bt = 0; + guest = guest_nice = 0; + getboottime(&boottime); + jif = boottime.tv_sec; + + for_each_possible_cpu(i) { + user += kcpustat_cpu(i).cpustat[CPUTIME_USER]; + nice += kcpustat_cpu(i).cpustat[CPUTIME_NICE]; + system += kcpustat_cpu(i).cpustat[CPUTIME_SYSTEM]; + idle += get_idle_time(i); + iowait += get_iowait_time(i); + irq += kcpustat_cpu(i).cpustat[CPUTIME_IRQ]; + softirq += kcpustat_cpu(i).cpustat[CPUTIME_SOFTIRQ]; + steal += kcpustat_cpu(i).cpustat[CPUTIME_STEAL]; + guest += kcpustat_cpu(i).cpustat[CPUTIME_GUEST]; + guest_nice += kcpustat_cpu(i).cpustat[CPUTIME_GUEST_NICE]; + cputime_bt += kcpustat_cpu(i).cpustat[CPUTIME_BT]; + sum += kstat_cpu_irqs_sum(i); + sum += arch_irq_stat_cpu(i); + + for (j = 0; j < NR_SOFTIRQS; j++) { + unsigned int softirq_stat = kstat_softirqs_cpu(j, i); + + per_softirq_sums[j] += softirq_stat; + sum_softirq += softirq_stat; + } + } + sum += arch_irq_stat(); + + seq_puts(p, "cpu "); + seq_put_decimal_ull(p, " ", nsec_to_clock_t(user)); + seq_put_decimal_ull(p, " ", nsec_to_clock_t(nice)); + seq_put_decimal_ull(p, " ", nsec_to_clock_t(system)); + seq_put_decimal_ull(p, " ", nsec_to_clock_t(idle)); + seq_put_decimal_ull(p, " ", nsec_to_clock_t(iowait)); + seq_put_decimal_ull(p, " ", nsec_to_clock_t(irq)); + seq_put_decimal_ull(p, " ", nsec_to_clock_t(softirq)); + seq_put_decimal_ull(p, " ", nsec_to_clock_t(steal)); + seq_put_decimal_ull(p, " ", nsec_to_clock_t(guest)); + seq_put_decimal_ull(p, " ", nsec_to_clock_t(guest_nice)); + seq_put_decimal_ull(p, " ", nsec_to_clock_t(cputime_bt)); + seq_putc(p, '\n'); + + for_each_online_cpu(i) { + /* Copy values here to work around gcc-2.95.3, gcc-2.96 */ + user = kcpustat_cpu(i).cpustat[CPUTIME_USER]; + nice = kcpustat_cpu(i).cpustat[CPUTIME_NICE]; + system = kcpustat_cpu(i).cpustat[CPUTIME_SYSTEM]; + idle = get_idle_time(i); + iowait = get_iowait_time(i); + irq = kcpustat_cpu(i).cpustat[CPUTIME_IRQ]; + softirq = kcpustat_cpu(i).cpustat[CPUTIME_SOFTIRQ]; + steal = kcpustat_cpu(i).cpustat[CPUTIME_STEAL]; + guest = kcpustat_cpu(i).cpustat[CPUTIME_GUEST]; + guest_nice = kcpustat_cpu(i).cpustat[CPUTIME_GUEST_NICE]; + cputime_bt = kcpustat_cpu(i).cpustat[CPUTIME_BT]; + seq_printf(p, "cpu%d", i); + seq_put_decimal_ull(p, " ", nsec_to_clock_t(user)); + seq_put_decimal_ull(p, " ", nsec_to_clock_t(nice)); + seq_put_decimal_ull(p, " ", nsec_to_clock_t(system)); + seq_put_decimal_ull(p, " ", nsec_to_clock_t(idle)); + seq_put_decimal_ull(p, " ", nsec_to_clock_t(iowait)); + seq_put_decimal_ull(p, " ", nsec_to_clock_t(irq)); + seq_put_decimal_ull(p, " ", nsec_to_clock_t(softirq)); + seq_put_decimal_ull(p, " ", nsec_to_clock_t(steal)); + seq_put_decimal_ull(p, " ", nsec_to_clock_t(guest)); + seq_put_decimal_ull(p, " ", nsec_to_clock_t(guest_nice)); + seq_put_decimal_ull(p, " ", nsec_to_clock_t(cputime_bt)); + seq_putc(p, '\n'); + } + seq_printf(p, "intr %llu", (unsigned long long)sum); + + /* sum again ? it could be updated? */ + for_each_irq_nr(j) + seq_put_decimal_ull(p, " ", kstat_irqs(j)); + + seq_printf(p, + "\nctxt %llu\n" + "btime %lu\n" + "processes %d\n" + "procs_running %lu\n" + "procs_blocked %lu\n", + nr_context_switches(), + (unsigned long)jif, + nr_forks(), + nr_running(), + nr_iowait()); + + seq_printf(p, "softirq %llu", (unsigned long long)sum_softirq); + + for (i = 0; i < NR_SOFTIRQS; i++) + seq_put_decimal_ull(p, " ", per_softirq_sums[i]); + seq_putc(p, '\n'); + + return 0; +} + +static int bt_stat_open(struct inode *inode, struct file *file) +{ + size_t size = 1024 + 128 * num_online_cpus(); + char *buf; + struct seq_file *m; + int res; + + /* minimum size to display an interrupt count : 2 bytes */ + size += 2 * nr_irqs; + + /* don't ask for more than the kmalloc() max size */ + if (size > KMALLOC_MAX_SIZE) + size = KMALLOC_MAX_SIZE; + buf = kmalloc(size, GFP_KERNEL); + if (!buf) + return -ENOMEM; + + res = single_open(file, show_bt_stat, NULL); + if (!res) { + m = file->private_data; + m->buf = buf; + m->size = ksize(buf); + } else { + kfree(buf); + } + + return res; +} + +static const struct file_operations proc_bt_stat_operations = { + .open = bt_stat_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +static int __init proc_bt_stat_init(void) +{ + if(sched_bt_on) + proc_create("bt_stat", 0000, NULL, &proc_bt_stat_operations); + + return 0; +} +fs_initcall(proc_bt_stat_init); diff --git a/fs/proc/stat.c b/fs/proc/stat.c index 3179babe7..4bbeb3fc9 100644 --- a/fs/proc/stat.c +++ b/fs/proc/stat.c @@ -23,7 +23,7 @@ #ifdef arch_idle_time -static u64 get_idle_time(int cpu) +u64 get_idle_time(int cpu) { u64 idle; @@ -33,7 +33,7 @@ static u64 get_idle_time(int cpu) return idle; } -static u64 get_iowait_time(int cpu) +u64 get_iowait_time(int cpu) { u64 iowait; @@ -45,7 +45,7 @@ static u64 get_iowait_time(int cpu) #else -static u64 get_idle_time(int cpu) +u64 get_idle_time(int cpu) { u64 idle, idle_usecs = -1ULL; @@ -61,7 +61,7 @@ static u64 get_idle_time(int cpu) return idle; } -static u64 get_iowait_time(int cpu) +u64 get_iowait_time(int cpu) { u64 iowait, iowait_usecs = -1ULL; diff --git a/include/linux/init_task.h b/include/linux/init_task.h index 8062e6cc6..b96a1eeb7 100644 --- a/include/linux/init_task.h +++ b/include/linux/init_task.h @@ -219,6 +219,15 @@ extern struct cred init_cred; #define INIT_TASK_SECURITY #endif +#ifdef CONFIG_BT_SCHED +#define INIT_PRIO MAX_PRIO - 20 - 40 +#define INIT_STATIC_PRIO MAX_PRIO - 20 - 40 +#define INIT_NORMAL_PRIO MAX_PRIO - 20 - 40 +#else +#define INIT_PRIO MAX_PRIO - 20 +#define INIT_STATIC_PRIO MAX_PRIO - 20 +#define INIT_NORMAL_PRIO MAX_PRIO - 20 +#endif /* * INIT_TASK is used to set up the first task table, touch at * your own risk!. Base=0, limit=0x1fffff (=2MB) @@ -230,9 +239,9 @@ extern struct cred init_cred; .stack = init_stack, \ .usage = ATOMIC_INIT(2), \ .flags = PF_KTHREAD, \ - .prio = MAX_PRIO-20, \ - .static_prio = MAX_PRIO-20, \ - .normal_prio = MAX_PRIO-20, \ + .prio = INIT_PRIO, \ + .static_prio = INIT_STATIC_PRIO, \ + .normal_prio = INIT_NORMAL_PRIO, \ .policy = SCHED_NORMAL, \ .cpus_allowed = CPU_MASK_ALL, \ .nr_cpus_allowed= NR_CPUS, \ diff --git a/include/linux/ioprio.h b/include/linux/ioprio.h index 2cdd74809..bf2539adc 100644 --- a/include/linux/ioprio.h +++ b/include/linux/ioprio.h @@ -61,7 +61,11 @@ static inline int task_nice_ioprio(struct task_struct *task) */ static inline int task_nice_ioclass(struct task_struct *task) { +#ifdef CONFIG_BT_SCHED + if (task->policy == SCHED_IDLE || task->policy == SCHED_BT) +#else if (task->policy == SCHED_IDLE) +#endif return IOPRIO_CLASS_IDLE; else if (task->policy == SCHED_FIFO || task->policy == SCHED_RR) return IOPRIO_CLASS_RT; diff --git a/include/linux/kernel_stat.h b/include/linux/kernel_stat.h index bfad33f11..a991b3c57 100644 --- a/include/linux/kernel_stat.h +++ b/include/linux/kernel_stat.h @@ -28,6 +28,9 @@ enum cpu_usage_stat { CPUTIME_STEAL, CPUTIME_GUEST, CPUTIME_GUEST_NICE, +#ifdef CONFIG_BT_SCHED + CPUTIME_BT, +#endif NR_STATS, }; diff --git a/include/linux/sched.h b/include/linux/sched.h index 866439c36..9c5c15f09 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -111,6 +111,16 @@ struct task_group; (task->flags & PF_FROZEN) == 0 && \ (task->state & TASK_NOLOAD) == 0) +#ifdef CONFIG_BT_SCHED +#define TASK_SUM_EXEC_RUNTIME(tsk) \ + (unsigned long long)((tsk)->se.sum_exec_runtime + (tsk)->bt.sum_exec_runtime) + +#else +#define TASK_SUM_EXEC_RUNTIME(tsk) \ + (unsigned long long)((tsk)->se.sum_exec_runtime) + +#endif + #ifdef CONFIG_DEBUG_ATOMIC_SLEEP /* @@ -429,6 +439,9 @@ struct sched_entity { u64 nr_migrations; struct sched_statistics statistics; +#ifdef CONFIG_BT_SCHED + struct sched_statistics *bt_statistics; +#endif #ifdef CONFIG_FAIR_GROUP_SCHED int depth; @@ -601,6 +614,9 @@ struct task_struct { const struct sched_class *sched_class; struct sched_entity se; +#ifdef CONFIG_BT_SCHED + struct sched_entity bt; +#endif struct sched_rt_entity rt; #ifdef CONFIG_CGROUP_SCHED struct task_group *sched_task_group; @@ -1477,17 +1493,7 @@ extern int yield_to(struct task_struct *p, bool preempt); extern void set_user_nice(struct task_struct *p, long nice); extern int task_prio(const struct task_struct *p); -/** - * task_nice - return the nice value of a given task. - * @p: the task in question. - * - * Return: The nice value [ -20 ... 0 ... 19 ]. - */ -static inline int task_nice(const struct task_struct *p) -{ - return PRIO_TO_NICE((p)->static_prio); -} - +extern int task_nice(const struct task_struct *p); extern int can_nice(const struct task_struct *p, const int nice); extern int task_curr(const struct task_struct *p); extern int idle_cpu(int cpu); diff --git a/include/linux/sched/batch.h b/include/linux/sched/batch.h new file mode 100644 index 000000000..4932340fd --- /dev/null +++ b/include/linux/sched/batch.h @@ -0,0 +1,60 @@ +/* + * Copyright (C) 2019 Tencent Ltd. All rights reserved. + * + * File Name ：batch.h + * Author ： + * Date ：2019-12-26 + * Descriptor： + */ + +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _SCHED_BATCH_H +#define _SCHED_BATCH_H + +#define MAX_CFS_PRIO 139 +#define MIN_BT_PRIO 140 +#define MAX_BT_PRIO 179 +#define BT_PRIO_WIDTH (MAX_BT_PRIO - MIN_BT_PRIO + 1) + +/* + * Convert user-nice values [ -20 ... 0 ... 19 ] + * to bt static priority [ MIN_BT_PRI + 1 ..MAX_BT_PRIO ], + * and back. + */ +#define NICE_TO_BT_PRIO(nice) (MAX_RT_PRIO + (nice) + 20 + 40) +#define PRIO_TO_BT_NICE(prio) ((prio) - MAX_RT_PRIO - 20 - 40) +#define TASK_BT_NICE(p) PRIO_TO_BT_NICE((p)->static_prio) + +extern unsigned int sched_bt_on; + +static inline int cfs_prio(int prio) +{ + if (prio >= MAX_RT_PRIO && prio < MIN_BT_PRIO) + return 1; + return 0; +} + +static inline int bt_prio(int prio) +{ + if (prio > MAX_CFS_PRIO && prio < MAX_PRIO) + return 1; + return 0; +} + +static inline void bt_prio_adjust_pos(int *prio) +{ + int priority = *prio; + + if (cfs_prio(priority)) + *prio = priority + BT_PRIO_WIDTH; +} + +static inline void bt_prio_adjust_neg(int *prio) +{ + int priority = *prio; + + if (bt_prio(priority)) + *prio = priority - BT_PRIO_WIDTH; +} + +#endif /* _SCHED_BATCH_H */ diff --git a/include/linux/sched/prio.h b/include/linux/sched/prio.h index 7d64feafc..84a615de1 100644 --- a/include/linux/sched/prio.h +++ b/include/linux/sched/prio.h @@ -22,7 +22,11 @@ #define MAX_USER_RT_PRIO 100 #define MAX_RT_PRIO MAX_USER_RT_PRIO +#ifdef CONFIG_BT_SCHED +#define MAX_PRIO (MAX_RT_PRIO + NICE_WIDTH + 40) +#else #define MAX_PRIO (MAX_RT_PRIO + NICE_WIDTH) +#endif #define DEFAULT_PRIO (MAX_RT_PRIO + NICE_WIDTH / 2) /* diff --git a/include/uapi/linux/sched.h b/include/uapi/linux/sched.h index 30a9e51bb..e988c49d4 100644 --- a/include/uapi/linux/sched.h +++ b/include/uapi/linux/sched.h @@ -40,6 +40,9 @@ /* SCHED_ISO: reserved but not implemented yet */ #define SCHED_IDLE 5 #define SCHED_DEADLINE 6 +#ifdef CONFIG_BT_SCHED +#define SCHED_BT 7 +#endif /* Can be ORed in to make sure the process is reverted back to SCHED_NORMAL on fork */ #define SCHED_RESET_ON_FORK 0x40000000 diff --git a/init/Kconfig b/init/Kconfig index 46075327c..36ed5a079 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -947,6 +947,12 @@ config NET_NS endif # NAMESPACES +config BT_SCHED + bool "offline sched class" + default n + help + Allow user to create offline task + config SCHED_AUTOGROUP bool "Automatic process group scheduling" select CGROUPS diff --git a/init/main.c b/init/main.c index c4a45145e..2254d098f 100644 --- a/init/main.c +++ b/init/main.c @@ -353,6 +353,15 @@ static int __init rdinit_setup(char *str) } __setup("rdinit=", rdinit_setup); +unsigned int sched_bt_on; +static int __init set_sched_bt_on(char *str) +{ + sched_bt_on = 1; + + return 1; +} +early_param("offline_class", set_sched_bt_on); + #ifndef CONFIG_SMP static const unsigned int setup_max_cpus = NR_CPUS; static inline void setup_nr_cpu_ids(void) { } diff --git a/kernel/delayacct.c b/kernel/delayacct.c index ca8ac2824..3bfe068a6 100644 --- a/kernel/delayacct.c +++ b/kernel/delayacct.c @@ -115,7 +115,7 @@ int __delayacct_add_tsk(struct taskstats *d, struct task_struct *tsk) */ t1 = tsk->sched_info.pcount; t2 = tsk->sched_info.run_delay; - t3 = tsk->se.sum_exec_runtime; + t3 = TASK_SUM_EXEC_RUNTIME(tsk); d->cpu_count += t1; diff --git a/kernel/exit.c b/kernel/exit.c index 5523fb0c2..86a467256 100644 --- a/kernel/exit.c +++ b/kernel/exit.c @@ -150,7 +150,7 @@ static void __exit_signal(struct task_struct *tsk) sig->inblock += task_io_get_inblock(tsk); sig->oublock += task_io_get_oublock(tsk); task_io_accounting_add(&sig->ioac, &tsk->ioac); - sig->sum_sched_runtime += tsk->se.sum_exec_runtime; + sig->sum_sched_runtime += TASK_SUM_EXEC_RUNTIME(tsk); sig->nr_threads--; __unhash_process(tsk, group_dead); write_sequnlock(&sig->stats_lock); diff --git a/kernel/sched/Makefile b/kernel/sched/Makefile index a9ee16bbc..fdadac8c8 100644 --- a/kernel/sched/Makefile +++ b/kernel/sched/Makefile @@ -20,6 +20,7 @@ obj-y += core.o loadavg.o clock.o cputime.o obj-y += idle_task.o fair.o rt.o deadline.o obj-y += wait.o wait_bit.o swait.o completion.o idle.o obj-$(CONFIG_SMP) += cpupri.o cpudeadline.o topology.o stop_task.o +obj-$(CONFIG_BT_SCHED) += batch.o bt_debug.o obj-$(CONFIG_SCHED_AUTOGROUP) += autogroup.o obj-$(CONFIG_SCHEDSTATS) += stats.o obj-$(CONFIG_SCHED_DEBUG) += debug.o diff --git a/kernel/sched/batch.c b/kernel/sched/batch.c new file mode 100644 index 000000000..5d7679395 --- /dev/null +++ b/kernel/sched/batch.c @@ -0,0 +1,1446 @@ +/* + * Copyright (C) 2019 Tencent Ltd. All rights reserved. + * + * File Name : batch.c + * Author : + * Date : 2019-12-26 + * Descriptor: + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include "batch.h" +#include "sched.h" +#include "fair.h" + +void set_bt_load_weight(struct task_struct *p) +{ + int prio = p->static_prio - MIN_BT_PRIO; + struct load_weight *load = &p->bt.load; + + load->weight = scale_load(sched_prio_to_weight[prio]); + load->inv_weight = sched_prio_to_wmult[prio]; +} + +const struct sched_class bt_sched_class; + +static inline struct task_struct *bt_task_of(struct sched_entity *bt_se) +{ + return container_of(bt_se, struct task_struct, bt); +} + +static inline struct rq *rq_of_bt_rq(struct bt_rq *bt_rq) +{ + return container_of(bt_rq, struct rq, bt); +} + +#define bt_entity_is_task(bt) 1 + +#define for_each_sched_bt_entity(bt) \ + for (; bt; bt = NULL) + +static inline struct bt_rq *task_bt_rq(struct task_struct *p) +{ + return &task_rq(p)->bt; +} + +static inline struct bt_rq *bt_rq_of(struct sched_entity *bt_se) +{ + struct task_struct *p = bt_task_of(bt_se); + struct rq *rq = task_rq(p); + + return &rq->bt; +} + +static inline struct sched_entity *parent_bt_entity(struct sched_entity *bt) +{ + return NULL; +} + +/************************************************************** + * Scheduling class tree data structure manipulation methods: + */ + +static inline int bt_entity_before(struct sched_entity *a, + struct sched_entity *b) +{ + return (s64)(a->vruntime - b->vruntime) < 0; +} + +static void update_bt_min_vruntime(struct bt_rq *bt_rq) +{ + struct sched_entity *curr = bt_rq->curr; + struct rb_node *leftmost = rb_first_cached(&bt_rq->tasks_timeline); + + u64 vruntime = bt_rq->min_vruntime; + + if (curr) { + if (curr->on_rq) + vruntime = bt_rq->curr->vruntime; + else + curr = NULL; + } + + if (leftmost) { + struct sched_entity *bt_se = rb_entry(leftmost, + struct sched_entity, + run_node); + + if (!curr) + vruntime = bt_se->vruntime; + else + vruntime = min_vruntime(vruntime, bt_se->vruntime); + } + + /* ensure we never gain time by being placed backwards. */ + bt_rq->min_vruntime = max_vruntime(bt_rq->min_vruntime, vruntime); +#ifndef CONFIG_64BIT + /* memory barrior for writting */ + smp_wmb(); + bt_rq->min_vruntime_copy = bt_rq->min_vruntime; +#endif +} + +/* + * Enqueue an entity into the rb-tree: + */ +static void __enqueue_bt_entity(struct bt_rq *bt_rq, struct sched_entity *bt_se) +{ + struct rb_node **link = &bt_rq->tasks_timeline.rb_root.rb_node; + struct rb_node *parent = NULL; + struct sched_entity *entry; + int leftmost = 1; + + /* + * Find the right place in the rbtree: + */ + while (*link) { + parent = *link; + entry = rb_entry(parent, struct sched_entity, run_node); + /* + * We dont care about collisions. Nodes with + * the same key stay together. + */ + if (bt_entity_before(bt_se, entry)) { + link = &parent->rb_left; + } else { + link = &parent->rb_right; + leftmost = 0; + } + } + + rb_link_node(&bt_se->run_node, parent, link); + rb_insert_color_cached(&bt_se->run_node, + &bt_rq->tasks_timeline, leftmost); +} + +static void __dequeue_bt_entity(struct bt_rq *bt_rq, struct sched_entity *bt_se) +{ + rb_erase_cached(&bt_se->run_node, &bt_rq->tasks_timeline); +} + +struct sched_entity *__pick_first_bt_entity(struct bt_rq *bt_rq) +{ + struct rb_node *left = rb_first_cached(&bt_rq->tasks_timeline); + + if (!left) + return NULL; + + return rb_entry(left, struct sched_entity, run_node); +} + +static struct sched_entity *__pick_next_bt_entity(struct sched_entity *bt_se) +{ + struct rb_node *next = rb_next(&bt_se->run_node); + + if (!next) + return NULL; + + return rb_entry(next, struct sched_entity, run_node); +} + +#ifdef CONFIG_SCHED_DEBUG +struct sched_entity *__pick_last_bt_entity(struct bt_rq *bt_rq) +{ + struct rb_node *last = rb_last(&bt_rq->tasks_timeline.rb_root); + + if (!last) + return NULL; + + return rb_entry(last, struct sched_entity, run_node); +} +#endif + +/* + * delta /= w + */ +static inline unsigned long +calc_delta_bt(unsigned long delta, struct sched_entity *bt_se) +{ + if (unlikely(bt_se->load.weight != NICE_0_LOAD)) + delta = __calc_delta(delta, NICE_0_LOAD, &bt_se->load); + + return delta; +} + +/* + * We calculate the wall-time slice from the period by taking a part + * proportional to the weight. + * + * s = p*P[w/rw] + */ +static u64 sched_bt_slice(struct bt_rq *bt_rq, struct sched_entity *se) +{ + u64 slice = __sched_period(bt_rq->nr_running + !se->on_rq); + + for_each_sched_bt_entity(se) { + struct load_weight *load; + struct load_weight lw; + + bt_rq = bt_rq_of(se); + load = &bt_rq->load; + + if (unlikely(!se->on_rq)) { + lw = bt_rq->load; + + update_load_add(&lw, se->load.weight); + load = &lw; + } + slice = __calc_delta(slice, se->load.weight, load); + } + return slice; +} + +/* + * We calculate the vruntime slice of a to-be-inserted task. + * + * vs = s/w + */ +static u64 sched_bt_vslice(struct bt_rq *bt_rq, struct sched_entity *se) +{ + return calc_delta_bt(sched_bt_slice(bt_rq, se), se); +} + +/* + * Update the current task's runtime bt_statistics. Skip current tasks that + * are not in our scheduling class. + */ +static inline void +__update_curr_bt(struct bt_rq *bt_rq, struct sched_entity *curr, + unsigned long delta_exec) +{ + unsigned long delta_exec_weighted; + + schedstat_set(curr->bt_statistics->exec_max, + max((u64)delta_exec, curr->bt_statistics->exec_max)); + + curr->sum_exec_runtime += delta_exec; + schedstat_add(bt_rq->exec_clock, delta_exec); + delta_exec_weighted = calc_delta_bt(delta_exec, curr); + + curr->vruntime += delta_exec_weighted; + update_bt_min_vruntime(bt_rq); +} + +static void update_curr_bt(struct bt_rq *bt_rq) +{ + struct sched_entity *curr = bt_rq->curr; + u64 now = rq_of_bt_rq(bt_rq)->clock_task; + unsigned long delta_exec; + + if (unlikely(!curr)) + return; + + /* + * Get the amount of time the current task was running + * since the last time we changed load (this cannot + * overflow on 32 bits): + */ + delta_exec = (unsigned long)(now - curr->exec_start); + if (unlikely((s64)delta_exec <= 0)) + return; + + __update_curr_bt(bt_rq, curr, delta_exec); + curr->exec_start = now; + + if (bt_entity_is_task(curr)) { + struct task_struct *curtask = bt_task_of(curr); + + trace_sched_stat_runtime(curtask, delta_exec, curr->vruntime); + cpuacct_charge(curtask, delta_exec); + } +} + +static inline void +update_stats_wait_start_bt(struct bt_rq *bt_rq, struct sched_entity *bt_se) +{ + schedstat_set(bt_se->bt_statistics->wait_start, + rq_of_bt_rq(bt_rq)->clock); +} + +/* + * Task is being enqueued - update stats: + */ +static void +update_stats_enqueue_bt(struct bt_rq *bt_rq, struct sched_entity *se) +{ + /* + * Are we enqueueing a waiting task? (for current tasks + * a dequeue/enqueue event is a NOP) + */ + if (se != bt_rq->curr) + update_stats_wait_start_bt(bt_rq, se); +} + +static void +update_stats_wait_end_bt(struct bt_rq *bt_rq, struct sched_entity *se) +{ + schedstat_set(se->bt_statistics->wait_max, + max(se->bt_statistics->wait_max, + rq_of_bt_rq(bt_rq)->clock - se->bt_statistics->wait_start)); + schedstat_set(se->bt_statistics->wait_count, + se->bt_statistics->wait_count + 1); + schedstat_set(se->bt_statistics->wait_sum, + se->bt_statistics->wait_sum + + rq_of_bt_rq(bt_rq)->clock - se->bt_statistics->wait_start); +#ifdef CONFIG_SCHEDSTATS + if (bt_entity_is_task(se)) { + trace_sched_stat_wait(bt_task_of(se), + rq_of_bt_rq(bt_rq)->clock - se->bt_statistics->wait_start); + } +#endif + schedstat_set(se->bt_statistics->wait_start, 0); +} + +static inline void +update_stats_dequeue_bt(struct bt_rq *bt_rq, struct sched_entity *se) +{ + /* + * Mark the end of the wait period if dequeueing a + * waiting task: + */ + if (se != bt_rq->curr) + update_stats_wait_end_bt(bt_rq, se); +} + +/* + * We are picking a new current task - update its stats: + */ +static inline void +update_stats_curr_start_bt(struct bt_rq *bt_rq, struct sched_entity *se) +{ + /* + * We are starting a new run period: + */ + se->exec_start = rq_clock_task(rq_of_bt_rq(bt_rq)); +} + +static void +account_bt_entity_enqueue(struct bt_rq *bt_rq, struct sched_entity *se) +{ + update_load_add(&bt_rq->load, se->load.weight); + + bt_rq->nr_running++; +} + +static void +account_bt_entity_dequeue(struct bt_rq *bt_rq, struct sched_entity *se) +{ + update_load_sub(&bt_rq->load, se->load.weight); + + bt_rq->nr_running--; +} + +static void enqueue_bt_sleeper(struct bt_rq *bt_rq, struct sched_entity *se) +{ +#if defined(CONFIG_SCHEDSTATS) || defined(CONFIG_LATENCYTOP) + struct task_struct *tsk = NULL; + + if (bt_entity_is_task(se)) + tsk = bt_task_of(se); + + if (se->bt_statistics->sleep_start) { + u64 delta = rq_of_bt_rq(bt_rq)->clock - se->bt_statistics->sleep_start; + + if ((s64)delta < 0) + delta = 0; + +#ifdef CONFIG_SCHEDSTATS + if (unlikely(delta > se->bt_statistics->sleep_max)) + se->bt_statistics->sleep_max = delta; +#endif + + se->bt_statistics->sleep_start = 0; +#ifdef CONFIG_SCHEDSTATS + se->bt_statistics->sum_sleep_runtime += delta; +#endif + + if (tsk) { + account_scheduler_latency(tsk, delta >> 10, 1); +#ifdef CONFIG_SCHEDSTATS + trace_sched_stat_sleep(tsk, delta); +#endif + } + } + if (se->bt_statistics->block_start) { + u64 delta = rq_of_bt_rq(bt_rq)->clock - se->bt_statistics->block_start; + + if ((s64)delta < 0) + delta = 0; + +#ifdef CONFIG_SCHEDSTATS + if (unlikely(delta > se->bt_statistics->block_max)) + se->bt_statistics->block_max = delta; +#endif + + se->bt_statistics->block_start = 0; +#ifdef CONFIG_SCHEDSTATS + se->bt_statistics->sum_sleep_runtime += delta; +#endif + + if (tsk) { +#ifdef CONFIG_SCHEDSTATS + if (tsk->in_iowait) { + se->bt_statistics->iowait_sum += delta; + se->bt_statistics->iowait_count++; + trace_sched_stat_iowait(tsk, delta); + } +#endif + + trace_sched_stat_blocked(tsk, delta); + + /* + * Blocking time is in units of nanosecs, so shift by + * 20 to get a milliseconds-range estimation of the + * amount of time that the task spent sleeping: + */ + if (unlikely(prof_on == SLEEP_PROFILING)) { + profile_hits(SLEEP_PROFILING, + (void *)get_wchan(tsk), + delta >> 20); + } + account_scheduler_latency(tsk, delta >> 10, 0); + } + } +#endif +} + +static void check_bt_spread(struct bt_rq *bt_rq, struct sched_entity *se) +{ +#ifdef CONFIG_SCHED_DEBUG + s64 d = se->vruntime - bt_rq->min_vruntime; + + if (d < 0) + d = -d; + + if (d > 3*sysctl_sched_latency) + schedstat_inc(bt_rq->nr_spread_over); +#endif +} + +static void +place_bt_entity(struct bt_rq *bt_rq, struct sched_entity *se, int initial) +{ + u64 vruntime = bt_rq->min_vruntime; + + /* + * The 'current' period is already promised to the current tasks, + * however the extra weight of the new task will slow them down a + * little, place the new task so that it fits in the slot that + * stays open at the end. + */ + if (initial && sched_feat(START_DEBIT)) + vruntime += sched_bt_vslice(bt_rq, se); + + /* sleeps up to a single latency don't count. */ + if (!initial) { + unsigned long thresh = sysctl_sched_latency; + + /* + * Halve their sleep time's effect, to allow + * for a gentler effect of sleepers: + */ + if (sched_feat(GENTLE_FAIR_SLEEPERS)) + thresh >>= 1; + + vruntime -= thresh; + } + + /* ensure we never gain time by being placed backwards. */ + se->vruntime = max_vruntime(se->vruntime, vruntime); +} + +static void +enqueue_bt_entity(struct bt_rq *bt_rq, struct sched_entity *se, int flags) +{ + /* + * Update the normalized vruntime before updating min_vruntime + * through callig update_curr(). + */ + if (!(flags & ENQUEUE_WAKEUP) || (flags & ENQUEUE_MIGRATED)) + se->vruntime += bt_rq->min_vruntime; + + /* + * Update run-time bt_statistics of the 'current'. + */ + update_curr_bt(bt_rq); + account_bt_entity_enqueue(bt_rq, se); + + if (flags & ENQUEUE_WAKEUP) { + place_bt_entity(bt_rq, se, 0); + enqueue_bt_sleeper(bt_rq, se); + } + + update_stats_enqueue_bt(bt_rq, se); + check_bt_spread(bt_rq, se); + if (se != bt_rq->curr) + __enqueue_bt_entity(bt_rq, se); + se->on_rq = 1; +} + +static void __clear_buddies_last_bt(struct sched_entity *se) +{ + for_each_sched_bt_entity(se) { + struct bt_rq *bt_rq = bt_rq_of(se); + + if (bt_rq->last == se) + bt_rq->last = NULL; + else + break; + } +} + +static void __clear_buddies_next_bt(struct sched_entity *se) +{ + for_each_sched_bt_entity(se) { + struct bt_rq *bt_rq = bt_rq_of(se); + + if (bt_rq->next == se) + bt_rq->next = NULL; + else + break; + } +} + +static void __clear_buddies_skip_bt(struct sched_entity *se) +{ + for_each_sched_bt_entity(se) { + struct bt_rq *bt_rq = bt_rq_of(se); + + if (bt_rq->skip == se) + bt_rq->skip = NULL; + else + break; + } +} + +static void clear_buddies_bt(struct bt_rq *bt_rq, struct sched_entity *se) +{ + if (bt_rq->last == se) + __clear_buddies_last_bt(se); + + if (bt_rq->next == se) + __clear_buddies_next_bt(se); + + if (bt_rq->skip == se) + __clear_buddies_skip_bt(se); +} + +static void +dequeue_bt_entity(struct bt_rq *bt_rq, struct sched_entity *se, int flags) +{ + /* + * Update run-time bt_statistics of the 'current'. + */ + update_curr_bt(bt_rq); + + update_stats_dequeue_bt(bt_rq, se); + if (flags & DEQUEUE_SLEEP) { +#if defined(CONFIG_SCHEDSTATS) || defined(CONFIG_LATENCYTOP) + if (bt_entity_is_task(se)) { + struct task_struct *tsk = bt_task_of(se); + + if (tsk->state & TASK_INTERRUPTIBLE) + se->bt_statistics->sleep_start = rq_of_bt_rq(bt_rq)->clock; + if (tsk->state & TASK_UNINTERRUPTIBLE) + se->bt_statistics->block_start = rq_of_bt_rq(bt_rq)->clock; + } +#endif + } + + clear_buddies_bt(bt_rq, se); + + if (se != bt_rq->curr) + __dequeue_bt_entity(bt_rq, se); + se->on_rq = 0; + account_bt_entity_dequeue(bt_rq, se); + + /* + * Normalize the entity after updating the min_vruntime because the + * update can refer to the ->curr item and we need to reflect this + * movement in our normalized position. + */ + if (!(flags & DEQUEUE_SLEEP)) + se->vruntime -= bt_rq->min_vruntime; + + update_bt_min_vruntime(bt_rq); +} + +/* + * Preempt the current task with a newly woken task if needed: + */ +static void +check_preempt_tick_bt(struct bt_rq *bt_rq, struct sched_entity *curr) +{ + unsigned long ideal_runtime, delta_exec; + struct sched_entity *se; + s64 delta; + + ideal_runtime = sched_bt_slice(bt_rq, curr); + delta_exec = curr->sum_exec_runtime - curr->prev_sum_exec_runtime; + if (delta_exec > ideal_runtime) { + resched_curr(rq_of_bt_rq(bt_rq)); + /* + * The current task ran long enough, ensure it doesn't get + * re-elected due to buddy favours. + */ + clear_buddies_bt(bt_rq, curr); + return; + } + + /* + * Ensure that a task that missed wakeup preemption by a + * narrow margin doesn't have to wait for a full slice. + * This also mitigates buddy induced latencies under load. + */ + if (delta_exec < sysctl_sched_min_granularity) + return; + + se = __pick_first_bt_entity(bt_rq); + delta = curr->vruntime - se->vruntime; + + if (delta < 0) + return; + + if (delta > ideal_runtime) + resched_curr(rq_of_bt_rq(bt_rq)); +} + +static void +set_next_bt_entity(struct bt_rq *bt_rq, struct sched_entity *se) +{ + /* 'current' is not kept within the tree. */ + if (se->on_rq) { + /* + * Any task has to be enqueued before it get to execute on + * a CPU. So account for the time it spent waiting on the + * runqueue. + */ + update_stats_wait_end_bt(bt_rq, se); + __dequeue_bt_entity(bt_rq, se); + } + + update_stats_curr_start_bt(bt_rq, se); + bt_rq->curr = se; +#ifdef CONFIG_SCHEDSTATS + /* + * Track our maximum slice length, if the CPU's load is at + * least twice that of our own weight (i.e. dont track it + * when there are only lesser-weight tasks around): + */ + if (bt_rq->load.weight >= 2*se->load.weight) { + se->bt_statistics->slice_max = max(se->bt_statistics->slice_max, + se->sum_exec_runtime - se->prev_sum_exec_runtime); + } +#endif + se->prev_sum_exec_runtime = se->sum_exec_runtime; +} + +static int +wakeup_preempt_bt_entity(struct sched_entity *curr, struct sched_entity *se); + +/* + * Pick the next process, keeping these things in mind, in this order: + * 1) keep things fair between processes/task groups + * 2) pick the "next" process, since someone really wants that to run + * 3) pick the "last" process, for cache locality + * 4) do not run the "skip" process, if something else is available + */ +static struct sched_entity *pick_next_bt_entity(struct bt_rq *bt_rq) +{ + struct sched_entity *se = __pick_first_bt_entity(bt_rq); + struct sched_entity *left = se; + + /* + * Avoid running the skip buddy, if running something else can + * be done without getting too unfair. + */ + if (bt_rq->skip == se) { + struct sched_entity *second = __pick_next_bt_entity(se); + + if (second && wakeup_preempt_bt_entity(second, left) < 1) + se = second; + } + + /* + * Prefer last buddy, try to return the CPU to a preempted task. + */ + if (bt_rq->last && wakeup_preempt_bt_entity(bt_rq->last, left) < 1) + se = bt_rq->last; + + /* + * Someone really wants this to run. If it's not unfair, run it. + */ + if (bt_rq->next && wakeup_preempt_bt_entity(bt_rq->next, left) < 1) + se = bt_rq->next; + + clear_buddies_bt(bt_rq, se); + + return se; +} + +static void put_prev_bt_entity(struct bt_rq *bt_rq, struct sched_entity *prev) +{ + /* + * If still on the runqueue then deactivate_task() + * was not called and update_curr() has to be done: + */ + if (prev->on_rq) + update_curr_bt(bt_rq); + + check_bt_spread(bt_rq, prev); + if (prev->on_rq) { + update_stats_wait_start_bt(bt_rq, prev); + /* Put 'current' back into the tree. */ + __enqueue_bt_entity(bt_rq, prev); + } + bt_rq->curr = NULL; +} + +static void +bt_entity_tick(struct bt_rq *bt_rq, struct sched_entity *curr, int queued) +{ + /* + * Update run-time bt_statistics of the 'current'. + */ + update_curr_bt(bt_rq); + +#ifdef CONFIG_SCHED_HRTICK + /* + * queued ticks are scheduled to match the slice, so don't bother + * validating it and just reschedule. + */ + if (queued) { + resched_curr(rq_of_bt_rq(bt_rq)); + return; + } +#endif + + if (bt_rq->nr_running > 1) + check_preempt_tick_bt(bt_rq, curr); +} + +/* + * The enqueue_task method is called before nr_running is + * increased. Here we update the fair scheduling stats and + * then put the task into the rbtree: + */ +static void +enqueue_task_bt(struct rq *rq, struct task_struct *p, int flags) +{ + struct bt_rq *bt_rq; + struct sched_entity *se = &p->bt; + + for_each_sched_bt_entity(se) { + if (se->on_rq) + break; + bt_rq = bt_rq_of(se); + enqueue_bt_entity(bt_rq, se, flags); + + bt_rq->h_nr_running++; + + flags = ENQUEUE_WAKEUP; + } + + if (!se) { + rq->bt_nr_running++; + add_nr_running(rq, 1); + } +} + +static void set_next_buddy_bt(struct sched_entity *se); + +/* + * The dequeue_task method is called before nr_running is + * decreased. We remove the task from the rbtree and + * update the fair scheduling stats: + */ +static void dequeue_task_bt(struct rq *rq, struct task_struct *p, int flags) +{ + struct bt_rq *bt_rq; + struct sched_entity *se = &p->bt; + int task_sleep = flags & DEQUEUE_SLEEP; + + for_each_sched_bt_entity(se) { + bt_rq = bt_rq_of(se); + dequeue_bt_entity(bt_rq, se, flags); + + bt_rq->h_nr_running--; + + /* Don't dequeue parent if it has other entities besides us */ + if (bt_rq->load.weight) { + /* + * Bias pick_next to pick a task from this cfs_rq, as + * p is sleeping when it is within its sched_slice. + */ + if (task_sleep && parent_bt_entity(se)) + set_next_buddy_bt(parent_bt_entity(se)); + + /* avoid re-evaluating load for this entity */ + se = parent_bt_entity(se); + break; + } + flags |= DEQUEUE_SLEEP; + } + + if (!se) { + sub_nr_running(rq, 1); + rq->bt_nr_running--; + } +} + +#ifdef CONFIG_SMP +/** + * idle_cpu - is a given cpu idle currently? + * @cpu: the processor in question. + */ +static int idle_cpu_bt(int cpu) +{ + struct rq *rq = cpu_rq(cpu); + + if (rq->curr != rq->idle) + return 0; + + if (rq->nr_running) + return 0; + +#ifdef CONFIG_SMP + if (!llist_empty(&rq->wake_list)) + return 0; +#endif + + return 1; +} + +static int select_idle_sibling_bt(struct task_struct *p, int target) +{ + struct sched_domain *sd; + struct sched_group *sg; + int i = task_cpu(p); + + if (idle_cpu_bt(target)) + return target; + + /* + * If the prevous cpu is cache affine and idle, don't be stupid. + */ + if (i != target && cpus_share_cache(i, target) && idle_cpu_bt(i)) + return i; + + /* + * Otherwise, iterate the domains and find an elegible idle cpu. + */ + sd = rcu_dereference(per_cpu(sd_llc, target)); + for_each_lower_domain(sd) { + sg = sd->groups; + do { + if (!cpumask_intersects(sched_group_cpus(sg), + tsk_cpus_allowed(p))) + goto next; + + for_each_cpu(i, sched_group_cpus(sg)) { + if (i == target || !idle_cpu_bt(i)) + goto next; + } + + target = cpumask_first_and(sched_group_cpus(sg), + tsk_cpus_allowed(p)); + goto done; +next: + sg = sg->next; + } while (sg != sd->groups); + } +done: + return target; +} + +static int select_idle_cpu(struct task_struct *p, int target) +{ + struct sched_domain *sd; + struct sched_group *sg; + int i = task_cpu(p); + + if (idle_cpu_bt(target)) + return target; + + /* + * If the prevous cpu is cache affine and idle, don't be stupid. + */ + if (i != target && cpus_share_cache(i, target) && idle_cpu_bt(i)) + return i; + + /* + * Otherwise, iterate the domains and find an elegible idle cpu. + */ + sd = rcu_dereference(per_cpu(sd_llc, target)); + for_each_lower_domain(sd) { + sg = sd->groups; + do { + if (!cpumask_intersects(sched_group_cpus(sg), + tsk_cpus_allowed(p))) + goto next; + + for_each_cpu(i, sched_group_cpus(sg)) { + if (idle_cpu_bt(i) && cpumask_test_cpu(i, tsk_cpus_allowed(p))) { + target = i; + goto done; + } + } +next: + sg = sg->next; + } while (sg != sd->groups); + } + +done: + return target; +} + + +static int +select_task_rq_bt(struct task_struct *p, int prev_cpu, int sd_flag, int wake_flags) +{ + struct sched_domain *tmp, *affine_sd = NULL, *sd = NULL; + int cpu = smp_processor_id(); + int new_cpu = prev_cpu; + int want_affine = 0; + + if (p->nr_cpus_allowed == 1) + return prev_cpu; + + if (sd_flag & SD_BALANCE_WAKE) { + if (cpumask_test_cpu(cpu, tsk_cpus_allowed(p))) + want_affine = 1; + new_cpu = prev_cpu; + } + + rcu_read_lock(); + for_each_domain(cpu, tmp) { + if (!(tmp->flags & SD_LOAD_BALANCE)) + continue; + + /* + * If both cpu and prev_cpu are part of this domain, + * cpu is a valid SD_WAKE_AFFINE target. + */ + if (want_affine && (tmp->flags & SD_WAKE_AFFINE) && + cpumask_test_cpu(prev_cpu, sched_domain_span(tmp))) { + affine_sd = tmp; + break; + } + + if (tmp->flags & sd_flag) + sd = tmp; + } + + if (affine_sd) { + new_cpu = select_idle_sibling_bt(p, prev_cpu); + goto unlock; + } + + new_cpu = select_idle_cpu(p, prev_cpu); + +unlock: + rcu_read_unlock(); + + return new_cpu; +} +#endif + +static unsigned long +wakeup_gran_bt(struct sched_entity *curr, struct sched_entity *se) +{ + unsigned long gran = sysctl_sched_wakeup_granularity; + + /* + * Since its curr running now, convert the gran from real-time + * to virtual-time in his units. + * + * By using 'se' instead of 'curr' we penalize light tasks, so + * they get preempted easier. That is, if 'se' < 'curr' then + * the resulting gran will be larger, therefore penalizing the + * lighter, if otoh 'se' > 'curr' then the resulting gran will + * be smaller, again penalizing the lighter task. + * + * This is especially important for buddies when the leftmost + * task is higher priority than the buddy. + */ + return calc_delta_bt(gran, se); +} + +/* + * Should 'se' preempt 'curr'. + * + * |s1 + * |s2 + * |s3 + * g + * |<--->|c + * + * w(c, s1) = -1 + * w(c, s2) = 0 + * w(c, s3) = 1 + * + */ +static int +wakeup_preempt_bt_entity(struct sched_entity *curr, struct sched_entity *se) +{ + s64 gran, vdiff = curr->vruntime - se->vruntime; + + if (vdiff <= 0) + return -1; + + gran = wakeup_gran_bt(curr, se); + if (vdiff > gran) + return 1; + + return 0; +} + +static void set_last_buddy_bt(struct sched_entity *se) +{ + if (bt_entity_is_task(se)) + return; + + for_each_sched_bt_entity(se) + bt_rq_of(se)->last = se; +} + +static void set_next_buddy_bt(struct sched_entity *se) +{ + if (bt_entity_is_task(se)) + return; + + for_each_sched_bt_entity(se) + bt_rq_of(se)->next = se; +} + +static void set_skip_buddy_bt(struct sched_entity *se) +{ + for_each_sched_bt_entity(se) + bt_rq_of(se)->skip = se; +} + +/* + * Preempt the current task with a newly woken task if needed: + */ +static void check_preempt_wakeup_bt(struct rq *rq, struct task_struct *p, int wake_flags) +{ + struct task_struct *curr = rq->curr; + struct sched_entity *se = &curr->bt, *pse = &p->bt; + struct bt_rq *bt_rq = task_bt_rq(curr); + int scale = bt_rq->nr_running >= sched_nr_latency; + int next_buddy_marked = 0; + + if (unlikely(se == pse)) + return; + + if (sched_feat(NEXT_BUDDY) && scale && !(wake_flags & WF_FORK)) { + set_next_buddy_bt(pse); + next_buddy_marked = 1; + } + + /* + * We can come here with TIF_NEED_RESCHED already set from new task + * wake up path. + * + * Note: this also catches the edge-case of curr being in a throttled + * group (e.g. via set_curr_task), since update_curr() (in the + * enqueue of curr) will have resulted in resched being set. This + * prevents us from potentially nominating it as a false LAST_BUDDY + * below. + */ + if (test_tsk_need_resched(curr)) + return; + + /* BT tasks are by definition preempted by non-bt tasks. */ + if (likely(p->policy < SCHED_BT)) + goto preempt; + + if (!sched_feat(WAKEUP_PREEMPTION)) + return; + + update_curr_bt(bt_rq_of(se)); + BUG_ON(!pse); + if (wakeup_preempt_bt_entity(se, pse) == 1) { + /* + * Bias pick_next to pick the sched entity that is + * triggering this preemption. + */ + if (!next_buddy_marked) + set_next_buddy_bt(pse); + goto preempt; + } + + return; + +preempt: + resched_curr(rq); + /* + * Only set the backward buddy when the current task is still + * on the rq. This can happen when a wakeup gets interleaved + * with schedule on the ->pre_schedule() or idle_balance() + * point, either of which can * drop the rq lock. + * + * Also, during early boot the idle thread is in the fair class, + * for obvious reasons its a bad idea to schedule back to it. + */ + if (unlikely(!se->on_rq || curr == rq->idle)) + return; + + if (sched_feat(LAST_BUDDY) && scale && bt_entity_is_task(se)) + set_last_buddy_bt(se); +} + +static struct task_struct *pick_next_task_bt(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) +{ + struct task_struct *p; + struct bt_rq *bt_rq; + struct sched_entity *se; + int new_tasks = -1; + + bt_rq = &rq->bt; +again: + if (!bt_rq->nr_running) + goto idle; + + put_prev_task(rq, prev); + + se = pick_next_bt_entity(bt_rq); + set_next_bt_entity(bt_rq, se); + + p = bt_task_of(se); + + return p; + +idle: + new_tasks = idle_balance(rq, rf); + + /* + * Because idle_balance() releases (and re-acquires) rq->lock, it is + * possible for any higher priority task to appear. In that case we + * must re-start the pick_next_entity() loop. + */ + if (new_tasks < 0) + return RETRY_TASK; + + if (new_tasks > 0) + goto again; + + return NULL; +} + +/* + * Account for a descheduled task: + */ +static void put_prev_task_bt(struct rq *rq, struct task_struct *prev) +{ + struct sched_entity *se = &prev->bt; + struct bt_rq *bt_rq; + + for_each_sched_bt_entity(se) { + bt_rq = bt_rq_of(se); + put_prev_bt_entity(bt_rq, se); + } +} + +/* + * sched_yield() is very simple + * + * The magic of dealing with the ->skip buddy is in pick_next_entity. + */ +static void yield_task_bt(struct rq *rq) +{ + struct task_struct *curr = rq->curr; + struct bt_rq *bt_rq = task_bt_rq(curr); + struct sched_entity *se = &curr->bt; + + /* + * Are we the only task in the tree? + */ + if (unlikely(rq->bt_nr_running == 1)) + return; + + clear_buddies_bt(bt_rq, se); + + update_rq_clock(rq); + /* + * Update run-time bt_statistics of the 'current'. + */ + update_curr_bt(bt_rq); + /* + * Tell update_rq_clock() that we've just updated, + * so we don't do microscopic update in schedule() + * and double the fastpath cost. + */ +// rq->skip_clock_update = 1; + + set_skip_buddy_bt(se); +} + +static bool yield_to_task_bt(struct rq *rq, struct task_struct *p, bool preempt) +{ + struct sched_entity *se = &p->bt; + + if (!se->on_rq) + return false; + + /* Tell the scheduler that we'd really like pse to run next. */ + set_next_buddy_bt(se); + + yield_task_bt(rq); + + return true; +} + +#ifdef CONFIG_SMP +static void rq_online_bt(struct rq *rq) +{ + update_sysctl(); +} + +static void rq_offline_bt(struct rq *rq) +{ + update_sysctl(); +} + +#endif /* CONFIG_SMP */ + +/* + * scheduler tick hitting a task of our scheduling class: + */ +static void task_tick_bt(struct rq *rq, struct task_struct *curr, int queued) +{ + struct bt_rq *bt_rq; + struct sched_entity *se = &curr->bt; + + for_each_sched_bt_entity(se) { + bt_rq = bt_rq_of(se); + bt_entity_tick(bt_rq, se, queued); + } + + if (static_branch_unlikely(&sched_numa_balancing)) + task_tick_numa(rq, curr); +} + +/* + * called on fork with the child task as argument from the parent's context + * - child not yet on the tasklist + * - preemption disabled + */ +static void task_fork_bt(struct task_struct *p) +{ + struct bt_rq *bt_rq; + struct sched_entity *se = &p->bt, *curr; + int this_cpu = smp_processor_id(); + struct rq *rq = this_rq(); + unsigned long flags; + + raw_spin_lock_irqsave(&rq->lock, flags); + + update_rq_clock(rq); + + bt_rq = task_bt_rq(current); + curr = bt_rq->curr; + + /* + * Not only the cpu but also the task_group of the parent might have + * been changed after parent->se.parent,cfs_rq were copied to + * child->se.parent,cfs_rq. So call __set_task_cpu() to make those + * of child point to valid ones. + */ + rcu_read_lock(); + __set_task_cpu(p, this_cpu); + rcu_read_unlock(); + + update_curr_bt(bt_rq); + + if (curr) + se->vruntime = curr->vruntime; + place_bt_entity(bt_rq, se, 1); + + if (sysctl_sched_child_runs_first && curr && bt_entity_before(curr, se)) { + /* + * Upon rescheduling, sched_class::put_prev_task() will place + * 'current' within the tree based on its new key value. + */ + swap(curr->vruntime, se->vruntime); + resched_curr(rq); + } + + se->vruntime -= bt_rq->min_vruntime; + + raw_spin_unlock_irqrestore(&rq->lock, flags); +} + +/* + * Priority of the task has changed. Check to see if we preempt + * the current task. + */ +static void +prio_changed_bt(struct rq *rq, struct task_struct *p, int oldprio) +{ + if (!p->bt.on_rq) + return; + + /* + * Reschedule if we are currently running on this runqueue and + * our priority decreased, or if we are not currently running on + * this runqueue and our priority is higher than the current's + */ + if (rq->curr == p) { + if (p->prio > oldprio) + resched_curr(rq); + } else + check_preempt_curr(rq, p, 0); +} + +static void switched_from_bt(struct rq *rq, struct task_struct *p) +{ + struct sched_entity *se = &p->bt; + struct bt_rq *bt_rq = bt_rq_of(se); + + /* + * Ensure the task's vruntime is normalized, so that when it's + * switched back to the fair class the enqueue_entity(.flags=0) will + * do the right thing. + * + * If it's on_rq, then the dequeue_entity(.flags=0) will already + * have normalized the vruntime, if it's !on_rq, then only when + * the task is sleeping will it still have non-normalized vruntime. + */ + if (!p->on_rq && p->state != TASK_RUNNING) { + /* + * Fix up our vruntime so that the current sleep doesn't + * cause 'unlimited' sleep bonus. + */ + place_bt_entity(bt_rq, se, 0); + se->vruntime -= bt_rq->min_vruntime; + } +} + +/* + * We switched to the sched_fair class. + */ +static void switched_to_bt(struct rq *rq, struct task_struct *p) +{ + BUG_ON(!bt_prio(p->static_prio)); + +// attach_task_bt_rq(p); + + if (!p->bt.on_rq) + return; + + /* + * We were most likely switched from sched_rt, so + * kick off the schedule if running, otherwise just see + * if we can still preempt the current task. + */ + if (rq->curr == p) + resched_curr(rq); + else + check_preempt_curr(rq, p, 0); +} + +/* Account for a task changing its policy or group. + * + * This routine is mostly called to set cfs_rq->curr field when a task + * migrates between groups/classes. + */ +static void set_curr_task_bt(struct rq *rq) +{ + struct sched_entity *se = &rq->curr->bt; + + for_each_sched_bt_entity(se) { + struct bt_rq *bt_rq = bt_rq_of(se); + + set_next_bt_entity(bt_rq, se); + } +} + +void init_bt_rq(struct bt_rq *bt_rq) +{ + bt_rq->tasks_timeline.rb_root = RB_ROOT; + bt_rq->tasks_timeline.rb_leftmost = NULL; + bt_rq->min_vruntime = (u64)(-(1LL << 20)); +#ifndef CONFIG_64BIT + bt_rq->min_vruntime_copy = bt_rq->min_vruntime; +#endif +} + +static unsigned int get_rr_interval_bt(struct rq *rq, struct task_struct *task) +{ + struct sched_entity *se = &task->bt; + unsigned int rr_interval = 0; + + /* + * Time slice is 0 for SCHED_OTHER tasks that are on an otherwise + * idle runqueue: + */ + if (rq->bt.load.weight) + rr_interval = NS_TO_JIFFIES(sched_bt_slice(bt_rq_of(se), se)); + + return rr_interval; +} + +/* + * All the scheduling class methods: + */ +const struct sched_class bt_sched_class = { + .next = &idle_sched_class, + .enqueue_task = enqueue_task_bt, + .dequeue_task = dequeue_task_bt, + .yield_task = yield_task_bt, + .yield_to_task = yield_to_task_bt, + + .check_preempt_curr = check_preempt_wakeup_bt, + + .pick_next_task = pick_next_task_bt, + .put_prev_task = put_prev_task_bt, + +#ifdef CONFIG_SMP + .select_task_rq = select_task_rq_bt, + .rq_online = rq_online_bt, + .rq_offline = rq_offline_bt, +#endif + + .set_curr_task = set_curr_task_bt, + .task_tick = task_tick_bt, + .task_fork = task_fork_bt, + + .prio_changed = prio_changed_bt, + .switched_from = switched_from_bt, + .switched_to = switched_to_bt, + + .get_rr_interval = get_rr_interval_bt, +}; diff --git a/kernel/sched/batch.h b/kernel/sched/batch.h new file mode 100644 index 000000000..7bc4ce267 --- /dev/null +++ b/kernel/sched/batch.h @@ -0,0 +1,83 @@ +/* + * Copyright (C) 2019 Tencent Ltd. All rights reserved. + * + * File Name ：batch.h + * Author ： + * Date ：2019-12-26 + * Descriptor： + */ + +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _BATCH_H +#define _BATCH_H + +#include +#include + +/* nflag of task_struct */ +#define TNF_SCHED_BT 0x00000001 + + +struct bt_rq { + struct load_weight load; + unsigned int nr_running, h_nr_running; + unsigned long nr_uninterruptible; + + u64 exec_clock; + u64 min_vruntime; +#ifndef CONFIG_64BIT + u64 min_vruntime_copy; +#endif + + struct rb_root_cached tasks_timeline; + struct rb_node *rb_leftmost; + + /* + * 'curr' points to currently running entity on this bt_rq. + * It is set to NULL otherwise (i.e when none are currently running). + */ + struct sched_entity *curr, *next, *last, *skip; + +#ifdef CONFIG_SCHED_DEBUG + unsigned int nr_spread_over; +#endif + +#ifdef CONFIG_SMP +/* + * Load-tracking only depends on SMP, BT_GROUP_SCHED dependency below may be + * removed when useful for applications beyond shares distribution (e.g. + * load-balance). + */ + + + /* + * h_load = weight * f(tg) + * + * Where f(tg) is the recursive weight fraction assigned to + * this group. + */ + unsigned long h_load; +#endif /* CONFIG_SMP */ +}; + +extern const struct sched_class bt_sched_class; + +extern void init_bt_rq(struct bt_rq *bt_rq); +extern struct sched_entity *__pick_first_bt_entity(struct bt_rq *bt_rq); +extern struct sched_entity *__pick_last_bt_entity(struct bt_rq *bt_rq); +extern void set_bt_load_weight(struct task_struct *p); + + +static inline int bt_policy(int policy) +{ + if (policy == SCHED_BT) + return 1; + return 0; +} + +static inline int task_has_bt_policy(struct task_struct *p) +{ + return bt_policy(p->policy); +} + +#endif diff --git a/kernel/sched/bt_debug.c b/kernel/sched/bt_debug.c new file mode 100644 index 000000000..0bb5f240d --- /dev/null +++ b/kernel/sched/bt_debug.c @@ -0,0 +1,72 @@ +/* + * Copyright (C) 2019 Tencent Ltd. All rights reserved. + * + * File Name : bt_debug.c + * Author : + * Date : 2019-12-26 + * Descriptor: + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "sched.h" +#include "bt_debug.h" + +void print_bt_rq(struct seq_file *m, int cpu, struct bt_rq *bt_rq) +{ + s64 MIN_vruntime = -1, min_vruntime, max_vruntime = -1, + spread, rq0_min_vruntime, spread0; + struct rq *rq = cpu_rq(cpu); + struct sched_entity *last; + unsigned long flags; + + SEQ_printf(m, "\nbt_rq[%d]:\n", cpu); + + SEQ_printf(m, " .%-30s: %lld.%06ld\n", "exec_clock", + SPLIT_NS(bt_rq->exec_clock)); + + raw_spin_lock_irqsave(&rq->lock, flags); + if (bt_rq->rb_leftmost) + MIN_vruntime = (__pick_first_bt_entity(bt_rq))->vruntime; + last = __pick_last_bt_entity(bt_rq); + if (last) + max_vruntime = last->vruntime; + min_vruntime = bt_rq->min_vruntime; + rq0_min_vruntime = cpu_rq(0)->bt.min_vruntime; + raw_spin_unlock_irqrestore(&rq->lock, flags); + SEQ_printf(m, " .%-30s: %lld.%06ld\n", "MIN_vruntime", + SPLIT_NS(MIN_vruntime)); + SEQ_printf(m, " .%-30s: %lld.%06ld\n", "min_vruntime", + SPLIT_NS(min_vruntime)); + SEQ_printf(m, " .%-30s: %lld.%06ld\n", "max_vruntime", + SPLIT_NS(max_vruntime)); + spread = max_vruntime - MIN_vruntime; + SEQ_printf(m, " .%-30s: %lld.%06ld\n", "spread", + SPLIT_NS(spread)); + spread0 = min_vruntime - rq0_min_vruntime; + SEQ_printf(m, " .%-30s: %lld.%06ld\n", "spread0", + SPLIT_NS(spread0)); + SEQ_printf(m, " .%-30s: %d\n", "nr_spread_over", + bt_rq->nr_spread_over); + SEQ_printf(m, " .%-30s: %d\n", "nr_running", bt_rq->nr_running); + SEQ_printf(m, " .%-30s: %ld\n", "load", bt_rq->load.weight); +} + +#ifdef CONFIG_SCHED_DEBUG +void print_bt_stats(struct seq_file *m, int cpu) +{ + struct bt_rq *bt_rq; + + rcu_read_lock(); + bt_rq = &cpu_rq(cpu)->bt; + print_bt_rq(m, cpu, bt_rq); + rcu_read_unlock(); +} +#endif diff --git a/kernel/sched/bt_debug.h b/kernel/sched/bt_debug.h new file mode 100644 index 000000000..869f80fdc --- /dev/null +++ b/kernel/sched/bt_debug.h @@ -0,0 +1,36 @@ +/* + * Copyright (C) 2019 Tencent Ltd. All rights reserved. + * + * File Name : bt_debug.h + * Author : + * Date : 2019-12-26 + * Descriptor: + */ + +#ifndef _BT_STAT_H +#define _BT_STAT_H + +/* + * This allows printing both to /proc/sched_debug and + * to the console + */ +#define SEQ_printf(m, x...) \ +do { \ + if (m) \ + seq_printf(m, x); \ + else \ + printk(x); \ +} while (0) + +extern long long nsec_high(unsigned long long nsec); +extern unsigned long nsec_low(unsigned long long nsec); +#define SPLIT_NS(x) (nsec_high(x), nsec_low(x)) + +extern void print_bt_stats(struct seq_file *m, int cpu); +#ifdef CONFIG_SCHED_DEBUG +extern void print_bt_stats(struct seq_file *m, int cpu); +#else +void print_bt_stats(struct seq_file *m, int cpu) {} +#endif + +#endif diff --git a/kernel/sched/core.c b/kernel/sched/core.c index ee2032d96..5ac14598b 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -34,6 +35,7 @@ #endif #include "sched.h" +#include "batch.h" #include "../workqueue_internal.h" #include "../smpboot.h" @@ -775,16 +777,38 @@ static inline void dequeue_task(struct rq *rq, struct task_struct *p, int flags) void activate_task(struct rq *rq, struct task_struct *p, int flags) { +#ifdef CONFIG_BT_SCHED + if (task_contributes_to_load(p)) { + rq->nr_uninterruptible--; + + if (unlikely(p->flags & TNF_SCHED_BT)) { + p->flags &= ~TNF_SCHED_BT; + rq->bt.nr_uninterruptible--; + } + } +#else if (task_contributes_to_load(p)) rq->nr_uninterruptible--; +#endif enqueue_task(rq, p, flags); } void deactivate_task(struct rq *rq, struct task_struct *p, int flags) { +#ifdef CONFIG_BT_SCHED + if (task_contributes_to_load(p)) { + rq->nr_uninterruptible++; + + if (unlikely(p->sched_class == &bt_sched_class)) { + p->flags |= TNF_SCHED_BT; + rq->bt.nr_uninterruptible++; + } + } +#else if (task_contributes_to_load(p)) rq->nr_uninterruptible++; +#endif dequeue_task(rq, p, flags); } @@ -1718,8 +1742,19 @@ ttwu_do_activate(struct rq *rq, struct task_struct *p, int wake_flags, lockdep_assert_held(&rq->lock); #ifdef CONFIG_SMP +#ifdef CONFIG_BT_SCHED + if (p->sched_contributes_to_load) { + rq->nr_uninterruptible--; + + if (unlikely(p->flags & TNF_SCHED_BT)) { + p->flags &= ~TNF_SCHED_BT; + rq->bt.nr_uninterruptible--; + } + } +#else if (p->sched_contributes_to_load) rq->nr_uninterruptible--; +#endif if (wake_flags & WF_MIGRATED) en_flags |= ENQUEUE_MIGRATED; @@ -2183,6 +2218,15 @@ static void __sched_fork(unsigned long clone_flags, struct task_struct *p) p->se.vruntime = 0; INIT_LIST_HEAD(&p->se.group_node); +#ifdef CONFIG_BT_SCHED + p->bt.on_rq = 0; + p->bt.exec_start = 0; + p->bt.sum_exec_runtime = 0; + p->bt.prev_sum_exec_runtime = 0; + p->bt.nr_migrations = 0; + p->bt.vruntime = 0; +#endif + #ifdef CONFIG_FAIR_GROUP_SCHED p->se.cfs_rq = NULL; #endif @@ -2190,6 +2234,9 @@ static void __sched_fork(unsigned long clone_flags, struct task_struct *p) #ifdef CONFIG_SCHEDSTATS /* Even if schedstat is disabled, there should not be garbage */ memset(&p->se.statistics, 0, sizeof(p->se.statistics)); +#ifdef CONFIG_BT_SCHED + p->bt.bt_statistics = &p->se.statistics; +#endif #endif RB_CLEAR_NODE(&p->dl.rb_node); @@ -2389,6 +2436,10 @@ int sched_fork(unsigned long clone_flags, struct task_struct *p) return -EAGAIN; } else if (rt_prio(p->prio)) { p->sched_class = &rt_sched_class; +#ifdef CONFIG_BT_SCHED + } else if(bt_prio(p->prio)){ + p->sched_class = &bt_sched_class; +#endif } else { p->sched_class = &fair_sched_class; } @@ -3003,7 +3054,7 @@ unsigned long long task_sched_runtime(struct task_struct *p) * been accounted, so we're correct here as well. */ if (!p->on_cpu || !task_on_rq_queued(p)) - return p->se.sum_exec_runtime; + return TASK_SUM_EXEC_RUNTIME(p); #endif rq = task_rq_lock(p, &rf); @@ -3017,7 +3068,7 @@ unsigned long long task_sched_runtime(struct task_struct *p) update_rq_clock(rq); p->sched_class->update_curr(rq); } - ns = p->se.sum_exec_runtime; + ns = TASK_SUM_EXEC_RUNTIME(p); task_rq_unlock(rq, p, &rf); return ns; @@ -3752,6 +3803,14 @@ void rt_mutex_setprio(struct task_struct *p, struct task_struct *pi_task) if (oldprio < prio) queue_flag |= ENQUEUE_HEAD; p->sched_class = &rt_sched_class; +#ifdef CONFIG_BT_SCHED + } else if (bt_prio(prio)) { + if (dl_prio(oldprio)) + p->dl.dl_boosted = 0; + if (rt_prio(oldprio)) + p->rt.timeout = 0; + p->sched_class = &bt_sched_class; +#endif } else { if (dl_prio(oldprio)) p->dl.dl_boosted = 0; @@ -3816,8 +3875,17 @@ void set_user_nice(struct task_struct *p, long nice) if (running) put_prev_task(rq, p); - p->static_prio = NICE_TO_PRIO(nice); - set_load_weight(p); +#ifdef CONFIG_BT_SCHED + if (task_has_bt_policy(p)) { + p->static_prio = NICE_TO_BT_PRIO(nice); + set_bt_load_weight(p); + } else +#endif + { + p->static_prio = NICE_TO_PRIO(nice); + set_load_weight(p); + } + old_prio = p->prio; p->prio = effective_prio(p); delta = p->prio - old_prio; @@ -3900,6 +3968,29 @@ int task_prio(const struct task_struct *p) return p->prio - MAX_RT_PRIO; } +/** + * task_nice - return the nice value of a given task. + * @p: the task in question. + * + * Return: The nice value [ -20 ... 0 ... 19 ]. + */ +inline int task_nice(const struct task_struct *p) +{ +#ifdef CONFIG_BT_SCHED + int task_nice = 0; + + if(bt_prio(p->static_prio)) + task_nice = TASK_BT_NICE(p); + else + task_nice = PRIO_TO_NICE((p)->static_prio); + + return task_nice; +#else + return PRIO_TO_NICE((p)->static_prio); +#endif +} +EXPORT_SYMBOL(task_nice); + /** * idle_cpu - is a given CPU idle currently? * @cpu: the processor in question. @@ -3967,6 +4058,15 @@ static void __setscheduler_params(struct task_struct *p, else if (fair_policy(policy)) p->static_prio = NICE_TO_PRIO(attr->sched_nice); +#ifdef CONFIG_BT_SCHED + if (unlikely(policy == SCHED_BT)) { + bt_prio_adjust_pos(&p->static_prio); + set_bt_load_weight(p); + } else if (policy == SCHED_NORMAL || policy == SCHED_BATCH || + policy == SCHED_IDLE) { + bt_prio_adjust_neg(&p->static_prio); + } +#endif /* * __sched_setscheduler() ensures attr->sched_priority == 0 when * !rt_policy. Always setting this ensures that things like @@ -3995,6 +4095,10 @@ static void __setscheduler(struct rq *rq, struct task_struct *p, p->sched_class = &dl_sched_class; else if (rt_prio(p->prio)) p->sched_class = &rt_sched_class; +#ifdef CONFIG_BT_SCHED + else if (bt_prio(p->prio)) + p->sched_class = &bt_sched_class; +#endif else p->sched_class = &fair_sched_class; } @@ -4031,6 +4135,10 @@ static int __sched_setscheduler(struct task_struct *p, /* The pi code expects interrupts enabled */ BUG_ON(pi && in_interrupt()); + + if(!sched_bt_on && SCHED_BT == policy) + return -EINVAL; + recheck: /* Double check policy once rq lock held: */ if (policy < 0) { @@ -5257,6 +5365,12 @@ void init_idle(struct task_struct *idle, int cpu) __sched_fork(0, idle); idle->state = TASK_RUNNING; idle->se.exec_start = sched_clock(); +#ifdef CONFIG_BT_SCHED + idle->bt.exec_start = idle->se.exec_start; +#if defined(CONFIG_SCHEDSTATS) || defined(CONFIG_LATENCYTOP) + idle->bt.bt_statistics = &idle->se.statistics; +#endif +#endif idle->flags |= PF_IDLE; kasan_unpoison_task_stack(idle); @@ -5881,6 +5995,10 @@ void __init sched_init(void) init_cfs_rq(&rq->cfs); init_rt_rq(&rq->rt); init_dl_rq(&rq->dl); +#ifdef CONFIG_BT_SCHED + rq->bt_nr_running = 0; + init_bt_rq(&rq->bt); +#endif #ifdef CONFIG_FAIR_GROUP_SCHED root_task_group.shares = ROOT_TASK_GROUP_LOAD; INIT_LIST_HEAD(&rq->leaf_cfs_rq_list); @@ -6069,6 +6187,9 @@ void normalize_rt_tasks(void) continue; p->se.exec_start = 0; +#ifdef CONFIG_BT_SCHED + p->bt.exec_start = 0; +#endif schedstat_set(p->se.statistics.wait_start, 0); schedstat_set(p->se.statistics.sleep_start, 0); schedstat_set(p->se.statistics.block_start, 0); diff --git a/kernel/sched/cpupri.c b/kernel/sched/cpupri.c index 2511aba36..755b75e36 100644 --- a/kernel/sched/cpupri.c +++ b/kernel/sched/cpupri.c @@ -30,6 +30,7 @@ #include #include #include +#include #include #include "cpupri.h" @@ -40,7 +41,11 @@ static int convert_prio(int prio) if (prio == CPUPRI_INVALID) cpupri = CPUPRI_INVALID; +#ifdef CONFIG_BT_SCHED + else if (prio >= MAX_PRIO - BT_PRIO_WIDTH) +#else else if (prio == MAX_PRIO) +#endif cpupri = CPUPRI_IDLE; else if (prio >= MAX_RT_PRIO) cpupri = CPUPRI_NORMAL; diff --git a/kernel/sched/cputime.c b/kernel/sched/cputime.c index 14d2dbf97..708be6d9b 100644 --- a/kernel/sched/cputime.c +++ b/kernel/sched/cputime.c @@ -108,6 +108,11 @@ static inline void task_group_account_field(struct task_struct *p, int index, * */ __this_cpu_add(kernel_cpustat.cpustat[index], tmp); +#ifdef CONFIG_BT_SCHED + if(p->sched_class == &bt_sched_class) { + __this_cpu_add(kernel_cpustat.cpustat[CPUTIME_BT], tmp); + } +#endif cpuacct_account_field(p, index, tmp); } @@ -273,7 +278,7 @@ static inline u64 account_other_time(u64 max) #ifdef CONFIG_64BIT static inline u64 read_sum_exec_runtime(struct task_struct *t) { - return t->se.sum_exec_runtime; + return TASK_SUM_EXEC_RUNTIME(t); } #else static u64 read_sum_exec_runtime(struct task_struct *t) @@ -283,7 +288,7 @@ static u64 read_sum_exec_runtime(struct task_struct *t) struct rq *rq; rq = task_rq_lock(t, &rf); - ns = t->se.sum_exec_runtime; + ns = TASK_SUM_EXEC_RUNTIME(t); task_rq_unlock(rq, t, &rf); return ns; @@ -661,7 +666,7 @@ out: void task_cputime_adjusted(struct task_struct *p, u64 *ut, u64 *st) { struct task_cputime cputime = { - .sum_exec_runtime = p->se.sum_exec_runtime, + .sum_exec_runtime = TASK_SUM_EXEC_RUNTIME(p), }; task_cputime(p, &cputime.utime, &cputime.stime); diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c index 2f93e4a2d..1901bc5ae 100644 --- a/kernel/sched/debug.c +++ b/kernel/sched/debug.c @@ -20,25 +20,14 @@ #include #include "sched.h" +#include "bt_debug.h" static DEFINE_SPINLOCK(sched_debug_lock); -/* - * This allows printing both to /proc/sched_debug and - * to the console - */ -#define SEQ_printf(m, x...) \ - do { \ - if (m) \ - seq_printf(m, x); \ - else \ - printk(x); \ - } while (0) - /* * Ease the printing of nsec fields: */ -static long long nsec_high(unsigned long long nsec) +long long nsec_high(unsigned long long nsec) { if ((long long)nsec < 0) { nsec = -nsec; @@ -50,7 +39,7 @@ static long long nsec_high(unsigned long long nsec) return nsec; } -static unsigned long nsec_low(unsigned long long nsec) +unsigned long nsec_low(unsigned long long nsec) { if ((long long)nsec < 0) nsec = -nsec; @@ -58,8 +47,6 @@ static unsigned long nsec_low(unsigned long long nsec) return do_div(nsec, 1000000); } -#define SPLIT_NS(x) nsec_high(x), nsec_low(x) - #define SCHED_FEAT(name, enabled) \ #name , @@ -705,6 +692,9 @@ do { \ print_cfs_stats(m, cpu); print_rt_stats(m, cpu); print_dl_stats(m, cpu); +#ifdef CONFIG_BT_SCHED + print_bt_stats(m, cpu); +#endif print_rq(m, rq, cpu); spin_unlock_irqrestore(&sched_debug_lock, flags); @@ -1004,6 +994,9 @@ void proc_sched_show_task(struct task_struct *p, struct pid_namespace *ns, "nr_involuntary_switches", (long long)p->nivcsw); P(se.load.weight); +#ifdef CONFIG_BT_SCHED + SEQ_printf(m, "%-45s:%21Ld\n", "bt.load.weight", (long long)scale_load_down(p->bt.load.weight)); +#endif #ifdef CONFIG_SMP P(se.avg.load_sum); P(se.avg.util_sum); diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index f36c0ac8e..d4dd7588c 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -37,6 +37,10 @@ #include #include "sched.h" +#include "fair.h" +#ifdef CONFIG_BT_SCHED +#include "batch.h" +#endif /* * Targeted preemption latency for CPU-bound tasks: @@ -78,7 +82,7 @@ unsigned int normalized_sysctl_sched_min_granularity = 750000ULL; /* * This value is kept at sysctl_sched_latency/sysctl_sched_min_granularity */ -static unsigned int sched_nr_latency = 8; +unsigned int sched_nr_latency = 8; /* * After fork, child runs first. If set to 0 (default) then @@ -132,19 +136,19 @@ unsigned int sysctl_sched_cfs_bandwidth_slice = 5000UL; */ unsigned int capacity_margin = 1280; -static inline void update_load_add(struct load_weight *lw, unsigned long inc) +inline void update_load_add(struct load_weight *lw, unsigned long inc) { lw->weight += inc; lw->inv_weight = 0; } -static inline void update_load_sub(struct load_weight *lw, unsigned long dec) +inline void update_load_sub(struct load_weight *lw, unsigned long dec) { lw->weight -= dec; lw->inv_weight = 0; } -static inline void update_load_set(struct load_weight *lw, unsigned long w) +inline void update_load_set(struct load_weight *lw, unsigned long w) { lw->weight = w; lw->inv_weight = 0; @@ -180,7 +184,7 @@ static unsigned int get_update_sysctl_factor(void) return factor; } -static void update_sysctl(void) +void update_sysctl(void) { unsigned int factor = get_update_sysctl_factor(); @@ -229,7 +233,7 @@ static void __update_inv_weight(struct load_weight *lw) * Or, weight =< lw.weight (because lw.weight is the runqueue weight), thus * weight/lw.weight <= 1, and therefore our shift will also be positive. */ -static u64 __calc_delta(u64 delta_exec, unsigned long weight, struct load_weight *lw) +u64 __calc_delta(u64 delta_exec, unsigned long weight, struct load_weight *lw) { u64 fact = scale_load_down(weight); int shift = WMULT_SHIFT; @@ -490,25 +494,6 @@ void account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec); /************************************************************** * Scheduling class tree data structure manipulation methods: */ - -static inline u64 max_vruntime(u64 max_vruntime, u64 vruntime) -{ - s64 delta = (s64)(vruntime - max_vruntime); - if (delta > 0) - max_vruntime = vruntime; - - return max_vruntime; -} - -static inline u64 min_vruntime(u64 min_vruntime, u64 vruntime) -{ - s64 delta = (s64)(vruntime - min_vruntime); - if (delta < 0) - min_vruntime = vruntime; - - return min_vruntime; -} - static inline int entity_before(struct sched_entity *a, struct sched_entity *b) { @@ -663,7 +648,7 @@ static inline u64 calc_delta_fair(u64 delta, struct sched_entity *se) * * p = (nr <= nl) ? l : l*nr/nl */ -static u64 __sched_period(unsigned long nr_running) +u64 __sched_period(unsigned long nr_running) { if (unlikely(nr_running > sched_nr_latency)) return nr_running * sysctl_sched_min_granularity; @@ -2637,7 +2622,7 @@ void task_tick_numa(struct rq *rq, struct task_struct *curr) * task needs to have done some actual work before we bother with * NUMA placement. */ - now = curr->se.sum_exec_runtime; + now = TASK_SUM_EXEC_RUNTIME(curr); period = (u64)curr->numa_scan_period * NSEC_PER_MSEC; if (now > curr->node_stamp + period) { @@ -2653,7 +2638,7 @@ void task_tick_numa(struct rq *rq, struct task_struct *curr) } #else -static void task_tick_numa(struct rq *rq, struct task_struct *curr) +void task_tick_numa(struct rq *rq, struct task_struct *curr) { } @@ -3547,7 +3532,7 @@ static inline unsigned long cfs_rq_load_avg(struct cfs_rq *cfs_rq) return cfs_rq->avg.load_avg; } -static int idle_balance(struct rq *this_rq, struct rq_flags *rf); +int idle_balance(struct rq *this_rq, struct rq_flags *rf); #else /* CONFIG_SMP */ @@ -3576,7 +3561,7 @@ attach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) {} static inline void detach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) {} -static inline int idle_balance(struct rq *rq, struct rq_flags *rf) +inline int idle_balance(struct rq *rq, struct rq_flags *rf) { return 0; } @@ -5327,7 +5312,11 @@ static unsigned long cpu_avg_load_per_task(int cpu) struct rq *rq = cpu_rq(cpu); unsigned long nr_running = READ_ONCE(rq->cfs.h_nr_running); unsigned long load_avg = weighted_cpuload(rq); +#ifdef CONFIG_BT_SCHED + unsigned long bt_running = READ_ONCE(rq->bt.h_nr_running); + nr_running -= bt_running; +#endif if (nr_running) return load_avg / nr_running; @@ -6230,7 +6219,11 @@ static void check_preempt_wakeup(struct rq *rq, struct task_struct *p, int wake_ /* Idle tasks are by definition preempted by non-idle tasks. */ if (unlikely(curr->policy == SCHED_IDLE) && +#ifdef CONFIG_BT_SCHED + likely(p->policy != SCHED_IDLE && p->policy != SCHED_BT)) +#else likely(p->policy != SCHED_IDLE)) +#endif goto preempt; /* @@ -6279,11 +6272,17 @@ pick_next_task_fair(struct rq *rq, struct task_struct *prev, struct rq_flags *rf struct cfs_rq *cfs_rq = &rq->cfs; struct sched_entity *se; struct task_struct *p; +#ifndef CONFIG_BT_SCHED int new_tasks; again: +#endif if (!cfs_rq->nr_running) +#ifdef CONFIG_BT_SCHED + return NULL; +#else goto idle; +#endif #ifdef CONFIG_FAIR_GROUP_SCHED if (prev->sched_class != &fair_sched_class) @@ -6322,7 +6321,11 @@ again: cfs_rq = &rq->cfs; if (!cfs_rq->nr_running) +#ifdef CONFIG_BT_SCHED + return NULL; +#else goto idle; +#endif goto simple; } @@ -6381,7 +6384,7 @@ simple: hrtick_start_fair(rq, p); return p; - +#ifndef CONFIG_BT_SCHED idle: new_tasks = idle_balance(rq, rf); @@ -6397,6 +6400,7 @@ idle: goto again; return NULL; +#endif } /* @@ -6427,7 +6431,7 @@ static void yield_task_fair(struct rq *rq) /* * Are we the only task in the tree? */ - if (unlikely(rq->nr_running == 1)) + if (unlikely(RQ_CFS_NR_RUNNING(rq) == 1)) return; clear_buddies(cfs_rq, se); @@ -7430,7 +7434,7 @@ static inline void update_sg_lb_stats(struct lb_env *env, sgs->group_util += cpu_util(i); sgs->sum_nr_running += rq->cfs.h_nr_running; - nr_running = rq->nr_running; + nr_running = RQ_CFS_NR_RUNNING(rq); if (nr_running > 1) *overload = true; @@ -7975,7 +7979,7 @@ static struct rq *find_busiest_queue(struct lb_env *env, * which is not scaled with the cpu capacity. */ - if (rq->nr_running == 1 && wl > env->imbalance && + if (RQ_CFS_NR_RUNNING(rq) == 1 && wl > env->imbalance && !check_cpu_capacity(rq, env->sd)) continue; @@ -8135,7 +8139,7 @@ redo: env.src_rq = busiest; ld_moved = 0; - if (busiest->nr_running > 1) { + if (RQ_CFS_NR_RUNNING(busiest) > 1) { /* * Attempt to move tasks. If find_busiest_group has found * an imbalance but busiest->nr_running <= 1, the group is @@ -8143,7 +8147,7 @@ redo: * correctly treated as an imbalance. */ env.flags |= LBF_ALL_PINNED; - env.loop_max = min(sysctl_sched_nr_migrate, busiest->nr_running); + env.loop_max = min(sysctl_sched_nr_migrate, RQ_CFS_NR_RUNNING(busiest)); more_balance: rq_lock_irqsave(busiest, &rf); @@ -8377,7 +8381,7 @@ update_next_balance(struct sched_domain *sd, unsigned long *next_balance) * idle_balance is called by schedule() if this_cpu is about to become * idle. Attempts to pull tasks from other CPUs. */ -static int idle_balance(struct rq *this_rq, struct rq_flags *rf) +int idle_balance(struct rq *this_rq, struct rq_flags *rf) { unsigned long next_balance = jiffies + HZ; int this_cpu = this_rq->cpu; @@ -8518,7 +8522,7 @@ static int active_load_balance_cpu_stop(void *data) goto out_unlock; /* Is there any task to move? */ - if (busiest_rq->nr_running <= 1) + if (RQ_CFS_NR_RUNNING(busiest_rq) <= 1) goto out_unlock; /* @@ -8916,7 +8920,11 @@ static inline bool nohz_kick_needed(struct rq *rq) int nr_busy, i, cpu = rq->cpu; bool kick = false; +#ifdef CONFIG_BT_SCHED + if (unlikely(rq->idle_balance && !rq->nr_running)) +#else if (unlikely(rq->idle_balance)) +#endif return false; /* @@ -9521,7 +9529,11 @@ static unsigned int get_rr_interval_fair(struct rq *rq, struct task_struct *task * All the scheduling class methods: */ const struct sched_class fair_sched_class = { +#ifdef CONFIG_BT_SCHED + .next = &bt_sched_class, +#else .next = &idle_sched_class, +#endif .enqueue_task = enqueue_task_fair, .dequeue_task = dequeue_task_fair, .yield_task = yield_task_fair, diff --git a/kernel/sched/fair.h b/kernel/sched/fair.h new file mode 100644 index 000000000..020b7d171 --- /dev/null +++ b/kernel/sched/fair.h @@ -0,0 +1,47 @@ +/* + * Copyright (C) 2019 Tencent Ltd. All rights reserved. + * + * File Name ：fair.h + * Author ： + * Date ：2019-12-26 + * Descriptor： + */ + +#ifndef _FAIR_H +#define _FAIR_H + +extern unsigned int sched_nr_latency; + +unsigned long calc_delta_mine(unsigned long delta_exec, + unsigned long weight, struct load_weight *lw); + +extern u64 +__calc_delta(u64 delta_exec, unsigned long weight, struct load_weight *lw); +extern int idle_balance(struct rq *this_rq, struct rq_flags *rf); +static inline u64 max_vruntime(u64 max_vruntime, u64 vruntime) +{ + s64 delta = (s64)(vruntime - max_vruntime); + + if (delta > 0) + max_vruntime = vruntime; + + return max_vruntime; +} + +static inline u64 min_vruntime(u64 min_vruntime, u64 vruntime) +{ + s64 delta = (s64)(vruntime - min_vruntime); + + if (delta < 0) + min_vruntime = vruntime; + + return min_vruntime; +} + +u64 __sched_period(unsigned long nr_running); + +void task_tick_numa(struct rq *rq, struct task_struct *curr); + +void update_sysctl(void); + +#endif diff --git a/kernel/sched/loadavg.c b/kernel/sched/loadavg.c index 89a989e4d..e7b328ff0 100644 --- a/kernel/sched/loadavg.c +++ b/kernel/sched/loadavg.c @@ -84,8 +84,8 @@ long calc_load_fold_active(struct rq *this_rq, long adjust) { long nr_active, delta = 0; - nr_active = this_rq->nr_running - adjust; - nr_active += (long)this_rq->nr_uninterruptible; + nr_active = RQ_CFS_NR_RUNNING(this_rq) - adjust; + nr_active += (long)RQ_CFS_NR_UNINTERRUPTIBLE(this_rq); if (nr_active != this_rq->calc_load_active) { delta = nr_active - this_rq->calc_load_active; diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 452b56923..d5a4c8647 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -39,6 +39,7 @@ #include "cpupri.h" #include "cpudeadline.h" #include "cpuacct.h" +#include "batch.h" #ifdef CONFIG_SCHED_DEBUG # define SCHED_WARN_ON(x) WARN_ONCE(x, #x) @@ -114,6 +115,23 @@ static inline void cpu_load_update_active(struct rq *this_rq) { } */ #define DL_SCALE (10) + +#ifdef CONFIG_BT_SCHED +#define RQ_CFS_NR_UNINTERRUPTIBLE(rq) \ + ((rq)->nr_uninterruptible - (rq)->bt.nr_uninterruptible) + +#define RQ_CFS_NR_RUNNING(rq) \ + ((rq)->nr_running - (rq)->bt_nr_running) + +#else +#define RQ_CFS_NR_UNINTERRUPTIBLE(rq) \ + ((rq)->nr_uninterruptible) + +#define RQ_CFS_NR_RUNNING(rq) \ + ((rq)->nr_running) + +#endif + /* * These are the 'tuning knobs' of the scheduler: */ @@ -141,9 +159,13 @@ static inline int dl_policy(int policy) { return policy == SCHED_DEADLINE; } + static inline bool valid_policy(int policy) { return idle_policy(policy) || fair_policy(policy) || +#ifdef CONFIG_BT_SCHED + bt_policy(policy) || +#endif rt_policy(policy) || dl_policy(policy); } @@ -690,6 +712,9 @@ struct rq { * remote CPUs use both these fields when doing load calculation. */ unsigned int nr_running; +#ifdef CONFIG_BT_SCHED + unsigned int bt_nr_running; +#endif #ifdef CONFIG_NUMA_BALANCING unsigned int nr_numa_running; unsigned int nr_preferred_running; @@ -713,6 +738,9 @@ struct rq { struct cfs_rq cfs; struct rt_rq rt; struct dl_rq dl; +#ifdef CONFIG_BT_SCHED + struct bt_rq bt; +#endif #ifdef CONFIG_FAIR_GROUP_SCHED /* list of leaf cfs_rq on this cpu: */ @@ -1106,6 +1134,13 @@ struct sched_group { unsigned long cpumask[0]; }; +#define tsk_cpus_allowed(tsk) (&(tsk)->cpus_allowed) + +static inline struct cpumask *sched_group_cpus(struct sched_group *sg) +{ + return to_cpumask(sg->cpumask); +} + static inline struct cpumask *sched_group_span(struct sched_group *sg) { return to_cpumask(sg->cpumask); @@ -1357,6 +1392,9 @@ static inline void finish_lock_switch(struct rq *rq, struct task_struct *prev) #define WF_FORK 0x02 /* child wakeup after fork */ #define WF_MIGRATED 0x4 /* internal use, task got migrated */ +extern inline void update_load_add(struct load_weight *lw, unsigned long inc); +extern inline void update_load_sub(struct load_weight *lw, unsigned long dec); +extern inline void update_load_set(struct load_weight *lw, unsigned long w); /* * To aid in avoiding the subversion of "niceness" due to uneven distribution * of tasks with abnormal "nice" values across CPUs the contribution that @@ -1601,7 +1639,7 @@ static inline void add_nr_running(struct rq *rq, unsigned count) if (prev_nr < 2 && rq->nr_running >= 2) { #ifdef CONFIG_SMP - if (!rq->rd->overload) + if (!rq->rd->overload && RQ_CFS_NR_RUNNING(rq) >= 2) rq->rd->overload = true; #endif } diff --git a/kernel/time/posix-cpu-timers.c b/kernel/time/posix-cpu-timers.c index 6e8c230ca..056a9a2fc 100644 --- a/kernel/time/posix-cpu-timers.c +++ b/kernel/time/posix-cpu-timers.c @@ -819,7 +819,7 @@ static void check_thread_timers(struct task_struct *tsk, tsk_expires->virt_exp = expires; tsk_expires->sched_exp = check_timers_list(++timers, firing, - tsk->se.sum_exec_runtime); + TASK_SUM_EXEC_RUNTIME(tsk)); /* * Check for the special case thread timers. @@ -1082,7 +1082,7 @@ static inline int fastpath_timer_check(struct task_struct *tsk) struct task_cputime task_sample; task_cputime(tsk, &task_sample.utime, &task_sample.stime); - task_sample.sum_exec_runtime = tsk->se.sum_exec_runtime; + task_sample.sum_exec_runtime = TASK_SUM_EXEC_RUNTIME(tsk); if (task_cputime_expired(&task_sample, &tsk->cputime_expires)) return 1; } diff --git a/package/default/config.default b/package/default/config.default index 85598294e..c53d9421b 100644 --- a/package/default/config.default +++ b/package/default/config.default @@ -184,6 +184,7 @@ CONFIG_IPC_NS=y CONFIG_USER_NS=y CONFIG_PID_NS=y CONFIG_NET_NS=y +CONFIG_BT_SCHED=y CONFIG_SCHED_AUTOGROUP=y # CONFIG_SYSFS_DEPRECATED is not set CONFIG_RELAY=y