diff --git a/fs/proc/Makefile b/fs/proc/Makefile index f7456c4e7..1f7b57472 100644 --- a/fs/proc/Makefile +++ b/fs/proc/Makefile @@ -26,6 +26,7 @@ proc-y += softirqs.o proc-y += namespaces.o proc-y += self.o proc-y += thread_self.o +proc-$(CONFIG_BT_SCHED) += bt_stat.o proc-$(CONFIG_PROC_SYSCTL) += proc_sysctl.o proc-$(CONFIG_NET) += proc_net.o proc-$(CONFIG_PROC_KCORE) += kcore.o diff --git a/fs/proc/base.c b/fs/proc/base.c index a628d0925..65ef7d5d8 100644 --- a/fs/proc/base.c +++ b/fs/proc/base.c @@ -481,7 +481,7 @@ static int proc_pid_schedstat(struct seq_file *m, struct pid_namespace *ns, seq_printf(m, "0 0 0\n"); else seq_printf(m, "%llu %llu %lu\n", - (unsigned long long)task->se.sum_exec_runtime, + TASK_SUM_EXEC_RUNTIME(task), (unsigned long long)task->sched_info.run_delay, task->sched_info.pcount); diff --git a/fs/proc/bt_stat.c b/fs/proc/bt_stat.c new file mode 100644 index 000000000..49b64f774 --- /dev/null +++ b/fs/proc/bt_stat.c @@ -0,0 +1,182 @@ +/* + * Copyright (C) 2019 Tencent Ltd. All rights reserved. + * + * File Name ：bt_stat.c + * Author ： + * Date ：2019-12-28 + * Descriptor： + */ + +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +extern u64 get_idle_time(int cpu); +extern u64 get_iowait_time(int cpu); + +static int show_bt_stat(struct seq_file *p, void *v) +{ + int i, j; + unsigned long jif; + u64 user, nice, system, idle, iowait, irq, softirq, steal, cputime_bt; + u64 guest, guest_nice; + u64 sum = 0; + u64 sum_softirq = 0; + unsigned int per_softirq_sums[NR_SOFTIRQS] = {0}; + struct timespec boottime; + + user = nice = system = idle = iowait = + irq = softirq = steal = cputime_bt = 0; + guest = guest_nice = 0; + getboottime(&boottime); + jif = boottime.tv_sec; + + for_each_possible_cpu(i) { + user += kcpustat_cpu(i).cpustat[CPUTIME_USER]; + nice += kcpustat_cpu(i).cpustat[CPUTIME_NICE]; + system += kcpustat_cpu(i).cpustat[CPUTIME_SYSTEM]; + idle += get_idle_time(i); + iowait += get_iowait_time(i); + irq += kcpustat_cpu(i).cpustat[CPUTIME_IRQ]; + softirq += kcpustat_cpu(i).cpustat[CPUTIME_SOFTIRQ]; + steal += kcpustat_cpu(i).cpustat[CPUTIME_STEAL]; + guest += kcpustat_cpu(i).cpustat[CPUTIME_GUEST]; + guest_nice += kcpustat_cpu(i).cpustat[CPUTIME_GUEST_NICE]; + cputime_bt += kcpustat_cpu(i).cpustat[CPUTIME_BT]; + sum += kstat_cpu_irqs_sum(i); + sum += arch_irq_stat_cpu(i); + + for (j = 0; j < NR_SOFTIRQS; j++) { + unsigned int softirq_stat = kstat_softirqs_cpu(j, i); + + per_softirq_sums[j] += softirq_stat; + sum_softirq += softirq_stat; + } + } + sum += arch_irq_stat(); + + seq_puts(p, "cpu "); + seq_put_decimal_ull(p, " ", nsec_to_clock_t(user)); + seq_put_decimal_ull(p, " ", nsec_to_clock_t(nice)); + seq_put_decimal_ull(p, " ", nsec_to_clock_t(system)); + seq_put_decimal_ull(p, " ", nsec_to_clock_t(idle)); + seq_put_decimal_ull(p, " ", nsec_to_clock_t(iowait)); + seq_put_decimal_ull(p, " ", nsec_to_clock_t(irq)); + seq_put_decimal_ull(p, " ", nsec_to_clock_t(softirq)); + seq_put_decimal_ull(p, " ", nsec_to_clock_t(steal)); + seq_put_decimal_ull(p, " ", nsec_to_clock_t(guest)); + seq_put_decimal_ull(p, " ", nsec_to_clock_t(guest_nice)); + seq_put_decimal_ull(p, " ", nsec_to_clock_t(cputime_bt)); + seq_putc(p, '\n'); + + for_each_online_cpu(i) { + /* Copy values here to work around gcc-2.95.3, gcc-2.96 */ + user = kcpustat_cpu(i).cpustat[CPUTIME_USER]; + nice = kcpustat_cpu(i).cpustat[CPUTIME_NICE]; + system = kcpustat_cpu(i).cpustat[CPUTIME_SYSTEM]; + idle = get_idle_time(i); + iowait = get_iowait_time(i); + irq = kcpustat_cpu(i).cpustat[CPUTIME_IRQ]; + softirq = kcpustat_cpu(i).cpustat[CPUTIME_SOFTIRQ]; + steal = kcpustat_cpu(i).cpustat[CPUTIME_STEAL]; + guest = kcpustat_cpu(i).cpustat[CPUTIME_GUEST]; + guest_nice = kcpustat_cpu(i).cpustat[CPUTIME_GUEST_NICE]; + cputime_bt = kcpustat_cpu(i).cpustat[CPUTIME_BT]; + seq_printf(p, "cpu%d", i); + seq_put_decimal_ull(p, " ", nsec_to_clock_t(user)); + seq_put_decimal_ull(p, " ", nsec_to_clock_t(nice)); + seq_put_decimal_ull(p, " ", nsec_to_clock_t(system)); + seq_put_decimal_ull(p, " ", nsec_to_clock_t(idle)); + seq_put_decimal_ull(p, " ", nsec_to_clock_t(iowait)); + seq_put_decimal_ull(p, " ", nsec_to_clock_t(irq)); + seq_put_decimal_ull(p, " ", nsec_to_clock_t(softirq)); + seq_put_decimal_ull(p, " ", nsec_to_clock_t(steal)); + seq_put_decimal_ull(p, " ", nsec_to_clock_t(guest)); + seq_put_decimal_ull(p, " ", nsec_to_clock_t(guest_nice)); + seq_put_decimal_ull(p, " ", nsec_to_clock_t(cputime_bt)); + seq_putc(p, '\n'); + } + seq_printf(p, "intr %llu", (unsigned long long)sum); + + /* sum again ? it could be updated? */ + for_each_irq_nr(j) + seq_put_decimal_ull(p, " ", kstat_irqs(j)); + + seq_printf(p, + "\nctxt %llu\n" + "btime %lu\n" + "processes %d\n" + "procs_running %lu\n" + "procs_blocked %lu\n", + nr_context_switches(), + (unsigned long)jif, + nr_forks(), + nr_running(), + nr_iowait()); + + seq_printf(p, "softirq %llu", (unsigned long long)sum_softirq); + + for (i = 0; i < NR_SOFTIRQS; i++) + seq_put_decimal_ull(p, " ", per_softirq_sums[i]); + seq_putc(p, '\n'); + + return 0; +} + +static int bt_stat_open(struct inode *inode, struct file *file) +{ + size_t size = 1024 + 128 * num_online_cpus(); + char *buf; + struct seq_file *m; + int res; + + /* minimum size to display an interrupt count : 2 bytes */ + size += 2 * nr_irqs; + + /* don't ask for more than the kmalloc() max size */ + if (size > KMALLOC_MAX_SIZE) + size = KMALLOC_MAX_SIZE; + buf = kmalloc(size, GFP_KERNEL); + if (!buf) + return -ENOMEM; + + res = single_open(file, show_bt_stat, NULL); + if (!res) { + m = file->private_data; + m->buf = buf; + m->size = ksize(buf); + } else { + kfree(buf); + } + + return res; +} + +static const struct file_operations proc_bt_stat_operations = { + .open = bt_stat_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +static int __init proc_bt_stat_init(void) +{ + if(sched_bt_on) + proc_create("bt_stat", 0000, NULL, &proc_bt_stat_operations); + + return 0; +} +fs_initcall(proc_bt_stat_init); diff --git a/fs/proc/stat.c b/fs/proc/stat.c index 3179babe7..4bbeb3fc9 100644 --- a/fs/proc/stat.c +++ b/fs/proc/stat.c @@ -23,7 +23,7 @@ #ifdef arch_idle_time -static u64 get_idle_time(int cpu) +u64 get_idle_time(int cpu) { u64 idle; @@ -33,7 +33,7 @@ static u64 get_idle_time(int cpu) return idle; } -static u64 get_iowait_time(int cpu) +u64 get_iowait_time(int cpu) { u64 iowait; @@ -45,7 +45,7 @@ static u64 get_iowait_time(int cpu) #else -static u64 get_idle_time(int cpu) +u64 get_idle_time(int cpu) { u64 idle, idle_usecs = -1ULL; @@ -61,7 +61,7 @@ static u64 get_idle_time(int cpu) return idle; } -static u64 get_iowait_time(int cpu) +u64 get_iowait_time(int cpu) { u64 iowait, iowait_usecs = -1ULL; diff --git a/include/linux/init_task.h b/include/linux/init_task.h index 8062e6cc6..b96a1eeb7 100644 --- a/include/linux/init_task.h +++ b/include/linux/init_task.h @@ -219,6 +219,15 @@ extern struct cred init_cred; #define INIT_TASK_SECURITY #endif +#ifdef CONFIG_BT_SCHED +#define INIT_PRIO MAX_PRIO - 20 - 40 +#define INIT_STATIC_PRIO MAX_PRIO - 20 - 40 +#define INIT_NORMAL_PRIO MAX_PRIO - 20 - 40 +#else +#define INIT_PRIO MAX_PRIO - 20 +#define INIT_STATIC_PRIO MAX_PRIO - 20 +#define INIT_NORMAL_PRIO MAX_PRIO - 20 +#endif /* * INIT_TASK is used to set up the first task table, touch at * your own risk!. Base=0, limit=0x1fffff (=2MB) @@ -230,9 +239,9 @@ extern struct cred init_cred; .stack = init_stack, \ .usage = ATOMIC_INIT(2), \ .flags = PF_KTHREAD, \ - .prio = MAX_PRIO-20, \ - .static_prio = MAX_PRIO-20, \ - .normal_prio = MAX_PRIO-20, \ + .prio = INIT_PRIO, \ + .static_prio = INIT_STATIC_PRIO, \ + .normal_prio = INIT_NORMAL_PRIO, \ .policy = SCHED_NORMAL, \ .cpus_allowed = CPU_MASK_ALL, \ .nr_cpus_allowed= NR_CPUS, \ diff --git a/include/linux/ioprio.h b/include/linux/ioprio.h index 2cdd74809..bf2539adc 100644 --- a/include/linux/ioprio.h +++ b/include/linux/ioprio.h @@ -61,7 +61,11 @@ static inline int task_nice_ioprio(struct task_struct *task) */ static inline int task_nice_ioclass(struct task_struct *task) { +#ifdef CONFIG_BT_SCHED + if (task->policy == SCHED_IDLE || task->policy == SCHED_BT) +#else if (task->policy == SCHED_IDLE) +#endif return IOPRIO_CLASS_IDLE; else if (task->policy == SCHED_FIFO || task->policy == SCHED_RR) return IOPRIO_CLASS_RT; diff --git a/include/linux/kernel_stat.h b/include/linux/kernel_stat.h index bfad33f11..a991b3c57 100644 --- a/include/linux/kernel_stat.h +++ b/include/linux/kernel_stat.h @@ -28,6 +28,9 @@ enum cpu_usage_stat { CPUTIME_STEAL, CPUTIME_GUEST, CPUTIME_GUEST_NICE, +#ifdef CONFIG_BT_SCHED + CPUTIME_BT, +#endif NR_STATS, }; diff --git a/include/linux/sched.h b/include/linux/sched.h index 866439c36..9c5c15f09 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -111,6 +111,16 @@ struct task_group; (task->flags & PF_FROZEN) == 0 && \ (task->state & TASK_NOLOAD) == 0) +#ifdef CONFIG_BT_SCHED +#define TASK_SUM_EXEC_RUNTIME(tsk) \ + (unsigned long long)((tsk)->se.sum_exec_runtime + (tsk)->bt.sum_exec_runtime) + +#else +#define TASK_SUM_EXEC_RUNTIME(tsk) \ + (unsigned long long)((tsk)->se.sum_exec_runtime) + +#endif + #ifdef CONFIG_DEBUG_ATOMIC_SLEEP /* @@ -429,6 +439,9 @@ struct sched_entity { u64 nr_migrations; struct sched_statistics statistics; +#ifdef CONFIG_BT_SCHED + struct sched_statistics *bt_statistics; +#endif #ifdef CONFIG_FAIR_GROUP_SCHED int depth; @@ -601,6 +614,9 @@ struct task_struct { const struct sched_class *sched_class; struct sched_entity se; +#ifdef CONFIG_BT_SCHED + struct sched_entity bt; +#endif struct sched_rt_entity rt; #ifdef CONFIG_CGROUP_SCHED struct task_group *sched_task_group; @@ -1477,17 +1493,7 @@ extern int yield_to(struct task_struct *p, bool preempt); extern void set_user_nice(struct task_struct *p, long nice); extern int task_prio(const struct task_struct *p); -/** - * task_nice - return the nice value of a given task. - * @p: the task in question. - * - * Return: The nice value [ -20 ... 0 ... 19 ]. - */ -static inline int task_nice(const struct task_struct *p) -{ - return PRIO_TO_NICE((p)->static_prio); -} - +extern int task_nice(const struct task_struct *p); extern int can_nice(const struct task_struct *p, const int nice); extern int task_curr(const struct task_struct *p); extern int idle_cpu(int cpu); diff --git a/include/linux/sched/batch.h b/include/linux/sched/batch.h new file mode 100644 index 000000000..4932340fd --- /dev/null +++ b/include/linux/sched/batch.h @@ -0,0 +1,60 @@ +/* + * Copyright (C) 2019 Tencent Ltd. All rights reserved. + * + * File Name ：batch.h + * Author ： + * Date ：2019-12-26 + * Descriptor： + */ + +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _SCHED_BATCH_H +#define _SCHED_BATCH_H + +#define MAX_CFS_PRIO 139 +#define MIN_BT_PRIO 140 +#define MAX_BT_PRIO 179 +#define BT_PRIO_WIDTH (MAX_BT_PRIO - MIN_BT_PRIO + 1) + +/* + * Convert user-nice values [ -20 ... 0 ... 19 ] + * to bt static priority [ MIN_BT_PRI + 1 ..MAX_BT_PRIO ], + * and back. + */ +#define NICE_TO_BT_PRIO(nice) (MAX_RT_PRIO + (nice) + 20 + 40) +#define PRIO_TO_BT_NICE(prio) ((prio) - MAX_RT_PRIO - 20 - 40) +#define TASK_BT_NICE(p) PRIO_TO_BT_NICE((p)->static_prio) + +extern unsigned int sched_bt_on; + +static inline int cfs_prio(int prio) +{ + if (prio >= MAX_RT_PRIO && prio < MIN_BT_PRIO) + return 1; + return 0; +} + +static inline int bt_prio(int prio) +{ + if (prio > MAX_CFS_PRIO && prio < MAX_PRIO) + return 1; + return 0; +} + +static inline void bt_prio_adjust_pos(int *prio) +{ + int priority = *prio; + + if (cfs_prio(priority)) + *prio = priority + BT_PRIO_WIDTH; +} + +static inline void bt_prio_adjust_neg(int *prio) +{ + int priority = *prio; + + if (bt_prio(priority)) + *prio = priority - BT_PRIO_WIDTH; +} + +#endif /* _SCHED_BATCH_H */ diff --git a/include/linux/sched/prio.h b/include/linux/sched/prio.h index 7d64feafc..84a615de1 100644 --- a/include/linux/sched/prio.h +++ b/include/linux/sched/prio.h @@ -22,7 +22,11 @@ #define MAX_USER_RT_PRIO 100 #define MAX_RT_PRIO MAX_USER_RT_PRIO +#ifdef CONFIG_BT_SCHED +#define MAX_PRIO (MAX_RT_PRIO + NICE_WIDTH + 40) +#else #define MAX_PRIO (MAX_RT_PRIO + NICE_WIDTH) +#endif #define DEFAULT_PRIO (MAX_RT_PRIO + NICE_WIDTH / 2) /* diff --git a/include/uapi/linux/sched.h b/include/uapi/linux/sched.h index 30a9e51bb..e988c49d4 100644 --- a/include/uapi/linux/sched.h +++ b/include/uapi/linux/sched.h @@ -40,6 +40,9 @@ /* SCHED_ISO: reserved but not implemented yet */ #define SCHED_IDLE 5 #define SCHED_DEADLINE 6 +#ifdef CONFIG_BT_SCHED +#define SCHED_BT 7 +#endif /* Can be ORed in to make sure the process is reverted back to SCHED_NORMAL on fork */ #define SCHED_RESET_ON_FORK 0x40000000 diff --git a/init/Kconfig b/init/Kconfig index 46075327c..36ed5a079 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -947,6 +947,12 @@ config NET_NS endif # NAMESPACES +config BT_SCHED + bool "offline sched class" + default n + help + Allow user to create offline task + config SCHED_AUTOGROUP bool "Automatic process group scheduling" select CGROUPS diff --git a/init/main.c b/init/main.c index c4a45145e..2254d098f 100644 --- a/init/main.c +++ b/init/main.c @@ -353,6 +353,15 @@ static int __init rdinit_setup(char *str) } __setup("rdinit=", rdinit_setup); +unsigned int sched_bt_on; +static int __init set_sched_bt_on(char *str) +{ + sched_bt_on = 1; + + return 1; +} +early_param("offline_class", set_sched_bt_on); + #ifndef CONFIG_SMP static const unsigned int setup_max_cpus = NR_CPUS; static inline void setup_nr_cpu_ids(void) { } diff --git a/kernel/delayacct.c b/kernel/delayacct.c index ca8ac2824..3bfe068a6 100644 --- a/kernel/delayacct.c +++ b/kernel/delayacct.c @@ -115,7 +115,7 @@ int __delayacct_add_tsk(struct taskstats *d, struct task_struct *tsk) */ t1 = tsk->sched_info.pcount; t2 = tsk->sched_info.run_delay; - t3 = tsk->se.sum_exec_runtime; + t3 = TASK_SUM_EXEC_RUNTIME(tsk); d->cpu_count += t1; diff --git a/kernel/exit.c b/kernel/exit.c index 5523fb0c2..86a467256 100644 --- a/kernel/exit.c +++ b/kernel/exit.c @@ -150,7 +150,7 @@ static void __exit_signal(struct task_struct *tsk) sig->inblock += task_io_get_inblock(tsk); sig->oublock += task_io_get_oublock(tsk); task_io_accounting_add(&sig->ioac, &tsk->ioac); - sig->sum_sched_runtime += tsk->se.sum_exec_runtime; + sig->sum_sched_runtime += TASK_SUM_EXEC_RUNTIME(tsk); sig->nr_threads--; __unhash_process(tsk, group_dead); write_sequnlock(&sig->stats_lock); diff --git a/kernel/sched/Makefile b/kernel/sched/Makefile index a9ee16bbc..fdadac8c8 100644 --- a/kernel/sched/Makefile +++ b/kernel/sched/Makefile @@ -20,6 +20,7 @@ obj-y += core.o loadavg.o clock.o cputime.o obj-y += idle_task.o fair.o rt.o deadline.o obj-y += wait.o wait_bit.o swait.o completion.o idle.o obj-$(CONFIG_SMP) += cpupri.o cpudeadline.o topology.o stop_task.o +obj-$(CONFIG_BT_SCHED) += batch.o bt_debug.o obj-$(CONFIG_SCHED_AUTOGROUP) += autogroup.o obj-$(CONFIG_SCHEDSTATS) += stats.o obj-$(CONFIG_SCHED_DEBUG) += debug.o diff --git a/kernel/sched/batch.c b/kernel/sched/batch.c new file mode 100644 index 000000000..5d7679395 --- /dev/null +++ b/kernel/sched/batch.c @@ -0,0 +1,1446 @@ +/* + * Copyright (C) 2019 Tencent Ltd. All rights reserved. + * + * File Name : batch.c + * Author : + * Date : 2019-12-26 + * Descriptor: + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include "batch.h" +#include "sched.h" +#include "fair.h" + +void set_bt_load_weight(struct task_struct *p) +{ + int prio = p->static_prio - MIN_BT_PRIO; + struct load_weight *load = &p->bt.load; + + load->weight = scale_load(sched_prio_to_weight[prio]); + load->inv_weight = sched_prio_to_wmult[prio]; +} + +const struct sched_class bt_sched_class; + +static inline struct task_struct *bt_task_of(struct sched_entity *bt_se) +{ + return container_of(bt_se, struct task_struct, bt); +} + +static inline struct rq *rq_of_bt_rq(struct bt_rq *bt_rq) +{ + return container_of(bt_rq, struct rq, bt); +} + +#define bt_entity_is_task(bt) 1 + +#define for_each_sched_bt_entity(bt) \ + for (; bt; bt = NULL) + +static inline struct bt_rq *task_bt_rq(struct task_struct *p) +{ + return &task_rq(p)->bt; +} + +static inline struct bt_rq *bt_rq_of(struct sched_entity *bt_se) +{ + struct task_struct *p = bt_task_of(bt_se); + struct rq *rq = task_rq(p); + + return &rq->bt; +} + +static inline struct sched_entity *parent_bt_entity(struct sched_entity *bt) +{ + return NULL; +} + +/************************************************************** + * Scheduling class tree data structure manipulation methods: + */ + +static inline int bt_entity_before(struct sched_entity *a, + struct sched_entity *b) +{ + return (s64)(a->vruntime - b->vruntime) < 0; +} + +static void update_bt_min_vruntime(struct bt_rq *bt_rq) +{ + struct sched_entity *curr = bt_rq->curr; + struct rb_node *leftmost = rb_first_cached(&bt_rq->tasks_timeline); + + u64 vruntime = bt_rq->min_vruntime; + + if (curr) { + if (curr->on_rq) + vruntime = bt_rq->curr->vruntime; + else + curr = NULL; + } + + if (leftmost) { + struct sched_entity *bt_se = rb_entry(leftmost, + struct sched_entity, + run_node); + + if (!curr) + vruntime = bt_se->vruntime; + else + vruntime = min_vruntime(vruntime, bt_se->vruntime); + } + + /* ensure we never gain time by being placed backwards. */ + bt_rq->min_vruntime = max_vruntime(bt_rq->min_vruntime, vruntime); +#ifndef CONFIG_64BIT + /* memory barrior for writting */ + smp_wmb(); + bt_rq->min_vruntime_copy = bt_rq->min_vruntime; +#endif +} + +/* + * Enqueue an entity into the rb-tree: + */ +static void __enqueue_bt_entity(struct bt_rq *bt_rq, struct sched_entity *bt_se) +{ + struct rb_node **link = &bt_rq->tasks_timeline.rb_root.rb_node; + struct rb_node *parent = NULL; + struct sched_entity *entry; + int leftmost = 1; + + /* + * Find the right place in the rbtree: + */ + while (*link) { + parent = *link; + entry = rb_entry(parent, struct sched_entity, run_node); + /* + * We dont care about collisions. Nodes with + * the same key stay together. + */ + if (bt_entity_before(bt_se, entry)) { + link = &parent->rb_left; + } else { + link = &parent->rb_right; + leftmost = 0; + } + } + + rb_link_node(&bt_se->run_node, parent, link); + rb_insert_color_cached(&bt_se->run_node, + &bt_rq->tasks_timeline, leftmost); +} + +static void __dequeue_bt_entity(struct bt_rq *bt_rq, struct sched_entity *bt_se) +{ + rb_erase_cached(&bt_se->run_node, &bt_rq->tasks_timeline); +} + +struct sched_entity *__pick_first_bt_entity(struct bt_rq *bt_rq) +{ + struct rb_node *left = rb_first_cached(&bt_rq->tasks_timeline); + + if (!left) + return NULL; + + return rb_entry(left, struct sched_entity, run_node); +} + +static struct sched_entity *__pick_next_bt_entity(struct sched_entity *bt_se) +{ + struct rb_node *next = rb_next(&bt_se->run_node); + + if (!next) + return NULL; + + return rb_entry(next, struct sched_entity, run_node); +} + +#ifdef CONFIG_SCHED_DEBUG +struct sched_entity *__pick_last_bt_entity(struct bt_rq *bt_rq) +{ + struct rb_node *last = rb_last(&bt_rq->tasks_timeline.rb_root); + + if (!last) + return NULL; + + return rb_entry(last, struct sched_entity, run_node); +} +#endif + +/* + * delta /= w + */ +static inline unsigned long +calc_delta_bt(unsigned long delta, struct sched_entity *bt_se) +{ + if (unlikely(bt_se->load.weight != NICE_0_LOAD)) + delta = __calc_delta(delta, NICE_0_LOAD, &bt_se->load); + + return delta; +} + +/* + * We calculate the wall-time slice from the period by taking a part + * proportional to the weight. + * + * s = p*P[w/rw] + */ +static u64 sched_bt_slice(struct bt_rq *bt_rq, struct sched_entity *se) +{ + u64 slice = __sched_period(bt_rq->nr_running + !se->on_rq); + + for_each_sched_bt_entity(se) { + struct load_weight *load; + struct load_weight lw; + + bt_rq = bt_rq_of(se); + load = &bt_rq->load; + + if (unlikely(!se->on_rq)) { + lw = bt_rq->load; + + update_load_add(&lw, se->load.weight); + load = &lw; + } + slice = __calc_delta(slice, se->load.weight, load); + } + return slice; +} + +/* + * We calculate the vruntime slice of a to-be-inserted task. + * + * vs = s/w + */ +static u64 sched_bt_vslice(struct bt_rq *bt_rq, struct sched_entity *se) +{ + return calc_delta_bt(sched_bt_slice(bt_rq, se), se); +} + +/* + * Update the current task's runtime bt_statistics. Skip current tasks that + * are not in our scheduling class. + */ +static inline void +__update_curr_bt(struct bt_rq *bt_rq, struct sched_entity *curr, + unsigned long delta_exec) +{ + unsigned long delta_exec_weighted; + + schedstat_set(curr->bt_statistics->exec_max, + max((u64)delta_exec, curr->bt_statistics->exec_max)); + + curr->sum_exec_runtime += delta_exec; + schedstat_add(bt_rq->exec_clock, delta_exec); + delta_exec_weighted = calc_delta_bt(delta_exec, curr); + + curr->vruntime += delta_exec_weighted; + update_bt_min_vruntime(bt_rq); +} + +static void update_curr_bt(struct bt_rq *bt_rq) +{ + struct sched_entity *curr = bt_rq->curr; + u64 now = rq_of_bt_rq(bt_rq)->clock_task; + unsigned long delta_exec; + + if (unlikely(!curr)) + return; + + /* + * Get the amount of time the current task was running + * since the last time we changed load (this cannot + * overflow on 32 bits): + */ + delta_exec = (unsigned long)(now - curr->exec_start); + if (unlikely((s64)delta_exec <= 0)) + return; + + __update_curr_bt(bt_rq, curr, delta_exec); + curr->exec_start = now; + + if (bt_entity_is_task(curr)) { + struct task_struct *curtask = bt_task_of(curr); + + trace_sched_stat_runtime(curtask, delta_exec, curr->vruntime); + cpuacct_charge(curtask, delta_exec); + } +} + +static inline void +update_stats_wait_start_bt(struct bt_rq *bt_rq, struct sched_entity *bt_se) +{ + schedstat_set(bt_se->bt_statistics->wait_start, + rq_of_bt_rq(bt_rq)->clock); +} + +/* + * Task is being enqueued - update stats: + */ +static void +update_stats_enqueue_bt(struct bt_rq *bt_rq, struct sched_entity *se) +{ + /* + * Are we enqueueing a waiting task? (for current tasks + * a dequeue/enqueue event is a NOP) + */ + if (se != bt_rq->curr) + update_stats_wait_start_bt(bt_rq, se); +} + +static void +update_stats_wait_end_bt(struct bt_rq *bt_rq, struct sched_entity *se) +{ + schedstat_set(se->bt_statistics->wait_max, + max(se->bt_statistics->wait_max, + rq_of_bt_rq(bt_rq)->clock - se->bt_statistics->wait_start)); + schedstat_set(se->bt_statistics->wait_count, + se->bt_statistics->wait_count + 1); + schedstat_set(se->bt_statistics->wait_sum, + se->bt_statistics->wait_sum + + rq_of_bt_rq(bt_rq)->clock - se->bt_statistics->wait_start); +#ifdef CONFIG_SCHEDSTATS + if (bt_entity_is_task(se)) { + trace_sched_stat_wait(bt_task_of(se), + rq_of_bt_rq(bt_rq)->clock - se->bt_statistics->wait_start); + } +#endif + schedstat_set(se->bt_statistics->wait_start, 0); +} + +static inline void +update_stats_dequeue_bt(struct bt_rq *bt_rq, struct sched_entity *se) +{ + /* + * Mark the end of the wait period if dequeueing a + * waiting task: + */ + if (se != bt_rq->curr) + update_stats_wait_end_bt(bt_rq, se); +} + +/* + * We are picking a new current task - update its stats: + */ +static inline void +update_stats_curr_start_bt(struct bt_rq *bt_rq, struct sched_entity *se) +{ + /* + * We are starting a new run period: + */ + se->exec_start = rq_clock_task(rq_of_bt_rq(bt_rq)); +} + +static void +account_bt_entity_enqueue(struct bt_rq *bt_rq, struct sched_entity *se) +{ + update_load_add(&bt_rq->load, se->load.weight); + + bt_rq->nr_running++; +} + +static void +account_bt_entity_dequeue(struct bt_rq *bt_rq, struct sched_entity *se) +{ + update_load_sub(&bt_rq->load, se->load.weight); + + bt_rq->nr_running--; +} + +static void enqueue_bt_sleeper(struct bt_rq *bt_rq, struct sched_entity *se) +{ +#if defined(CONFIG_SCHEDSTATS) || defined(CONFIG_LATENCYTOP) + struct task_struct *tsk = NULL; + + if (bt_entity_is_task(se)) + tsk = bt_task_of(se); + + if (se->bt_statistics->sleep_start) { + u64 delta = rq_of_bt_rq(bt_rq)->clock - se->bt_statistics->sleep_start; + + if ((s64)delta < 0) + delta = 0; + +#ifdef CONFIG_SCHEDSTATS + if (unlikely(delta > se->bt_statistics->sleep_max)) + se->bt_statistics->sleep_max = delta; +#endif + + se->bt_statistics->sleep_start = 0; +#ifdef CONFIG_SCHEDSTATS + se->bt_statistics->sum_sleep_runtime += delta; +#endif + + if (tsk) { + account_scheduler_latency(tsk, delta >> 10, 1); +#ifdef CONFIG_SCHEDSTATS + trace_sched_stat_sleep(tsk, delta); +#endif + } + } + if (se->bt_statistics->block_start) { + u64 delta = rq_of_bt_rq(bt_rq)->clock - se->bt_statistics->block_start; + + if ((s64)delta < 0) + delta = 0; + +#ifdef CONFIG_SCHEDSTATS + if (unlikely(delta > se->bt_statistics->block_max)) + se->bt_statistics->block_max = delta; +#endif + + se->bt_statistics->block_start = 0; +#ifdef CONFIG_SCHEDSTATS + se->bt_statistics->sum_sleep_runtime += delta; +#endif + + if (tsk) { +#ifdef CONFIG_SCHEDSTATS + if (tsk->in_iowait) { + se->bt_statistics->iowait_sum += delta; + se->bt_statistics->iowait_count++; + trace_sched_stat_iowait(tsk, delta); + } +#endif + + trace_sched_stat_blocked(tsk, delta); + + /* + * Blocking time is in units of nanosecs, so shift by + * 20 to get a milliseconds-range estimation of the + * amount of time that the task spent sleeping: + */ + if (unlikely(prof_on == SLEEP_PROFILING)) { + profile_hits(SLEEP_PROFILING, + (void *)get_wchan(tsk), + delta >> 20); + } + account_scheduler_latency(tsk, delta >> 10, 0); + } + } +#endif +} + +static void check_bt_spread(struct bt_rq *bt_rq, struct sched_entity *se) +{ +#ifdef CONFIG_SCHED_DEBUG + s64 d = se->vruntime - bt_rq->min_vruntime; + + if (d < 0) + d = -d; + + if (d > 3*sysctl_sched_latency) + schedstat_inc(bt_rq->nr_spread_over); +#endif +} + +static void +place_bt_entity(struct bt_rq *bt_rq, struct sched_entity *se, int initial) +{ + u64 vruntime = bt_rq->min_vruntime; + + /* + * The 'current' period is already promised to the current tasks, + * however the extra weight of the new task will slow them down a + * little, place the new task so that it fits in the slot that + * stays open at the end. + */ + if (initial && sched_feat(START_DEBIT)) + vruntime += sched_bt_vslice(bt_rq, se); + + /* sleeps up to a single latency don't count. */ + if (!initial) { + unsigned long thresh = sysctl_sched_latency; + + /* + * Halve their sleep time's effect, to allow + * for a gentler effect of sleepers: + */ + if (sched_feat(GENTLE_FAIR_SLEEPERS)) + thresh >>= 1; + + vruntime -= thresh; + } + + /* ensure we never gain time by being placed backwards. */ + se->vruntime = max_vruntime(se->vruntime, vruntime); +} + +static void +enqueue_bt_entity(struct bt_rq *bt_rq, struct sched_entity *se, int flags) +{ + /* + * Update the normalized vruntime before updating min_vruntime + * through callig update_curr(). + */ + if (!(flags & ENQUEUE_WAKEUP) || (flags & ENQUEUE_MIGRATED)) + se->vruntime += bt_rq->min_vruntime; + + /* + * Update run-time bt_statistics of the 'current'. + */ + update_curr_bt(bt_rq); + account_bt_entity_enqueue(bt_rq, se); + + if (flags & ENQUEUE_WAKEUP) { + place_bt_entity(bt_rq, se, 0); + enqueue_bt_sleeper(bt_rq, se); + } + + update_stats_enqueue_bt(bt_rq, se); + check_bt_spread(bt_rq, se); + if (se != bt_rq->curr) + __enqueue_bt_entity(bt_rq, se); + se->on_rq = 1; +} + +static void __clear_buddies_last_bt(struct sched_entity *se) +{ + for_each_sched_bt_entity(se) { + struct bt_rq *bt_rq = bt_rq_of(se); + + if (bt_rq->last == se) + bt_rq->last = NULL; + else + break; + } +} + +static void __clear_buddies_next_bt(struct sched_entity *se) +{ + for_each_sched_bt_entity(se) { + struct bt_rq *bt_rq = bt_rq_of(se); + + if (bt_rq->next == se) + bt_rq->next = NULL; + else + break; + } +} + +static void __clear_buddies_skip_bt(struct sched_entity *se) +{ + for_each_sched_bt_entity(se) { + struct bt_rq *bt_rq = bt_rq_of(se); + + if (bt_rq->skip == se) + bt_rq->skip = NULL; + else + break; + } +} + +static void clear_buddies_bt(struct bt_rq *bt_rq, struct sched_entity *se) +{ + if (bt_rq->last == se) + __clear_buddies_last_bt(se); + + if (bt_rq->next == se) + __clear_buddies_next_bt(se); + + if (bt_rq->skip == se) + __clear_buddies_skip_bt(se); +} + +static void +dequeue_bt_entity(struct bt_rq *bt_rq, struct sched_entity *se, int flags) +{ + /* + * Update run-time bt_statistics of the 'current'. + */ + update_curr_bt(bt_rq); + + update_stats_dequeue_bt(bt_rq, se); + if (flags & DEQUEUE_SLEEP) { +#if defined(CONFIG_SCHEDSTATS) || defined(CONFIG_LATENCYTOP) + if (bt_entity_is_task(se)) { + struct task_struct *tsk = bt_task_of(se); + + if (tsk->state & TASK_INTERRUPTIBLE) + se->bt_statistics->sleep_start = rq_of_bt_rq(bt_rq)->clock; + if (tsk->state & TASK_UNINTERRUPTIBLE) + se->bt_statistics->block_start = rq_of_bt_rq(bt_rq)->clock; + } +#endif + } + + clear_buddies_bt(bt_rq, se); + + if (se != bt_rq->curr) + __dequeue_bt_entity(bt_rq, se); + se->on_rq = 0; + account_bt_entity_dequeue(bt_rq, se); + + /* + * Normalize the entity after updating the min_vruntime because the + * update can refer to the ->curr item and we need to reflect this + * movement in our normalized position. + */ + if (!(flags & DEQUEUE_SLEEP)) + se->vruntime -= bt_rq->min_vruntime; + + update_bt_min_vruntime(bt_rq); +} + +/* + * Preempt the current task with a newly woken task if needed: + */ +static void +check_preempt_tick_bt(struct bt_rq *bt_rq, struct sched_entity *curr) +{ + unsigned long ideal_runtime, delta_exec; + struct sched_entity *se; + s64 delta; + + ideal_runtime = sched_bt_slice(bt_rq, curr); + delta_exec = curr->sum_exec_runtime - curr->prev_sum_exec_runtime; + if (delta_exec > ideal_runtime) { + resched_curr(rq_of_bt_rq(bt_rq)); + /* + * The current task ran long enough, ensure it doesn't get + * re-elected due to buddy favours. + */ + clear_buddies_bt(bt_rq, curr); + return; + } + + /* + * Ensure that a task that missed wakeup preemption by a + * narrow margin doesn't have to wait for a full slice. + * This also mitigates buddy induced latencies under load. + */ + if (delta_exec < sysctl_sched_min_granularity) + return; + + se = __pick_first_bt_entity(bt_rq); + delta = curr->vruntime - se->vruntime; + + if (delta < 0) + return; + + if (delta > ideal_runtime) + resched_curr(rq_of_bt_rq(bt_rq)); +} + +static void +set_next_bt_entity(struct bt_rq *bt_rq, struct sched_entity *se) +{ + /* 'current' is not kept within the tree. */ + if (se->on_rq) { + /* + * Any task has to be enqueued before it get to execute on + * a CPU. So account for the time it spent waiting on the + * runqueue. + */ + update_stats_wait_end_bt(bt_rq, se); + __dequeue_bt_entity(bt_rq, se); + } + + update_stats_curr_start_bt(bt_rq, se); + bt_rq->curr = se; +#ifdef CONFIG_SCHEDSTATS + /* + * Track our maximum slice length, if the CPU's load is at + * least twice that of our own weight (i.e. dont track it + * when there are only lesser-weight tasks around): + */ + if (bt_rq->load.weight >= 2*se->load.weight) { + se->bt_statistics->slice_max = max(se->bt_statistics->slice_max, + se->sum_exec_runtime - se->prev_sum_exec_runtime); + } +#endif + se->prev_sum_exec_runtime = se->sum_exec_runtime; +} + +static int +wakeup_preempt_bt_entity(struct sched_entity *curr, struct sched_entity *se); + +/* + * Pick the next process, keeping these things in mind, in this order: + * 1) keep things fair between processes/task groups + * 2) pick the "next" process, since someone really wants that to run + * 3) pick the "last" process, for cache locality + * 4) do not run the "skip" process, if something else is available + */ +static struct sched_entity *pick_next_bt_entity(struct bt_rq *bt_rq) +{ + struct sched_entity *se = __pick_first_bt_entity(bt_rq); + struct sched_entity *left = se; + + /* + * Avoid running the skip buddy, if running something else can + * be done without getting too unfair. + */ + if (bt_rq->skip == se) { + struct sched_entity *second = __pick_next_bt_entity(se); + + if (second && wakeup_preempt_bt_entity(second, left) < 1) + se = second; + } + + /* + * Prefer last buddy, try to return the CPU to a preempted task. + */ + if (bt_rq->last && wakeup_preempt_bt_entity(bt_rq->last, left) < 1) + se = bt_rq->last; + + /* + * Someone really wants this to run. If it's not unfair, run it. + */ + if (bt_rq->next && wakeup_preempt_bt_entity(bt_rq->next, left) < 1) + se = bt_rq->next; + + clear_buddies_bt(bt_rq, se); + + return se; +} + +static void put_prev_bt_entity(struct bt_rq *bt_rq, struct sched_entity *prev) +{ + /* + * If still on the runqueue then deactivate_task() + * was not called and update_curr() has to be done: + */ + if (prev->on_rq) + update_curr_bt(bt_rq); + + check_bt_spread(bt_rq, prev); + if (prev->on_rq) { + update_stats_wait_start_bt(bt_rq, prev); + /* Put 'current' back into the tree. */ + __enqueue_bt_entity(bt_rq, prev); + } + bt_rq->curr = NULL; +} + +static void +bt_entity_tick(struct bt_rq *bt_rq, struct sched_entity *curr, int queued) +{ + /* + * Update run-time bt_statistics of the 'current'. + */ + update_curr_bt(bt_rq); + +#ifdef CONFIG_SCHED_HRTICK + /* + * queued ticks are scheduled to match the slice, so don't bother + * validating it and just reschedule. + */ + if (queued) { + resched_curr(rq_of_bt_rq(bt_rq)); + return; + } +#endif + + if (bt_rq->nr_running > 1) + check_preempt_tick_bt(bt_rq, curr); +} + +/* + * The enqueue_task method is called before nr_running is + * increased. Here we update the fair scheduling stats and + * then put the task into the rbtree: + */ +static void +enqueue_task_bt(struct rq *rq, struct task_struct *p, int flags) +{ + struct bt_rq *bt_rq; + struct sched_entity *se = &p->bt; + + for_each_sched_bt_entity(se) { + if (se->on_rq) + break; + bt_rq = bt_rq_of(se); + enqueue_bt_entity(bt_rq, se, flags); + + bt_rq->h_nr_running++; + + flags = ENQUEUE_WAKEUP; + } + + if (!se) { + rq->bt_nr_running++; + add_nr_running(rq, 1); + } +} + +static void set_next_buddy_bt(struct sched_entity *se); + +/* + * The dequeue_task method is called before nr_running is + * decreased. We remove the task from the rbtree and + * update the fair scheduling stats: + */ +static void dequeue_task_bt(struct rq *rq, struct task_struct *p, int flags) +{ + struct bt_rq *bt_rq; + struct sched_entity *se = &p->bt; + int task_sleep = flags & DEQUEUE_SLEEP; + + for_each_sched_bt_entity(se) { + bt_rq = bt_rq_of(se); + dequeue_bt_entity(bt_rq, se, flags); + + bt_rq->h_nr_running--; + + /* Don't dequeue parent if it has other entities besides us */ + if (bt_rq->load.weight) { + /* + * Bias pick_next to pick a task from this cfs_rq, as + * p is sleeping when it is within its sched_slice. + */ + if (task_sleep && parent_bt_entity(se)) + set_next_buddy_bt(parent_bt_entity(se)); + + /* avoid re-evaluating load for this entity */ + se = parent_bt_entity(se); + break; + } + flags |= DEQUEUE_SLEEP; + } + + if (!se) { + sub_nr_running(rq, 1); + rq->bt_nr_running--; + } +} + +#ifdef CONFIG_SMP +/** + * idle_cpu - is a given cpu idle currently? + * @cpu: the processor in question. + */ +static int idle_cpu_bt(int cpu) +{ + struct rq *rq = cpu_rq(cpu); + + if (rq->curr != rq->idle) + return 0; + + if (rq->nr_running) + return 0; + +#ifdef CONFIG_SMP + if (!llist_empty(&rq->wake_list)) + return 0; +#endif + + return 1; +} + +static int select_idle_sibling_bt(struct task_struct *p, int target) +{ + struct sched_domain *sd; + struct sched_group *sg; + int i = task_cpu(p); + + if (idle_cpu_bt(target)) + return target; + + /* + * If the prevous cpu is cache affine and idle, don't be stupid. + */ + if (i != target && cpus_share_cache(i, target) && idle_cpu_bt(i)) + return i; + + /* + * Otherwise, iterate the domains and find an elegible idle cpu. + */ + sd = rcu_dereference(per_cpu(sd_llc, target)); + for_each_lower_domain(sd) { + sg = sd->groups; + do { + if (!cpumask_intersects(sched_group_cpus(sg), + tsk_cpus_allowed(p))) + goto next; + + for_each_cpu(i, sched_group_cpus(sg)) { + if (i == target || !idle_cpu_bt(i)) + goto next; + } + + target = cpumask_first_and(sched_group_cpus(sg), + tsk_cpus_allowed(p)); + goto done; +next: + sg = sg->next; + } while (sg != sd->groups); + } +done: + return target; +} + +static int select_idle_cpu(struct task_struct *p, int target) +{ + struct sched_domain *sd; + struct sched_group *sg; + int i = task_cpu(p); + + if (idle_cpu_bt(target)) + return target; + + /* + * If the prevous cpu is cache affine and idle, don't be stupid. + */ + if (i != target && cpus_share_cache(i, target) && idle_cpu_bt(i)) + return i; + + /* + * Otherwise, iterate the domains and find an elegible idle cpu. + */ + sd = rcu_dereference(per_cpu(sd_llc, target)); + for_each_lower_domain(sd) { + sg = sd->groups; + do { + if (!cpumask_intersects(sched_group_cpus(sg), + tsk_cpus_allowed(p))) + goto next; + + for_each_cpu(i, sched_group_cpus(sg)) { + if (idle_cpu_bt(i) && cpumask_test_cpu(i, tsk_cpus_allowed(p))) { + target = i; + goto done; + } + } +next: + sg = sg->next; + } while (sg != sd->groups); + } + +done: + return target; +} + + +static int +select_task_rq_bt(struct task_struct *p, int prev_cpu, int sd_flag, int wake_flags) +{ + struct sched_domain *tmp, *affine_sd = NULL, *sd = NULL; + int cpu = smp_processor_id(); + int new_cpu = prev_cpu; + int want_affine = 0; + + if (p->nr_cpus_allowed == 1) + return prev_cpu; + + if (sd_flag & SD_BALANCE_WAKE) { + if (cpumask_test_cpu(cpu, tsk_cpus_allowed(p))) + want_affine = 1; + new_cpu = prev_cpu; + } + + rcu_read_lock(); + for_each_domain(cpu, tmp) { + if (!(tmp->flags & SD_LOAD_BALANCE)) + continue; + + /* + * If both cpu and prev_cpu are part of this domain, + * cpu is a valid SD_WAKE_AFFINE target. + */ + if (want_affine && (tmp->flags & SD_WAKE_AFFINE) && + cpumask_test_cpu(prev_cpu, sched_domain_span(tmp))) { + affine_sd = tmp; + break; + } + + if (tmp->flags & sd_flag) + sd = tmp; + } + + if (affine_sd) { + new_cpu = select_idle_sibling_bt(p, prev_cpu); + goto unlock; + } + + new_cpu = select_idle_cpu(p, prev_cpu); + +unlock: + rcu_read_unlock(); + + return new_cpu; +} +#endif + +static unsigned long +wakeup_gran_bt(struct sched_entity *curr, struct sched_entity *se) +{ + unsigned long gran = sysctl_sched_wakeup_granularity; + + /* + * Since its curr running now, convert the gran from real-time + * to virtual-time in his units. + * + * By using 'se' instead of 'curr' we penalize light tasks, so + * they get preempted easier. That is, if 'se' < 'curr' then + * the resulting gran will be larger, therefore penalizing the + * lighter, if otoh 'se' > 'curr' then the resulting gran will + * be smaller, again penalizing the lighter task. + * + * This is especially important for buddies when the leftmost + * task is higher priority than the buddy. + */ + return calc_delta_bt(gran, se); +} + +/* + * Should 'se' preempt 'curr'. + * + * |s1 + * |s2 + * |s3 + * g + * |<--->|c + * + * w(c, s1) = -1 + * w(c, s2) = 0 + * w(c, s3) = 1 + * + */ +static int +wakeup_preempt_bt_entity(struct sched_entity *curr, struct sched_entity *se) +{ + s64 gran, vdiff = curr->vruntime - se->vruntime; + + if (vdiff <= 0) + return -1; + + gran = wakeup_gran_bt(curr, se); + if (vdiff > gran) + return 1; + + return 0; +} + +static void set_last_buddy_bt(struct sched_entity *se) +{ + if (bt_entity_is_task(se)) + return; + + for_each_sched_bt_entity(se) + bt_rq_of(se)->last = se; +} + +static void set_next_buddy_bt(struct sched_entity *se) +{ + if (bt_entity_is_task(se)) + return; + + for_each_sched_bt_entity(se) + bt_rq_of(se)->next = se; +} + +static void set_skip_buddy_bt(struct sched_entity *se) +{ + for_each_sched_bt_entity(se) + bt_rq_of(se)->skip = se; +} + +/* + * Preempt the current task with a newly woken task if needed: + */ +static void check_preempt_wakeup_bt(struct rq *rq, struct task_struct *p, int wake_flags) +{ + struct task_struct *curr = rq->curr; + struct sched_entity *se = &curr->bt, *pse = &p->bt; + struct bt_rq *bt_rq = task_bt_rq(curr); + int scale = bt_rq->nr_running >= sched_nr_latency; + int next_buddy_marked = 0; + + if (unlikely(se == pse)) + return; + + if (sched_feat(NEXT_BUDDY) && scale && !(wake_flags & WF_FORK)) { + set_next_buddy_bt(pse); + next_buddy_marked = 1; + } + + /* + * We can come here with TIF_NEED_RESCHED already set from new task + * wake up path. + * + * Note: this also catches the edge-case of curr being in a throttled + * group (e.g. via set_curr_task), since update_curr() (in the + * enqueue of curr) will have resulted in resched being set. This + * prevents us from potentially nominating it as a false LAST_BUDDY + * below. + */ + if (test_tsk_need_resched(curr)) + return; + + /* BT tasks are by definition preempted by non-bt tasks. */ + if (likely(p->policy < SCHED_BT)) + goto preempt; + + if (!sched_feat(WAKEUP_PREEMPTION)) + return; + + update_curr_bt(bt_rq_of(se)); + BUG_ON(!pse); + if (wakeup_preempt_bt_entity(se, pse) == 1) { + /* + * Bias pick_next to pick the sched entity that is + * triggering this preemption. + */ + if (!next_buddy_marked) + set_next_buddy_bt(pse); + goto preempt; + } + + return; + +preempt: + resched_curr(rq); + /* + * Only set the backward buddy when the current task is still + * on the rq. This can happen when a wakeup gets interleaved + * with schedule on the ->pre_schedule() or idle_balance() + * point, either of which can * drop the rq lock. + * + * Also, during early boot the idle thread is in the fair class, + * for obvious reasons its a bad idea to schedule back to it. + */ + if (unlikely(!se->on_rq || curr == rq->idle)) + return; + + if (sched_feat(LAST_BUDDY) && scale && bt_entity_is_task(se)) + set_last_buddy_bt(se); +} + +static struct task_struct *pick_next_task_bt(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) +{ + struct task_struct *p; + struct bt_rq *bt_rq; + struct sched_entity *se; + int new_tasks = -1; + + bt_rq = &rq->bt; +again: + if (!bt_rq->nr_running) + goto idle; + + put_prev_task(rq, prev); + + se = pick_next_bt_entity(bt_rq); + set_next_bt_entity(bt_rq, se); + + p = bt_task_of(se); + + return p; + +idle: + new_tasks = idle_balance(rq, rf); + + /* + * Because idle_balance() releases (and re-acquires) rq->lock, it is + * possible for any higher priority task to appear. In that case we + * must re-start the pick_next_entity() loop. + */ + if (new_tasks < 0) + return RETRY_TASK; + + if (new_tasks > 0) + goto again; + + return NULL; +} + +/* + * Account for a descheduled task: + */ +static void put_prev_task_bt(struct rq *rq, struct task_struct *prev) +{ + struct sched_entity *se = &prev->bt; + struct bt_rq *bt_rq; + + for_each_sched_bt_entity(se) { + bt_rq = bt_rq_of(se); + put_prev_bt_entity(bt_rq, se); + } +} + +/* + * sched_yield() is very simple + * + * The magic of dealing with the ->skip buddy is in pick_next_entity. + */ +static void yield_task_bt(struct rq *rq) +{ + struct task_struct *curr = rq->curr; + struct bt_rq *bt_rq = task_bt_rq(curr); + struct sched_entity *se = &curr->bt; + + /* + * Are we the only task in the tree? + */ + if (unlikely(rq->bt_nr_running == 1)) + return; + + clear_buddies_bt(bt_rq, se); + + update_rq_clock(rq); + /* + * Update run-time bt_statistics of the 'current'. + */ + update_curr_bt(bt_rq); + /* + * Tell update_rq_clock() that we've just updated, + * so we don't do microscopic update in schedule() + * and double the fastpath cost. + */ +// rq->skip_clock_update = 1; + + set_skip_buddy_bt(se); +} + +static bool yield_to_task_bt(struct rq *rq, struct task_struct *p, bool preempt) +{ + struct sched_entity *se = &p->bt; + + if (!se->on_rq) + return false; + + /* Tell the scheduler that we'd really like pse to run next. */ + set_next_buddy_bt(se); + + yield_task_bt(rq); + + return true; +} + +#ifdef CONFIG_SMP +static void rq_online_bt(struct rq *rq) +{ + update_sysctl(); +} + +static void rq_offline_bt(struct rq *rq) +{ + update_sysctl(); +} + +#endif /* CONFIG_SMP */ + +/* + * scheduler tick hitting a task of our scheduling class: + */ +static void task_tick_bt(struct rq *rq, struct task_struct *curr, int queued) +{ + struct bt_rq *bt_rq; + struct sched_entity *se = &curr->bt; + + for_each_sched_bt_entity(se) { + bt_rq = bt_rq_of(se); + bt_entity_tick(bt_rq, se, queued); + } + + if (static_branch_unlikely(&sched_numa_balancing)) + task_tick_numa(rq, curr); +} + +/* + * called on fork with the child task as argument from the parent's context + * - child not yet on the tasklist + * - preemption disabled + */ +static void task_fork_bt(struct task_struct *p) +{ + struct bt_rq *bt_rq; + struct sched_entity *se = &p->bt, *curr; + int this_cpu = smp_processor_id(); + struct rq *rq = this_rq(); + unsigned long flags; + + raw_spin_lock_irqsave(&rq->lock, flags); + + update_rq_clock(rq); + + bt_rq = task_bt_rq(current); + curr = bt_rq->curr; + + /* + * Not only the cpu but also the task_group of the parent might have + * been changed after parent->se.parent,cfs_rq were copied to + * child->se.parent,cfs_rq. So call __set_task_cpu() to make those + * of child point to valid ones. + */ + rcu_read_lock(); + __set_task_cpu(p, this_cpu); + rcu_read_unlock(); + + update_curr_bt(bt_rq); + + if (curr) + se->vruntime = curr->vruntime; + place_bt_entity(bt_rq, se, 1); + + if (sysctl_sched_child_runs_first && curr && bt_entity_before(curr, se)) { + /* + * Upon rescheduling, sched_class::put_prev_task() will place + * 'current' within the tree based on its new key value. + */ + swap(curr->vruntime, se->vruntime); + resched_curr(rq); + } + + se->vruntime -= bt_rq->min_vruntime; + + raw_spin_unlock_irqrestore(&rq->lock, flags); +} + +/* + * Priority of the task has changed. Check to see if we preempt + * the current task. + */ +static void +prio_changed_bt(struct rq *rq, struct task_struct *p, int oldprio) +{ + if (!p->bt.on_rq) + return; + + /* + * Reschedule if we are currently running on this runqueue and + * our priority decreased, or if we are not currently running on + * this runqueue and our priority is higher than the current's + */ + if (rq->curr == p) { + if (p->prio > oldprio) + resched_curr(rq); + } else + check_preempt_curr(rq, p, 0); +} + +static void switched_from_bt(struct rq *rq, struct task_struct *p) +{ + struct sched_entity *se = &p->bt; + struct bt_rq *bt_rq = bt_rq_of(se); + + /* + * Ensure the task's vruntime is normalized, so that when it's + * switched back to the fair class the enqueue_entity(.flags=0) will + * do the right thing. + * + * If it's on_rq, then the dequeue_entity(.flags=0) will already + * have normalized the vruntime, if it's !on_rq, then only when + * the task is sleeping will it still have non-normalized vruntime. + */ + if (!p->on_rq && p->state != TASK_RUNNING) { + /* + * Fix up our vruntime so that the current sleep doesn't + * cause 'unlimited' sleep bonus. + */ + place_bt_entity(bt_rq, se, 0); + se->vruntime -= bt_rq->min_vruntime; + } +} + +/* + * We switched to the sched_fair class. + */ +static void switched_to_bt(struct rq *rq, struct task_struct *p) +{ + BUG_ON(!bt_prio(p->static_prio)); + +// attach_task_bt_rq(p); + + if (!p->bt.on_rq) + return; + + /* + * We were most likely switched from sched_rt, so + * kick off the schedule if running, otherwise just see + * if we can still preempt the current task. + */ + if (rq->curr == p) + resched_curr(rq); + else + check_preempt_curr(rq, p, 0); +} + +/* Account for a task changing its policy or group. + * + * This routine is mostly called to set cfs_rq->curr field when a task + * migrates between groups/classes. + */ +static void set_curr_task_bt(struct rq *rq) +{ + struct sched_entity *se = &rq->curr->bt; + + for_each_sched_bt_entity(se) { + struct bt_rq *bt_rq = bt_rq_of(se); + + set_next_bt_entity(bt_rq, se); + } +} + +void init_bt_rq(struct bt_rq *bt_rq) +{ + bt_rq->tasks_timeline.rb_root = RB_ROOT; + bt_rq->tasks_timeline.rb_leftmost = NULL; + bt_rq->min_vruntime = (u64)(-(1LL << 20)); +#ifndef CONFIG_64BIT + bt_rq->min_vruntime_copy = bt_rq->min_vruntime; +#endif +} + +static unsigned int get_rr_interval_bt(struct rq *rq, struct task_struct *task) +{ + struct sched_entity *se = &task->bt; + unsigned int rr_interval = 0; + + /* + * Time slice is 0 for SCHED_OTHER tasks that are on an otherwise + * idle runqueue: + */ + if (rq->bt.load.weight) + rr_interval = NS_TO_JIFFIES(sched_bt_slice(bt_rq_of(se), se)); + + return rr_interval; +} + +/* + * All the scheduling class methods: + */ +const struct sched_class bt_sched_class = { + .next = &idle_sched_class, + .enqueue_task = enqueue_task_bt, + .dequeue_task = dequeue_task_bt, + .yield_task = yield_task_bt, + .yield_to_task = yield_to_task_bt, + + .check_preempt_curr = check_preempt_wakeup_bt, + + .pick_next_task = pick_next_task_bt, + .put_prev_task = put_prev_task_bt, + +#ifdef CONFIG_SMP + .select_task_rq = select_task_rq_bt, + .rq_online = rq_online_bt, + .rq_offline = rq_offline_bt, +#endif + + .set_curr_task = set_curr_task_bt, + .task_tick = task_tick_bt, + .task_fork = task_fork_bt, + + .prio_changed = prio_changed_bt, + .switched_from = switched_from_bt, + .switched_to = switched_to_bt, + + .get_rr_interval = get_rr_interval_bt, +}; diff --git a/kernel/sched/batch.h b/kernel/sched/batch.h new file mode 100644 index 000000000..7bc4ce267 --- /dev/null +++ b/kernel/sched/batch.h @@ -0,0 +1,83 @@ +/* + * Copyright (C) 2019 Tencent Ltd. All rights reserved. + * + * File Name ：batch.h + * Author ： + * Date ：2019-12-26 + * Descriptor： + */ + +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _BATCH_H +#define _BATCH_H + +#include +#include + +/* nflag of task_struct */ +#define TNF_SCHED_BT 0x00000001 + + +struct bt_rq { + struct load_weight load; + unsigned int nr_running, h_nr_running; + unsigned long nr_uninterruptible; + + u64 exec_clock; + u64 min_vruntime; +#ifndef CONFIG_64BIT + u64 min_vruntime_copy; +#endif + + struct rb_root_cached tasks_timeline; + struct rb_node *rb_leftmost; + + /* + * 'curr' points to currently running entity on this bt_rq. + * It is set to NULL otherwise (i.e when none are currently running). + */ + struct sched_entity *curr, *next, *last, *skip; + +#ifdef CONFIG_SCHED_DEBUG + unsigned int nr_spread_over; +#endif + +#ifdef CONFIG_SMP +/* + * Load-tracking only depends on SMP, BT_GROUP_SCHED dependency below may be + * removed when useful for applications beyond shares distribution (e.g. + * load-balance). + */ + + + /* + * h_load = weight * f(tg) + * + * Where f(tg) is the recursive weight fraction assigned to + * this group. + */ + unsigned long h_load; +#endif /* CONFIG_SMP */ +}; + +extern const struct sched_class bt_sched_class; + +extern void init_bt_rq(struct bt_rq *bt_rq); +extern struct sched_entity *__pick_first_bt_entity(struct bt_rq *bt_rq); +extern struct sched_entity *__pick_last_bt_entity(struct bt_rq *bt_rq); +extern void set_bt_load_weight(struct task_struct *p); + + +static inline int bt_policy(int policy) +{ + if (policy == SCHED_BT) + return 1; + return 0; +} + +static inline int task_has_bt_policy(struct task_struct *p) +{ + return bt_policy(p->policy); +} + +#endif diff --git a/kernel/sched/bt_debug.c b/kernel/sched/bt_debug.c new file mode 100644 index 000000000..0bb5f240d --- /dev/null +++ b/kernel/sched/bt_debug.c @@ -0,0 +1,72 @@ +/* + * Copyright (C) 2019 Tencent Ltd. All rights reserved. + * + * File Name : bt_debug.c + * Author : + * Date : 2019-12-26 + * Descriptor: + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "sched.h" +#include "bt_debug.h" + +void print_bt_rq(struct seq_file *m, int cpu, struct bt_rq *bt_rq) +{ + s64 MIN_vruntime = -1, min_vruntime, max_vruntime = -1, + spread, rq0_min_vruntime, spread0; + struct rq *rq = cpu_rq(cpu); + struct sched_entity *last; + unsigned long flags; + + SEQ_printf(m, "\nbt_rq[%d]:\n", cpu); + + SEQ_printf(m, " .%-30s: %lld.%06ld\n", "exec_clock", + SPLIT_NS(bt_rq->exec_clock)); + + raw_spin_lock_irqsave(&rq->lock, flags); + if (bt_rq->rb_leftmost) + MIN_vruntime = (__pick_first_bt_entity(bt_rq))->vruntime; + last = __pick_last_bt_entity(bt_rq); + if (last) + max_vruntime = last->vruntime; + min_vruntime = bt_rq->min_vruntime; + rq0_min_vruntime = cpu_rq(0)->bt.min_vruntime; + raw_spin_unlock_irqrestore(&rq->lock, flags); + SEQ_printf(m, " .%-30s: %lld.%06ld\n", "MIN_vruntime", + SPLIT_NS(MIN_vruntime)); + SEQ_printf(m, " .%-30s: %lld.%06ld\n", "min_vruntime", + SPLIT_NS(min_vruntime)); + SEQ_printf(m, " .%-30s: %lld.%06ld\n", "max_vruntime", + SPLIT_NS(max_vruntime)); + spread = max_vruntime - MIN_vruntime; + SEQ_printf(m, " .%-30s: %lld.%06ld\n", "spread", + SPLIT_NS(spread)); + spread0 = min_vruntime - rq0_min_vruntime; + SEQ_printf(m, " .%-30s: %lld.%06ld\n", "spread0", + SPLIT_NS(spread0)); + SEQ_printf(m, " .%-30s: %d\n", "nr_spread_over", + bt_rq->nr_spread_over); + SEQ_printf(m, " .%-30s: %d\n", "nr_running", bt_rq->nr_running); + SEQ_printf(m, " .%-30s: %ld\n", "load", bt_rq->load.weight); +} + +#ifdef CONFIG_SCHED_DEBUG +void print_bt_stats(struct seq_file *m, int cpu) +{ + struct bt_rq *bt_rq; + + rcu_read_lock(); + bt_rq = &cpu_rq(cpu)->bt; + print_bt_rq(m, cpu, bt_rq); + rcu_read_unlock(); +} +#endif diff --git a/kernel/sched/bt_debug.h b/kernel/sched/bt_debug.h new file mode 100644 index 000000000..869f80fdc --- /dev/null +++ b/kernel/sched/bt_debug.h @@ -0,0 +1,36 @@ +/* + * Copyright (C) 2019 Tencent Ltd. All rights reserved. + * + * File Name : bt_debug.h + * Author : + * Date : 2019-12-26 + * Descriptor: + */ + +#ifndef _BT_STAT_H +#define _BT_STAT_H + +/* + * This allows printing both to /proc/sched_debug and + * to the console + */ +#define SEQ_printf(m, x...) \ +do { \ + if (m) \ + seq_printf(m, x); \ + else \ + printk(x); \ +} while (0) + +extern long long nsec_high(unsigned long long nsec); +extern unsigned long nsec_low(unsigned long long nsec); +#define SPLIT_NS(x) (nsec_high(x), nsec_low(x)) + +extern void print_bt_stats(struct seq_file *m, int cpu); +#ifdef CONFIG_SCHED_DEBUG +extern void print_bt_stats(struct seq_file *m, int cpu); +#else +void print_bt_stats(struct seq_file *m, int cpu) {} +#endif + +#endif diff --git a/kernel/sched/core.c b/kernel/sched/core.c index ee2032d96..5ac14598b 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -34,6 +35,7 @@ #endif #include "sched.h" +#include "batch.h" #include "../workqueue_internal.h" #include "../smpboot.h" @@ -775,16 +777,38 @@ static inline void dequeue_task(struct rq *rq, struct task_struct *p, int flags) void activate_task(struct rq *rq, struct task_struct *p, int flags) { +#ifdef CONFIG_BT_SCHED + if (task_contributes_to_load(p)) { + rq->nr_uninterruptible--; + + if (unlikely(p->flags & TNF_SCHED_BT)) { + p->flags &= ~TNF_SCHED_BT; + rq->bt.nr_uninterruptible--; + } + } +#else if (task_contributes_to_load(p)) rq->nr_uninterruptible--; +#endif enqueue_task(rq, p, flags); } void deactivate_task(struct rq *rq, struct task_struct *p, int flags) { +#ifdef CONFIG_BT_SCHED + if (task_contributes_to_load(p)) { + rq->nr_uninterruptible++; + + if (unlikely(p->sched_class == &bt_sched_class)) { + p->flags |= TNF_SCHED_BT; + rq->bt.nr_uninterruptible++; + } + } +#else if (task_contributes_to_load(p)) rq->nr_uninterruptible++; +#endif dequeue_task(rq, p, flags); } @@ -1718,8 +1742,19 @@ ttwu_do_activate(struct rq *rq, struct task_struct *p, int wake_flags, lockdep_assert_held(&rq->lock); #ifdef CONFIG_SMP +#ifdef CONFIG_BT_SCHED + if (p->sched_contributes_to_load) { + rq->nr_uninterruptible--; + + if (unlikely(p->flags & TNF_SCHED_BT)) { + p->flags &= ~TNF_SCHED_BT; + rq->bt.nr_uninterruptible--; + } + } +#else if (p->sched_contributes_to_load) rq->nr_uninterruptible--; +#endif if (wake_flags & WF_MIGRATED) en_flags |= ENQUEUE_MIGRATED; @@ -2183,6 +2218,15 @@ static void __sched_fork(unsigned long clone_flags, struct task_struct *p) p->se.vruntime = 0; INIT_LIST_HEAD(&p->se.group_node); +#ifdef CONFIG_BT_SCHED + p->bt.on_rq = 0; + p->bt.exec_start = 0; + p->bt.sum_exec_runtime = 0; + p->bt.prev_sum_exec_runtime = 0; + p->bt.nr_migrations = 0; + p->bt.vruntime = 0; +#endif + #ifdef CONFIG_FAIR_GROUP_SCHED p->se.cfs_rq = NULL; #endif @@ -2190,6 +2234,9 @@ static void __sched_fork(unsigned long clone_flags, struct task_struct *p) #ifdef CONFIG_SCHEDSTATS /* Even if schedstat is disabled, there should not be garbage */ memset(&p->se.statistics, 0, sizeof(p->se.statistics)); +#ifdef CONFIG_BT_SCHED + p->bt.bt_statistics = &p->se.statistics; +#endif #endif RB_CLEAR_NODE(&p->dl.rb_node); @@ -2389,6 +2436,10 @@ int sched_fork(unsigned long clone_flags, struct task_struct *p) return -EAGAIN; } else if (rt_prio(p->prio)) { p->sched_class = &rt_sched_class; +#ifdef CONFIG_BT_SCHED + } else if(bt_prio(p->prio)){ + p->sched_class = &bt_sched_class; +#endif } else { p->sched_class = &fair_sched_class; } @@ -3003,7 +3054,7 @@ unsigned long long task_sched_runtime(struct task_struct *p) * been accounted, so we're correct here as well. */ if (!p->on_cpu || !task_on_rq_queued(p)) - return p->se.sum_exec_runtime; + return TASK_SUM_EXEC_RUNTIME(p); #endif rq = task_rq_lock(p, &rf); @@ -3017,7 +3068,7 @@ unsigned long long task_sched_runtime(struct task_struct *p) update_rq_clock(rq); p->sched_class->update_curr(rq); } - ns = p->se.sum_exec_runtime; + ns = TASK_SUM_EXEC_RUNTIME(p); task_rq_unlock(rq, p, &rf); return ns; @@ -3752,6 +3803,14 @@ void rt_mutex_setprio(struct task_struct *p, struct task_struct *pi_task) if (oldprio < prio) queue_flag |= ENQUEUE_HEAD; p->sched_class = &rt_sched_class; +#ifdef CONFIG_BT_SCHED + } else if (bt_prio(prio)) { + if (dl_prio(oldprio)) + p->dl.dl_boosted = 0; + if (rt_prio(oldprio)) + p->rt.timeout = 0; + p->sched_class = &bt_sched_class; +#endif } else { if (dl_prio(oldprio)) p->dl.dl_boosted = 0; @@ -3816,8 +3875,17 @@ void set_user_nice(struct task_struct *p, long nice) if (running) put_prev_task(rq, p); - p->static_prio = NICE_TO_PRIO(nice); - set_load_weight(p); +#ifdef CONFIG_BT_SCHED + if (task_has_bt_policy(p)) { + p->static_prio = NICE_TO_BT_PRIO(nice); + set_bt_load_weight(p); + } else +#endif + { + p->static_prio = NICE_TO_PRIO(nice); + set_load_weight(p); + } + old_prio = p->prio; p->prio = effective_prio(p); delta = p->prio - old_prio; @@ -3900,6 +3968,29 @@ int task_prio(const struct task_struct *p) return p->prio - MAX_RT_PRIO; } +/** + * task_nice - return the nice value of a given task. + * @p: the task in question. + * + * Return: The nice value [ -20 ... 0 ... 19 ]. + */ +inline int task_nice(const struct task_struct *p) +{ +#ifdef CONFIG_BT_SCHED + int task_nice = 0; + + if(bt_prio(p->static_prio)) + task_nice = TASK_BT_NICE(p); + else + task_nice = PRIO_TO_NICE((p)->static_prio); + + return task_nice; +#else + return PRIO_TO_NICE((p)->static_prio); +#endif +} +EXPORT_SYMBOL(task_nice); + /** * idle_cpu - is a given CPU idle currently? * @cpu: the processor in question. @@ -3967,6 +4058,15 @@ static void __setscheduler_params(struct task_struct *p, else if (fair_policy(policy)) p->static_prio = NICE_TO_PRIO(attr->sched_nice); +#ifdef CONFIG_BT_SCHED + if (unlikely(policy == SCHED_BT)) { + bt_prio_adjust_pos(&p->static_prio); + set_bt_load_weight(p); + } else if (policy == SCHED_NORMAL || policy == SCHED_BATCH || + policy == SCHED_IDLE) { + bt_prio_adjust_neg(&p->static_prio); + } +#endif /* * __sched_setscheduler() ensures attr->sched_priority == 0 when * !rt_policy. Always setting this ensures that things like @@ -3995,6 +4095,10 @@ static void __setscheduler(struct rq *rq, struct task_struct *p, p->sched_class = &dl_sched_class; else if (rt_prio(p->prio)) p->sched_class = &rt_sched_class; +#ifdef CONFIG_BT_SCHED + else if (bt_prio(p->prio)) + p->sched_class = &bt_sched_class; +#endif else p->sched_class = &fair_sched_class; } @@ -4031,6 +4135,10 @@ static int __sched_setscheduler(struct task_struct *p, /* The pi code expects interrupts enabled */ BUG_ON(pi && in_interrupt()); + + if(!sched_bt_on && SCHED_BT == policy) + return -EINVAL; + recheck: /* Double check policy once rq lock held: */ if (policy < 0) { @@ -5257,6 +5365,12 @@ void init_idle(struct task_struct *idle, int cpu) __sched_fork(0, idle); idle->state = TASK_RUNNING; idle->se.exec_start = sched_clock(); +#ifdef CONFIG_BT_SCHED + idle->bt.exec_start = idle->se.exec_start; +#if defined(CONFIG_SCHEDSTATS) || defined(CONFIG_LATENCYTOP) + idle->bt.bt_statistics = &idle->se.statistics; +#endif +#endif idle->flags |= PF_IDLE; kasan_unpoison_task_stack(idle); @@ -5881,6 +5995,10 @@ void __init sched_init(void) init_cfs_rq(&rq->cfs); init_rt_rq(&rq->rt); init_dl_rq(&rq->dl); +#ifdef CONFIG_BT_SCHED + rq->bt_nr_running = 0; + init_bt_rq(&rq->bt); +#endif #ifdef CONFIG_FAIR_GROUP_SCHED root_task_group.shares = ROOT_TASK_GROUP_LOAD; INIT_LIST_HEAD(&rq->leaf_cfs_rq_list); @@ -6069,6 +6187,9 @@ void normalize_rt_tasks(void) continue; p->se.exec_start = 0; +#ifdef CONFIG_BT_SCHED + p->bt.exec_start = 0; +#endif schedstat_set(p->se.statistics.wait_start, 0); schedstat_set(p->se.statistics.sleep_start, 0); schedstat_set(p->se.statistics.block_start, 0); diff --git a/kernel/sched/cpupri.c b/kernel/sched/cpupri.c index 2511aba36..755b75e36 100644 --- a/kernel/sched/cpupri.c +++ b/kernel/sched/cpupri.c @@ -30,6 +30,7 @@ #include #include #include +#include #include #include "cpupri.h" @@ -40,7 +41,11 @@ static int convert_prio(int prio) if (prio == CPUPRI_INVALID) cpupri = CPUPRI_INVALID; +#ifdef CONFIG_BT_SCHED + else if (prio >= MAX_PRIO - BT_PRIO_WIDTH) +#else else if (prio == MAX_PRIO) +#endif cpupri = CPUPRI_IDLE; else if (prio >= MAX_RT_PRIO) cpupri = CPUPRI_NORMAL; diff --git a/kernel/sched/cputime.c b/kernel/sched/cputime.c index 14d2dbf97..708be6d9b 100644 --- a/kernel/sched/cputime.c +++ b/kernel/sched/cputime.c @@ -108,6 +108,11 @@ static inline void task_group_account_field(struct task_struct *p, int index, * */ __this_cpu_add(kernel_cpustat.cpustat[index], tmp); +#ifdef CONFIG_BT_SCHED + if(p->sched_class == &bt_sched_class) { + __this_cpu_add(kernel_cpustat.cpustat[CPUTIME_BT], tmp); + } +#endif cpuacct_account_field(p, index, tmp); } @@ -273,7 +278,7 @@ static inline u64 account_other_time(u64 max) #ifdef CONFIG_64BIT static inline u64 read_sum_exec_runtime(struct task_struct *t) { - return t->se.sum_exec_runtime; + return TASK_SUM_EXEC_RUNTIME(t); } #else static u64 read_sum_exec_runtime(struct task_struct *t) @@ -283,7 +288,7 @@ static u64 read_sum_exec_runtime(struct task_struct *t) struct rq *rq; rq = task_rq_lock(t, &rf); - ns = t->se.sum_exec_runtime; + ns = TASK_SUM_EXEC_RUNTIME(t); task_rq_unlock(rq, t, &rf); return ns; @@ -661,7 +666,7 @@ out: void task_cputime_adjusted(struct task_struct *p, u64 *ut, u64 *st) { struct task_cputime cputime = { - .sum_exec_runtime = p->se.sum_exec_runtime, + .sum_exec_runtime = TASK_SUM_EXEC_RUNTIME(p), }; task_cputime(p, &cputime.utime, &cputime.stime); diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c index 2f93e4a2d..1901bc5ae 100644 --- a/kernel/sched/debug.c +++ b/kernel/sched/debug.c @@ -20,25 +20,14 @@ #include #include "sched.h" +#include "bt_debug.h" static DEFINE_SPINLOCK(sched_debug_lock); -/* - * This allows printing both to /proc/sched_debug and - * to the console - */ -#define SEQ_printf(m, x...) \ - do { \ - if (m) \ - seq_printf(m, x); \ - else \ - printk(x); \ - } while (0) - /* * Ease the printing of nsec fields: */ -static long long nsec_high(unsigned long long nsec) +long long nsec_high(unsigned long long nsec) { if ((long long)nsec < 0) { nsec = -nsec; @@ -50,7 +39,7 @@ static long long nsec_high(unsigned long long nsec) return nsec; } -static unsigned long nsec_low(unsigned long long nsec) +unsigned long nsec_low(unsigned long long nsec) { if ((long long)nsec < 0) nsec = -nsec; @@ -58,8 +47,6 @@ static unsigned long nsec_low(unsigned long long nsec) return do_div(nsec, 1000000); } -#define SPLIT_NS(x) nsec_high(x), nsec_low(x) - #define SCHED_FEAT(name, enabled) \ #name , @@ -705,6 +692,9 @@ do { \ print_cfs_stats(m, cpu); print_rt_stats(m, cpu); print_dl_stats(m, cpu); +#ifdef CONFIG_BT_SCHED + print_bt_stats(m, cpu); +#endif print_rq(m, rq, cpu); spin_unlock_irqrestore(&sched_debug_lock, flags); @@ -1004,6 +994,9 @@ void proc_sched_show_task(struct task_struct *p, struct pid_namespace *ns, "nr_involuntary_switches", (long long)p->nivcsw); P(se.load.weight); +#ifdef CONFIG_BT_SCHED + SEQ_printf(m, "%-45s:%21Ld\n", "bt.load.weight", (long long)scale_load_down(p->bt.load.weight)); +#endif #ifdef CONFIG_SMP P(se.avg.load_sum); P(se.avg.util_sum); diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index f36c0ac8e..d4dd7588c 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -37,6 +37,10 @@ #include #include "sched.h" +#include "fair.h" +#ifdef CONFIG_BT_SCHED +#include "batch.h" +#endif /* * Targeted preemption latency for CPU-bound tasks: @@ -78,7 +82,7 @@ unsigned int normalized_sysctl_sched_min_granularity = 750000ULL; /* * This value is kept at sysctl_sched_latency/sysctl_sched_min_granularity */ -static unsigned int sched_nr_latency = 8; +unsigned int sched_nr_latency = 8; /* * After fork, child runs first. If set to 0 (default) then @@ -132,19 +136,19 @@ unsigned int sysctl_sched_cfs_bandwidth_slice = 5000UL; */ unsigned int capacity_margin = 1280; -static inline void update_load_add(struct load_weight *lw, unsigned long inc) +inline void update_load_add(struct load_weight *lw, unsigned long inc) { lw->weight += inc; lw->inv_weight = 0; } -static inline void update_load_sub(struct load_weight *lw, unsigned long dec) +inline void update_load_sub(struct load_weight *lw, unsigned long dec) { lw->weight -= dec; lw->inv_weight = 0; } -static inline void update_load_set(struct load_weight *lw, unsigned long w) +inline void update_load_set(struct load_weight *lw, unsigned long w) { lw->weight = w; lw->inv_weight = 0; @@ -180,7 +184,7 @@ static unsigned int get_update_sysctl_factor(void) return factor; } -static void update_sysctl(void) +void update_sysctl(void) { unsigned int factor = get_update_sysctl_factor(); @@ -229,7 +233,7 @@ static void __update_inv_weight(struct load_weight *lw) * Or, weight =< lw.weight (because lw.weight is the runqueue weight), thus * weight/lw.weight <= 1, and therefore our shift will also be positive. */ -static u64 __calc_delta(u64 delta_exec, unsigned long weight, struct load_weight *lw) +u64 __calc_delta(u64 delta_exec, unsigned long weight, struct load_weight *lw) { u64 fact = scale_load_down(weight); int shift = WMULT_SHIFT; @@ -490,25 +494,6 @@ void account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec); /************************************************************** * Scheduling class tree data structure manipulation methods: */ - -static inline u64 max_vruntime(u64 max_vruntime, u64 vruntime) -{ - s64 delta = (s64)(vruntime - max_vruntime); - if (delta > 0) - max_vruntime = vruntime; - - return max_vruntime; -} - -static inline u64 min_vruntime(u64 min_vruntime, u64 vruntime) -{ - s64 delta = (s64)(vruntime - min_vruntime); - if (delta < 0) - min_vruntime = vruntime; - - return min_vruntime; -} - static inline int entity_before(struct sched_entity *a, struct sched_entity *b) { @@ -663,7 +648,7 @@ static inline u64 calc_delta_fair(u64 delta, struct sched_entity *se) * * p = (nr <= nl) ? l : l*nr/nl */ -static u64 __sched_period(unsigned long nr_running) +u64 __sched_period(unsigned long nr_running) { if (unlikely(nr_running > sched_nr_latency)) return nr_running * sysctl_sched_min_granularity; @@ -2637,7 +2622,7 @@ void task_tick_numa(struct rq *rq, struct task_struct *curr) * task needs to have done some actual work before we bother with * NUMA placement. */ - now = curr->se.sum_exec_runtime; + now = TASK_SUM_EXEC_RUNTIME(curr); period = (u64)curr->numa_scan_period * NSEC_PER_MSEC; if (now > curr->node_stamp + period) { @@ -2653,7 +2638,7 @@ void task_tick_numa(struct rq *rq, struct task_struct *curr) } #else -static void task_tick_numa(struct rq *rq, struct task_struct *curr) +void task_tick_numa(struct rq *rq, struct task_struct *curr) { } @@ -3547,7 +3532,7 @@ static inline unsigned long cfs_rq_load_avg(struct cfs_rq *cfs_rq) return cfs_rq->avg.load_avg; } -static int idle_balance(struct rq *this_rq, struct rq_flags *rf); +int idle_balance(struct rq *this_rq, struct rq_flags *rf); #else /* CONFIG_SMP */ @@ -3576,7 +3561,7 @@ attach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) {} static inline void detach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) {} -static inline int idle_balance(struct rq *rq, struct rq_flags *rf) +inline int idle_balance(struct rq *rq, struct rq_flags *rf) { return 0; } @@ -5327,7 +5312,11 @@ static unsigned long cpu_avg_load_per_task(int cpu) struct rq *rq = cpu_rq(cpu); unsigned long nr_running = READ_ONCE(rq->cfs.h_nr_running); unsigned long load_avg = weighted_cpuload(rq); +#ifdef CONFIG_BT_SCHED + unsigned long bt_running = READ_ONCE(rq->bt.h_nr_running); + nr_running -= bt_running; +#endif if (nr_running) return load_avg / nr_running; @@ -6230,7 +6219,11 @@ static void check_preempt_wakeup(struct rq *rq, struct task_struct *p, int wake_ /* Idle tasks are by definition preempted by non-idle tasks. */ if (unlikely(curr->policy == SCHED_IDLE) && +#ifdef CONFIG_BT_SCHED + likely(p->policy != SCHED_IDLE && p->policy != SCHED_BT)) +#else likely(p->policy != SCHED_IDLE)) +#endif goto preempt; /* @@ -6279,11 +6272,17 @@ pick_next_task_fair(struct rq *rq, struct task_struct *prev, struct rq_flags *rf struct cfs_rq *cfs_rq = &rq->cfs; struct sched_entity *se; struct task_struct *p; +#ifndef CONFIG_BT_SCHED int new_tasks; again: +#endif if (!cfs_rq->nr_running) +#ifdef CONFIG_BT_SCHED + return NULL; +#else goto idle; +#endif #ifdef CONFIG_FAIR_GROUP_SCHED if (prev->sched_class != &fair_sched_class) @@ -6322,7 +6321,11 @@ again: cfs_rq = &rq->cfs; if (!cfs_rq->nr_running) +#ifdef CONFIG_BT_SCHED + return NULL; +#else goto idle; +#endif goto simple; } @@ -6381,7 +6384,7 @@ simple: hrtick_start_fair(rq, p); return p; - +#ifndef CONFIG_BT_SCHED idle: new_tasks = idle_balance(rq, rf); @@ -6397,6 +6400,7 @@ idle: goto again; return NULL; +#endif } /* @@ -6427,7 +6431,7 @@ static void yield_task_fair(struct rq *rq) /* * Are we the only task in the tree? */ - if (unlikely(rq->nr_running == 1)) + if (unlikely(RQ_CFS_NR_RUNNING(rq) == 1)) return; clear_buddies(cfs_rq, se); @@ -7430,7 +7434,7 @@ static inline void update_sg_lb_stats(struct lb_env *env, sgs->group_util += cpu_util(i); sgs->sum_nr_running += rq->cfs.h_nr_running; - nr_running = rq->nr_running; + nr_running = RQ_CFS_NR_RUNNING(rq); if (nr_running > 1) *overload = true; @@ -7975,7 +7979,7 @@ static struct rq *find_busiest_queue(struct lb_env *env, * which is not scaled with the cpu capacity. */ - if (rq->nr_running == 1 && wl > env->imbalance && + if (RQ_CFS_NR_RUNNING(rq) == 1 && wl > env->imbalance && !check_cpu_capacity(rq, env->sd)) continue; @@ -8135,7 +8139,7 @@ redo: env.src_rq = busiest; ld_moved = 0; - if (busiest->nr_running > 1) { + if (RQ_CFS_NR_RUNNING(busiest) > 1) { /* * Attempt to move tasks. If find_busiest_group has found * an imbalance but busiest->nr_running <= 1, the group is @@ -8143,7 +8147,7 @@ redo: * correctly treated as an imbalance. */ env.flags |= LBF_ALL_PINNED; - env.loop_max = min(sysctl_sched_nr_migrate, busiest->nr_running); + env.loop_max = min(sysctl_sched_nr_migrate, RQ_CFS_NR_RUNNING(busiest)); more_balance: rq_lock_irqsave(busiest, &rf); @@ -8377,7 +8381,7 @@ update_next_balance(struct sched_domain *sd, unsigned long *next_balance) * idle_balance is called by schedule() if this_cpu is about to become * idle. Attempts to pull tasks from other CPUs. */ -static int idle_balance(struct rq *this_rq, struct rq_flags *rf) +int idle_balance(struct rq *this_rq, struct rq_flags *rf) { unsigned long next_balance = jiffies + HZ; int this_cpu = this_rq->cpu; @@ -8518,7 +8522,7 @@ static int active_load_balance_cpu_stop(void *data) goto out_unlock; /* Is there any task to move? */ - if (busiest_rq->nr_running <= 1) + if (RQ_CFS_NR_RUNNING(busiest_rq) <= 1) goto out_unlock; /* @@ -8916,7 +8920,11 @@ static inline bool nohz_kick_needed(struct rq *rq) int nr_busy, i, cpu = rq->cpu; bool kick = false; +#ifdef CONFIG_BT_SCHED + if (unlikely(rq->idle_balance && !rq->nr_running)) +#else if (unlikely(rq->idle_balance)) +#endif return false; /* @@ -9521,7 +9529,11 @@ static unsigned int get_rr_interval_fair(struct rq *rq, struct task_struct *task * All the scheduling class methods: */ const struct sched_class fair_sched_class = { +#ifdef CONFIG_BT_SCHED + .next = &bt_sched_class, +#else .next = &idle_sched_class, +#endif .enqueue_task = enqueue_task_fair, .dequeue_task = dequeue_task_fair, .yield_task = yield_task_fair, diff --git a/kernel/sched/fair.h b/kernel/sched/fair.h new file mode 100644 index 000000000..020b7d171 --- /dev/null +++ b/kernel/sched/fair.h @@ -0,0 +1,47 @@ +/* + * Copyright (C) 2019 Tencent Ltd. All rights reserved. + * + * File Name ：fair.h + * Author ： + * Date ：2019-12-26 + * Descriptor： + */ + +#ifndef _FAIR_H +#define _FAIR_H + +extern unsigned int sched_nr_latency; + +unsigned long calc_delta_mine(unsigned long delta_exec, + unsigned long weight, struct load_weight *lw); + +extern u64 +__calc_delta(u64 delta_exec, unsigned long weight, struct load_weight *lw); +extern int idle_balance(struct rq *this_rq, struct rq_flags *rf); +static inline u64 max_vruntime(u64 max_vruntime, u64 vruntime) +{ + s64 delta = (s64)(vruntime - max_vruntime); + + if (delta > 0) + max_vruntime = vruntime; + + return max_vruntime; +} + +static inline u64 min_vruntime(u64 min_vruntime, u64 vruntime) +{ + s64 delta = (s64)(vruntime - min_vruntime); + + if (delta < 0) + min_vruntime = vruntime; + + return min_vruntime; +} + +u64 __sched_period(unsigned long nr_running); + +void task_tick_numa(struct rq *rq, struct task_struct *curr); + +void update_sysctl(void); + +#endif diff --git a/kernel/sched/loadavg.c b/kernel/sched/loadavg.c index 89a989e4d..e7b328ff0 100644 --- a/kernel/sched/loadavg.c +++ b/kernel/sched/loadavg.c @@ -84,8 +84,8 @@ long calc_load_fold_active(struct rq *this_rq, long adjust) { long nr_active, delta = 0; - nr_active = this_rq->nr_running - adjust; - nr_active += (long)this_rq->nr_uninterruptible; + nr_active = RQ_CFS_NR_RUNNING(this_rq) - adjust; + nr_active += (long)RQ_CFS_NR_UNINTERRUPTIBLE(this_rq); if (nr_active != this_rq->calc_load_active) { delta = nr_active - this_rq->calc_load_active; diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 452b56923..d5a4c8647 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -39,6 +39,7 @@ #include "cpupri.h" #include "cpudeadline.h" #include "cpuacct.h" +#include "batch.h" #ifdef CONFIG_SCHED_DEBUG # define SCHED_WARN_ON(x) WARN_ONCE(x, #x) @@ -114,6 +115,23 @@ static inline void cpu_load_update_active(struct rq *this_rq) { } */ #define DL_SCALE (10) + +#ifdef CONFIG_BT_SCHED +#define RQ_CFS_NR_UNINTERRUPTIBLE(rq) \ + ((rq)->nr_uninterruptible - (rq)->bt.nr_uninterruptible) + +#define RQ_CFS_NR_RUNNING(rq) \ + ((rq)->nr_running - (rq)->bt_nr_running) + +#else +#define RQ_CFS_NR_UNINTERRUPTIBLE(rq) \ + ((rq)->nr_uninterruptible) + +#define RQ_CFS_NR_RUNNING(rq) \ + ((rq)->nr_running) + +#endif + /* * These are the 'tuning knobs' of the scheduler: */ @@ -141,9 +159,13 @@ static inline int dl_policy(int policy) { return policy == SCHED_DEADLINE; } + static inline bool valid_policy(int policy) { return idle_policy(policy) || fair_policy(policy) || +#ifdef CONFIG_BT_SCHED + bt_policy(policy) || +#endif rt_policy(policy) || dl_policy(policy); } @@ -690,6 +712,9 @@ struct rq { * remote CPUs use both these fields when doing load calculation. */ unsigned int nr_running; +#ifdef CONFIG_BT_SCHED + unsigned int bt_nr_running; +#endif #ifdef CONFIG_NUMA_BALANCING unsigned int nr_numa_running; unsigned int nr_preferred_running; @@ -713,6 +738,9 @@ struct rq { struct cfs_rq cfs; struct rt_rq rt; struct dl_rq dl; +#ifdef CONFIG_BT_SCHED + struct bt_rq bt; +#endif #ifdef CONFIG_FAIR_GROUP_SCHED /* list of leaf cfs_rq on this cpu: */ @@ -1106,6 +1134,13 @@ struct sched_group { unsigned long cpumask[0]; }; +#define tsk_cpus_allowed(tsk) (&(tsk)->cpus_allowed) + +static inline struct cpumask *sched_group_cpus(struct sched_group *sg) +{ + return to_cpumask(sg->cpumask); +} + static inline struct cpumask *sched_group_span(struct sched_group *sg) { return to_cpumask(sg->cpumask); @@ -1357,6 +1392,9 @@ static inline void finish_lock_switch(struct rq *rq, struct task_struct *prev) #define WF_FORK 0x02 /* child wakeup after fork */ #define WF_MIGRATED 0x4 /* internal use, task got migrated */ +extern inline void update_load_add(struct load_weight *lw, unsigned long inc); +extern inline void update_load_sub(struct load_weight *lw, unsigned long dec); +extern inline void update_load_set(struct load_weight *lw, unsigned long w); /* * To aid in avoiding the subversion of "niceness" due to uneven distribution * of tasks with abnormal "nice" values across CPUs the contribution that @@ -1601,7 +1639,7 @@ static inline void add_nr_running(struct rq *rq, unsigned count) if (prev_nr < 2 && rq->nr_running >= 2) { #ifdef CONFIG_SMP - if (!rq->rd->overload) + if (!rq->rd->overload && RQ_CFS_NR_RUNNING(rq) >= 2) rq->rd->overload = true; #endif } diff --git a/kernel/time/posix-cpu-timers.c b/kernel/time/posix-cpu-timers.c index 6e8c230ca..056a9a2fc 100644 --- a/kernel/time/posix-cpu-timers.c +++ b/kernel/time/posix-cpu-timers.c @@ -819,7 +819,7 @@ static void check_thread_timers(struct task_struct *tsk, tsk_expires->virt_exp = expires; tsk_expires->sched_exp = check_timers_list(++timers, firing, - tsk->se.sum_exec_runtime); + TASK_SUM_EXEC_RUNTIME(tsk)); /* * Check for the special case thread timers. @@ -1082,7 +1082,7 @@ static inline int fastpath_timer_check(struct task_struct *tsk) struct task_cputime task_sample; task_cputime(tsk, &task_sample.utime, &task_sample.stime); - task_sample.sum_exec_runtime = tsk->se.sum_exec_runtime; + task_sample.sum_exec_runtime = TASK_SUM_EXEC_RUNTIME(tsk); if (task_cputime_expired(&task_sample, &tsk->cputime_expires)) return 1; } diff --git a/package/default/config.default b/package/default/config.default index 85598294e..c53d9421b 100644 --- a/package/default/config.default +++ b/package/default/config.default @@ -184,6 +184,7 @@ CONFIG_IPC_NS=y CONFIG_USER_NS=y CONFIG_PID_NS=y CONFIG_NET_NS=y +CONFIG_BT_SCHED=y CONFIG_SCHED_AUTOGROUP=y # CONFIG_SYSFS_DEPRECATED is not set CONFIG_RELAY=y