zhnn
/
TencentOS-kernel
forked from TencentOS/TencentOS-kernel
0
0
Fork 1
Code Issues Pull Requests Projects Releases Wiki Activity

sched: add offline scheduler class 

The offline scheduler named BT sched is based on the CFS scheduler. We
also use the rb-tree as the run queue to save the runnable tasks. And the
vruntime concept is also used in the offline scheduler. And the priority
of offline scheduler is from 140 to 179. So now the schedulers in the
kernel are as follows: stop, RT, CFS, BT and idle.

Signed-off-by: Xiaoming Gao <newtongao@tencent.com>
Signed-off-by: Hua Liu <shookliu@tencent.com>
Signed-off-by: Xiaogguang Chen <xiaoggchen@tencent.com>
Signed-off-by: Zhiguang Peng <zgpeng@tencent.com>
Signed-off-by: Bin Fan <tombinfan@tencent.com>
Signed-off-by: He Chen <heddchen@tencent.com>
This commit is contained in:
He Chen 2020-03-02 16:14:50 +08:00 committed by Xiaoming Gao
parent f540f245de
commit b48a63ef2c
30 changed files with 2234 additions and 87 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
fs/proc/Makefile
View File

@ -26,6 +26,7 @@ proc-y += softirqs.o
proc-y += namespaces.o
proc-y += self.o
proc-y += thread_self.o
proc-$(CONFIG_BT_SCHED) += bt_stat.o
proc-$(CONFIG_PROC_SYSCTL) += proc_sysctl.o
proc-$(CONFIG_NET) += proc_net.o
proc-$(CONFIG_PROC_KCORE) += kcore.o

2
fs/proc/base.c
View File

@ -481,7 +481,7 @@ static int proc_pid_schedstat(struct seq_file *m, struct pid_namespace *ns,
seq_printf(m, "0 0 0\n");
else
seq_printf(m, "%llu %llu %lu\n",
(unsigned long long)task->se.sum_exec_runtime,
TASK_SUM_EXEC_RUNTIME(task),
(unsigned long long)task->sched_info.run_delay,
task->sched_info.pcount);

182
fs/proc/bt_stat.c Normal file
View File

@ -0,0 +1,182 @@
/*
* Copyright (C) 2019 Tencent Ltd. All rights reserved.
*
* File Name ：bt_stat.c
* Author ：
* Date ：2019-12-28
* Descriptor：
*/
// SPDX-License-Identifier: GPL-2.0
#include <linux/cpumask.h>
#include <linux/fs.h>
#include <linux/init.h>
#include <linux/interrupt.h>
#include <linux/kernel_stat.h>
#include <linux/proc_fs.h>
#include <linux/sched.h>
#include <linux/sched/stat.h>
#include <linux/sched/batch.h>
#include <linux/seq_file.h>
#include <linux/slab.h>
#include <linux/time.h>
#include <linux/irqnr.h>
#include <linux/sched/cputime.h>
#include <linux/tick.h>
extern u64 get_idle_time(int cpu);
extern u64 get_iowait_time(int cpu);
static int show_bt_stat(struct seq_file *p, void *v)
{
int i, j;
unsigned long jif;
u64 user, nice, system, idle, iowait, irq, softirq, steal, cputime_bt;
u64 guest, guest_nice;
u64 sum = 0;
u64 sum_softirq = 0;
unsigned int per_softirq_sums[NR_SOFTIRQS] = {0};
struct timespec boottime;
user = nice = system = idle = iowait =
irq = softirq = steal = cputime_bt = 0;
guest = guest_nice = 0;
getboottime(&boottime);
jif = boottime.tv_sec;
for_each_possible_cpu(i) {
user += kcpustat_cpu(i).cpustat[CPUTIME_USER];
nice += kcpustat_cpu(i).cpustat[CPUTIME_NICE];
system += kcpustat_cpu(i).cpustat[CPUTIME_SYSTEM];
idle += get_idle_time(i);
iowait += get_iowait_time(i);
irq += kcpustat_cpu(i).cpustat[CPUTIME_IRQ];
softirq += kcpustat_cpu(i).cpustat[CPUTIME_SOFTIRQ];
steal += kcpustat_cpu(i).cpustat[CPUTIME_STEAL];
guest += kcpustat_cpu(i).cpustat[CPUTIME_GUEST];
guest_nice += kcpustat_cpu(i).cpustat[CPUTIME_GUEST_NICE];
cputime_bt += kcpustat_cpu(i).cpustat[CPUTIME_BT];
sum += kstat_cpu_irqs_sum(i);
sum += arch_irq_stat_cpu(i);
for (j = 0; j < NR_SOFTIRQS; j++) {
unsigned int softirq_stat = kstat_softirqs_cpu(j, i);
per_softirq_sums[j] += softirq_stat;
sum_softirq += softirq_stat;
}
}
sum += arch_irq_stat();
seq_puts(p, "cpu ");
seq_put_decimal_ull(p, " ", nsec_to_clock_t(user));
seq_put_decimal_ull(p, " ", nsec_to_clock_t(nice));
seq_put_decimal_ull(p, " ", nsec_to_clock_t(system));
seq_put_decimal_ull(p, " ", nsec_to_clock_t(idle));
seq_put_decimal_ull(p, " ", nsec_to_clock_t(iowait));
seq_put_decimal_ull(p, " ", nsec_to_clock_t(irq));
seq_put_decimal_ull(p, " ", nsec_to_clock_t(softirq));
seq_put_decimal_ull(p, " ", nsec_to_clock_t(steal));
seq_put_decimal_ull(p, " ", nsec_to_clock_t(guest));
seq_put_decimal_ull(p, " ", nsec_to_clock_t(guest_nice));
seq_put_decimal_ull(p, " ", nsec_to_clock_t(cputime_bt));
seq_putc(p, '\n');
for_each_online_cpu(i) {
/* Copy values here to work around gcc-2.95.3, gcc-2.96 */
user = kcpustat_cpu(i).cpustat[CPUTIME_USER];
nice = kcpustat_cpu(i).cpustat[CPUTIME_NICE];
system = kcpustat_cpu(i).cpustat[CPUTIME_SYSTEM];
idle = get_idle_time(i);
iowait = get_iowait_time(i);
irq = kcpustat_cpu(i).cpustat[CPUTIME_IRQ];
softirq = kcpustat_cpu(i).cpustat[CPUTIME_SOFTIRQ];
steal = kcpustat_cpu(i).cpustat[CPUTIME_STEAL];
guest = kcpustat_cpu(i).cpustat[CPUTIME_GUEST];
guest_nice = kcpustat_cpu(i).cpustat[CPUTIME_GUEST_NICE];
cputime_bt = kcpustat_cpu(i).cpustat[CPUTIME_BT];
seq_printf(p, "cpu%d", i);
seq_put_decimal_ull(p, " ", nsec_to_clock_t(user));
seq_put_decimal_ull(p, " ", nsec_to_clock_t(nice));
seq_put_decimal_ull(p, " ", nsec_to_clock_t(system));
seq_put_decimal_ull(p, " ", nsec_to_clock_t(idle));
seq_put_decimal_ull(p, " ", nsec_to_clock_t(iowait));
seq_put_decimal_ull(p, " ", nsec_to_clock_t(irq));
seq_put_decimal_ull(p, " ", nsec_to_clock_t(softirq));
seq_put_decimal_ull(p, " ", nsec_to_clock_t(steal));
seq_put_decimal_ull(p, " ", nsec_to_clock_t(guest));
seq_put_decimal_ull(p, " ", nsec_to_clock_t(guest_nice));
seq_put_decimal_ull(p, " ", nsec_to_clock_t(cputime_bt));
seq_putc(p, '\n');
}
seq_printf(p, "intr %llu", (unsigned long long)sum);
/* sum again ? it could be updated? */
for_each_irq_nr(j)
seq_put_decimal_ull(p, " ", kstat_irqs(j));
seq_printf(p,
"\nctxt %llu\n"
"btime %lu\n"
"processes %d\n"
"procs_running %lu\n"
"procs_blocked %lu\n",
nr_context_switches(),
(unsigned long)jif,
nr_forks(),
nr_running(),
nr_iowait());
seq_printf(p, "softirq %llu", (unsigned long long)sum_softirq);
for (i = 0; i < NR_SOFTIRQS; i++)
seq_put_decimal_ull(p, " ", per_softirq_sums[i]);
seq_putc(p, '\n');
return 0;
}
static int bt_stat_open(struct inode *inode, struct file *file)
{
size_t size = 1024 + 128 * num_online_cpus();
char *buf;
struct seq_file *m;
int res;
/* minimum size to display an interrupt count : 2 bytes */
size += 2 * nr_irqs;
/* don't ask for more than the kmalloc() max size */
if (size > KMALLOC_MAX_SIZE)
size = KMALLOC_MAX_SIZE;
buf = kmalloc(size, GFP_KERNEL);
if (!buf)
return -ENOMEM;
res = single_open(file, show_bt_stat, NULL);
if (!res) {
m = file->private_data;
m->buf = buf;
m->size = ksize(buf);
} else {
kfree(buf);
}
return res;
}
static const struct file_operations proc_bt_stat_operations = {
.open = bt_stat_open,
.read = seq_read,
.llseek = seq_lseek,
.release = single_release,
};
static int __init proc_bt_stat_init(void)
{
if(sched_bt_on)
proc_create("bt_stat", 0000, NULL, &proc_bt_stat_operations);
return 0;
}
fs_initcall(proc_bt_stat_init);

8
fs/proc/stat.c
View File

@ -23,7 +23,7 @@
#ifdef arch_idle_time
static u64 get_idle_time(int cpu)
u64 get_idle_time(int cpu)
{
u64 idle;
@ -33,7 +33,7 @@ static u64 get_idle_time(int cpu)
return idle;
}
static u64 get_iowait_time(int cpu)
u64 get_iowait_time(int cpu)
{
u64 iowait;
@ -45,7 +45,7 @@ static u64 get_iowait_time(int cpu)
#else
static u64 get_idle_time(int cpu)
u64 get_idle_time(int cpu)
{
u64 idle, idle_usecs = -1ULL;
@ -61,7 +61,7 @@ static u64 get_idle_time(int cpu)
return idle;
}
static u64 get_iowait_time(int cpu)
u64 get_iowait_time(int cpu)
{
u64 iowait, iowait_usecs = -1ULL;

15
include/linux/init_task.h
View File

@ -219,6 +219,15 @@ extern struct cred init_cred;
#define INIT_TASK_SECURITY
#endif
#ifdef CONFIG_BT_SCHED
#define INIT_PRIO MAX_PRIO - 20 - 40
#define INIT_STATIC_PRIO MAX_PRIO - 20 - 40
#define INIT_NORMAL_PRIO MAX_PRIO - 20 - 40
#else
#define INIT_PRIO MAX_PRIO - 20
#define INIT_STATIC_PRIO MAX_PRIO - 20
#define INIT_NORMAL_PRIO MAX_PRIO - 20
#endif
/*
* INIT_TASK is used to set up the first task table, touch at
* your own risk!. Base=0, limit=0x1fffff (=2MB)
@ -230,9 +239,9 @@ extern struct cred init_cred;
.stack = init_stack, \
.usage = ATOMIC_INIT(2), \
.flags = PF_KTHREAD, \
.prio = MAX_PRIO-20, \
.static_prio = MAX_PRIO-20, \
.normal_prio = MAX_PRIO-20, \
.prio = INIT_PRIO, \
.static_prio = INIT_STATIC_PRIO, \
.normal_prio = INIT_NORMAL_PRIO, \
.policy = SCHED_NORMAL, \
.cpus_allowed = CPU_MASK_ALL, \
.nr_cpus_allowed= NR_CPUS, \

4
include/linux/ioprio.h
View File

@ -61,7 +61,11 @@ static inline int task_nice_ioprio(struct task_struct *task)
*/
static inline int task_nice_ioclass(struct task_struct *task)
{
#ifdef CONFIG_BT_SCHED
if (task->policy == SCHED_IDLE || task->policy == SCHED_BT)
#else
if (task->policy == SCHED_IDLE)
#endif
return IOPRIO_CLASS_IDLE;
else if (task->policy == SCHED_FIFO || task->policy == SCHED_RR)
return IOPRIO_CLASS_RT;

3
include/linux/kernel_stat.h
View File

@ -28,6 +28,9 @@ enum cpu_usage_stat {
CPUTIME_STEAL,
CPUTIME_GUEST,
CPUTIME_GUEST_NICE,
#ifdef CONFIG_BT_SCHED
CPUTIME_BT,
#endif
NR_STATS,
};

28
include/linux/sched.h
View File

@ -111,6 +111,16 @@ struct task_group;
(task->flags & PF_FROZEN) == 0 && \
(task->state & TASK_NOLOAD) == 0)
#ifdef CONFIG_BT_SCHED
#define TASK_SUM_EXEC_RUNTIME(tsk) \
(unsigned long long)((tsk)->se.sum_exec_runtime + (tsk)->bt.sum_exec_runtime)
#else
#define TASK_SUM_EXEC_RUNTIME(tsk) \
(unsigned long long)((tsk)->se.sum_exec_runtime)
#endif
#ifdef CONFIG_DEBUG_ATOMIC_SLEEP
/*
@ -429,6 +439,9 @@ struct sched_entity {
u64 nr_migrations;
struct sched_statistics statistics;
#ifdef CONFIG_BT_SCHED
struct sched_statistics *bt_statistics;
#endif
#ifdef CONFIG_FAIR_GROUP_SCHED
int depth;
@ -601,6 +614,9 @@ struct task_struct {
const struct sched_class *sched_class;
struct sched_entity se;
#ifdef CONFIG_BT_SCHED
struct sched_entity bt;
#endif
struct sched_rt_entity rt;
#ifdef CONFIG_CGROUP_SCHED
struct task_group *sched_task_group;
@ -1477,17 +1493,7 @@ extern int yield_to(struct task_struct *p, bool preempt);
extern void set_user_nice(struct task_struct *p, long nice);
extern int task_prio(const struct task_struct *p);
/**
* task_nice - return the nice value of a given task.
* @p: the task in question.
*
* Return: The nice value [ -20 ... 0 ... 19 ].
*/
static inline int task_nice(const struct task_struct *p)
{
return PRIO_TO_NICE((p)->static_prio);
}
extern int task_nice(const struct task_struct *p);
extern int can_nice(const struct task_struct *p, const int nice);
extern int task_curr(const struct task_struct *p);
extern int idle_cpu(int cpu);

60
include/linux/sched/batch.h Normal file
View File

@ -0,0 +1,60 @@
/*
* Copyright (C) 2019 Tencent Ltd. All rights reserved.
*
* File Name ：batch.h
* Author ：
* Date ：2019-12-26
* Descriptor：
*/
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _SCHED_BATCH_H
#define _SCHED_BATCH_H
#define MAX_CFS_PRIO 139
#define MIN_BT_PRIO 140
#define MAX_BT_PRIO 179
#define BT_PRIO_WIDTH (MAX_BT_PRIO - MIN_BT_PRIO + 1)
/*
* Convert user-nice values [ -20 ... 0 ... 19 ]
* to bt static priority [ MIN_BT_PRI + 1 ..MAX_BT_PRIO ],
* and back.
*/
#define NICE_TO_BT_PRIO(nice) (MAX_RT_PRIO + (nice) + 20 + 40)
#define PRIO_TO_BT_NICE(prio) ((prio) - MAX_RT_PRIO - 20 - 40)
#define TASK_BT_NICE(p) PRIO_TO_BT_NICE((p)->static_prio)
extern unsigned int sched_bt_on;
static inline int cfs_prio(int prio)
{
if (prio >= MAX_RT_PRIO && prio < MIN_BT_PRIO)
return 1;
return 0;
}
static inline int bt_prio(int prio)
{
if (prio > MAX_CFS_PRIO && prio < MAX_PRIO)
return 1;
return 0;
}
static inline void bt_prio_adjust_pos(int *prio)
{
int priority = *prio;
if (cfs_prio(priority))
*prio = priority + BT_PRIO_WIDTH;
}
static inline void bt_prio_adjust_neg(int *prio)
{
int priority = *prio;
if (bt_prio(priority))
*prio = priority - BT_PRIO_WIDTH;
}
#endif /* _SCHED_BATCH_H */

4
include/linux/sched/prio.h
View File

@ -22,7 +22,11 @@
#define MAX_USER_RT_PRIO 100
#define MAX_RT_PRIO MAX_USER_RT_PRIO
#ifdef CONFIG_BT_SCHED
#define MAX_PRIO (MAX_RT_PRIO + NICE_WIDTH + 40)
#else
#define MAX_PRIO (MAX_RT_PRIO + NICE_WIDTH)
#endif
#define DEFAULT_PRIO (MAX_RT_PRIO + NICE_WIDTH / 2)
/*

3
include/uapi/linux/sched.h
View File

@ -40,6 +40,9 @@
/* SCHED_ISO: reserved but not implemented yet */
#define SCHED_IDLE 5
#define SCHED_DEADLINE 6
#ifdef CONFIG_BT_SCHED
#define SCHED_BT 7
#endif
/* Can be ORed in to make sure the process is reverted back to SCHED_NORMAL on fork */
#define SCHED_RESET_ON_FORK 0x40000000

6
init/Kconfig
View File

@ -947,6 +947,12 @@ config NET_NS
endif # NAMESPACES
config BT_SCHED
bool "offline sched class"
default n
help
Allow user to create offline task
config SCHED_AUTOGROUP
bool "Automatic process group scheduling"
select CGROUPS

9
init/main.c
View File

@ -353,6 +353,15 @@ static int __init rdinit_setup(char *str)
}
__setup("rdinit=", rdinit_setup);
unsigned int sched_bt_on;
static int __init set_sched_bt_on(char *str)
{
sched_bt_on = 1;
return 1;
}
early_param("offline_class", set_sched_bt_on);
#ifndef CONFIG_SMP
static const unsigned int setup_max_cpus = NR_CPUS;
static inline void setup_nr_cpu_ids(void) { }

2
kernel/delayacct.c
View File

@ -115,7 +115,7 @@ int __delayacct_add_tsk(struct taskstats *d, struct task_struct *tsk)
*/
t1 = tsk->sched_info.pcount;
t2 = tsk->sched_info.run_delay;
t3 = tsk->se.sum_exec_runtime;
t3 = TASK_SUM_EXEC_RUNTIME(tsk);
d->cpu_count += t1;

2
kernel/exit.c
View File

@ -150,7 +150,7 @@ static void __exit_signal(struct task_struct *tsk)
sig->inblock += task_io_get_inblock(tsk);
sig->oublock += task_io_get_oublock(tsk);
task_io_accounting_add(&sig->ioac, &tsk->ioac);
sig->sum_sched_runtime += tsk->se.sum_exec_runtime;
sig->sum_sched_runtime += TASK_SUM_EXEC_RUNTIME(tsk);
sig->nr_threads--;
__unhash_process(tsk, group_dead);
write_sequnlock(&sig->stats_lock);

1
kernel/sched/Makefile
View File

@ -20,6 +20,7 @@ obj-y += core.o loadavg.o clock.o cputime.o
obj-y += idle_task.o fair.o rt.o deadline.o
obj-y += wait.o wait_bit.o swait.o completion.o idle.o
obj-$(CONFIG_SMP) += cpupri.o cpudeadline.o topology.o stop_task.o
obj-$(CONFIG_BT_SCHED) += batch.o bt_debug.o
obj-$(CONFIG_SCHED_AUTOGROUP) += autogroup.o
obj-$(CONFIG_SCHEDSTATS) += stats.o
obj-$(CONFIG_SCHED_DEBUG) += debug.o

1446
kernel/sched/batch.c Normal file
View File

File diff suppressed because it is too large Load Diff

83
kernel/sched/batch.h Normal file
View File

@ -0,0 +1,83 @@
/*
* Copyright (C) 2019 Tencent Ltd. All rights reserved.
*
* File Name ：batch.h
* Author ：
* Date ：2019-12-26
* Descriptor：
*/
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _BATCH_H
#define _BATCH_H
#include <linux/sched.h>
#include <uapi/linux/sched.h>
/* nflag of task_struct */
#define TNF_SCHED_BT 0x00000001
struct bt_rq {
struct load_weight load;
unsigned int nr_running, h_nr_running;
unsigned long nr_uninterruptible;
u64 exec_clock;
u64 min_vruntime;
#ifndef CONFIG_64BIT
u64 min_vruntime_copy;
#endif
struct rb_root_cached tasks_timeline;
struct rb_node *rb_leftmost;
/*
* 'curr' points to currently running entity on this bt_rq.
* It is set to NULL otherwise (i.e when none are currently running).
*/
struct sched_entity *curr, *next, *last, *skip;
#ifdef CONFIG_SCHED_DEBUG
unsigned int nr_spread_over;
#endif
#ifdef CONFIG_SMP
/*
* Load-tracking only depends on SMP, BT_GROUP_SCHED dependency below may be
* removed when useful for applications beyond shares distribution (e.g.
* load-balance).
*/
/*
* h_load = weight * f(tg)
*
* Where f(tg) is the recursive weight fraction assigned to
* this group.
*/
unsigned long h_load;
#endif /* CONFIG_SMP */
};
extern const struct sched_class bt_sched_class;
extern void init_bt_rq(struct bt_rq *bt_rq);
extern struct sched_entity *__pick_first_bt_entity(struct bt_rq *bt_rq);
extern struct sched_entity *__pick_last_bt_entity(struct bt_rq *bt_rq);
extern void set_bt_load_weight(struct task_struct *p);
static inline int bt_policy(int policy)
{
if (policy == SCHED_BT)
return 1;
return 0;
}
static inline int task_has_bt_policy(struct task_struct *p)
{
return bt_policy(p->policy);
}
#endif

72
kernel/sched/bt_debug.c Normal file
View File

@ -0,0 +1,72 @@
/*
* Copyright (C) 2019 Tencent Ltd. All rights reserved.
*
* File Name : bt_debug.c
* Author :
* Date : 2019-12-26
* Descriptor:
*/
#include <linux/proc_fs.h>
#include <linux/sched/mm.h>
#include <linux/sched/task.h>
#include <linux/seq_file.h>
#include <linux/kallsyms.h>
#include <linux/utsname.h>
#include <linux/mempolicy.h>
#include <linux/debugfs.h>
#include "sched.h"
#include "bt_debug.h"
void print_bt_rq(struct seq_file *m, int cpu, struct bt_rq *bt_rq)
{
s64 MIN_vruntime = -1, min_vruntime, max_vruntime = -1,
spread, rq0_min_vruntime, spread0;
struct rq *rq = cpu_rq(cpu);
struct sched_entity *last;
unsigned long flags;
SEQ_printf(m, "\nbt_rq[%d]:\n", cpu);
SEQ_printf(m, " .%-30s: %lld.%06ld\n", "exec_clock",
SPLIT_NS(bt_rq->exec_clock));
raw_spin_lock_irqsave(&rq->lock, flags);
if (bt_rq->rb_leftmost)
MIN_vruntime = (__pick_first_bt_entity(bt_rq))->vruntime;
last = __pick_last_bt_entity(bt_rq);
if (last)
max_vruntime = last->vruntime;
min_vruntime = bt_rq->min_vruntime;
rq0_min_vruntime = cpu_rq(0)->bt.min_vruntime;
raw_spin_unlock_irqrestore(&rq->lock, flags);
SEQ_printf(m, " .%-30s: %lld.%06ld\n", "MIN_vruntime",
SPLIT_NS(MIN_vruntime));
SEQ_printf(m, " .%-30s: %lld.%06ld\n", "min_vruntime",
SPLIT_NS(min_vruntime));
SEQ_printf(m, " .%-30s: %lld.%06ld\n", "max_vruntime",
SPLIT_NS(max_vruntime));
spread = max_vruntime - MIN_vruntime;
SEQ_printf(m, " .%-30s: %lld.%06ld\n", "spread",
SPLIT_NS(spread));
spread0 = min_vruntime - rq0_min_vruntime;
SEQ_printf(m, " .%-30s: %lld.%06ld\n", "spread0",
SPLIT_NS(spread0));
SEQ_printf(m, " .%-30s: %d\n", "nr_spread_over",
bt_rq->nr_spread_over);
SEQ_printf(m, " .%-30s: %d\n", "nr_running", bt_rq->nr_running);
SEQ_printf(m, " .%-30s: %ld\n", "load", bt_rq->load.weight);
}
#ifdef CONFIG_SCHED_DEBUG
void print_bt_stats(struct seq_file *m, int cpu)
{
struct bt_rq *bt_rq;
rcu_read_lock();
bt_rq = &cpu_rq(cpu)->bt;
print_bt_rq(m, cpu, bt_rq);
rcu_read_unlock();
}
#endif

36
kernel/sched/bt_debug.h Normal file
View File

@ -0,0 +1,36 @@
/*
* Copyright (C) 2019 Tencent Ltd. All rights reserved.
*
* File Name : bt_debug.h
* Author :
* Date : 2019-12-26
* Descriptor:
*/
#ifndef _BT_STAT_H
#define _BT_STAT_H
/*
* This allows printing both to /proc/sched_debug and
* to the console
*/
#define SEQ_printf(m, x...) \
do { \
if (m) \
seq_printf(m, x); \
else \
printk(x); \
} while (0)
extern long long nsec_high(unsigned long long nsec);
extern unsigned long nsec_low(unsigned long long nsec);
#define SPLIT_NS(x) (nsec_high(x), nsec_low(x))
extern void print_bt_stats(struct seq_file *m, int cpu);
#ifdef CONFIG_SCHED_DEBUG
extern void print_bt_stats(struct seq_file *m, int cpu);
#else
void print_bt_stats(struct seq_file *m, int cpu) {}
#endif
#endif

129
kernel/sched/core.c
View File

@ -10,6 +10,7 @@
#include <uapi/linux/sched/types.h>
#include <linux/sched/loadavg.h>
#include <linux/sched/hotplug.h>
#include <linux/sched/batch.h>
#include <linux/wait_bit.h>
#include <linux/cpuset.h>
#include <linux/delayacct.h>
@ -34,6 +35,7 @@
#endif
#include "sched.h"
#include "batch.h"
#include "../workqueue_internal.h"
#include "../smpboot.h"
@ -775,16 +777,38 @@ static inline void dequeue_task(struct rq *rq, struct task_struct *p, int flags)
void activate_task(struct rq *rq, struct task_struct *p, int flags)
{
#ifdef CONFIG_BT_SCHED
if (task_contributes_to_load(p)) {
rq->nr_uninterruptible--;
if (unlikely(p->flags & TNF_SCHED_BT)) {
p->flags &= ~TNF_SCHED_BT;
rq->bt.nr_uninterruptible--;
}
}
#else
if (task_contributes_to_load(p))
rq->nr_uninterruptible--;
#endif
enqueue_task(rq, p, flags);
}
void deactivate_task(struct rq *rq, struct task_struct *p, int flags)
{
#ifdef CONFIG_BT_SCHED
if (task_contributes_to_load(p)) {
rq->nr_uninterruptible++;
if (unlikely(p->sched_class == &bt_sched_class)) {
p->flags |= TNF_SCHED_BT;
rq->bt.nr_uninterruptible++;
}
}
#else
if (task_contributes_to_load(p))
rq->nr_uninterruptible++;
#endif
dequeue_task(rq, p, flags);
}
@ -1718,8 +1742,19 @@ ttwu_do_activate(struct rq *rq, struct task_struct *p, int wake_flags,
lockdep_assert_held(&rq->lock);
#ifdef CONFIG_SMP
#ifdef CONFIG_BT_SCHED
if (p->sched_contributes_to_load) {
rq->nr_uninterruptible--;
if (unlikely(p->flags & TNF_SCHED_BT)) {
p->flags &= ~TNF_SCHED_BT;
rq->bt.nr_uninterruptible--;
}
}
#else
if (p->sched_contributes_to_load)
rq->nr_uninterruptible--;
#endif
if (wake_flags & WF_MIGRATED)
en_flags |= ENQUEUE_MIGRATED;
@ -2183,6 +2218,15 @@ static void __sched_fork(unsigned long clone_flags, struct task_struct *p)
p->se.vruntime = 0;
INIT_LIST_HEAD(&p->se.group_node);
#ifdef CONFIG_BT_SCHED
p->bt.on_rq = 0;
p->bt.exec_start = 0;
p->bt.sum_exec_runtime = 0;
p->bt.prev_sum_exec_runtime = 0;
p->bt.nr_migrations = 0;
p->bt.vruntime = 0;
#endif
#ifdef CONFIG_FAIR_GROUP_SCHED
p->se.cfs_rq = NULL;
#endif
@ -2190,6 +2234,9 @@ static void __sched_fork(unsigned long clone_flags, struct task_struct *p)
#ifdef CONFIG_SCHEDSTATS
/* Even if schedstat is disabled, there should not be garbage */
memset(&p->se.statistics, 0, sizeof(p->se.statistics));
#ifdef CONFIG_BT_SCHED
p->bt.bt_statistics = &p->se.statistics;
#endif
#endif
RB_CLEAR_NODE(&p->dl.rb_node);
@ -2389,6 +2436,10 @@ int sched_fork(unsigned long clone_flags, struct task_struct *p)
return -EAGAIN;
} else if (rt_prio(p->prio)) {
p->sched_class = &rt_sched_class;
#ifdef CONFIG_BT_SCHED
} else if(bt_prio(p->prio)){
p->sched_class = &bt_sched_class;
#endif
} else {
p->sched_class = &fair_sched_class;
}
@ -3003,7 +3054,7 @@ unsigned long long task_sched_runtime(struct task_struct *p)
* been accounted, so we're correct here as well.
*/
if (!p->on_cpu || !task_on_rq_queued(p))
return p->se.sum_exec_runtime;
return TASK_SUM_EXEC_RUNTIME(p);
#endif
rq = task_rq_lock(p, &rf);
@ -3017,7 +3068,7 @@ unsigned long long task_sched_runtime(struct task_struct *p)
update_rq_clock(rq);
p->sched_class->update_curr(rq);
}
ns = p->se.sum_exec_runtime;
ns = TASK_SUM_EXEC_RUNTIME(p);
task_rq_unlock(rq, p, &rf);
return ns;
@ -3752,6 +3803,14 @@ void rt_mutex_setprio(struct task_struct *p, struct task_struct *pi_task)
if (oldprio < prio)
queue_flag |= ENQUEUE_HEAD;
p->sched_class = &rt_sched_class;
#ifdef CONFIG_BT_SCHED
} else if (bt_prio(prio)) {
if (dl_prio(oldprio))
p->dl.dl_boosted = 0;
if (rt_prio(oldprio))
p->rt.timeout = 0;
p->sched_class = &bt_sched_class;
#endif
} else {
if (dl_prio(oldprio))
p->dl.dl_boosted = 0;
@ -3816,8 +3875,17 @@ void set_user_nice(struct task_struct *p, long nice)
if (running)
put_prev_task(rq, p);
p->static_prio = NICE_TO_PRIO(nice);
set_load_weight(p);
#ifdef CONFIG_BT_SCHED
if (task_has_bt_policy(p)) {
p->static_prio = NICE_TO_BT_PRIO(nice);
set_bt_load_weight(p);
} else
#endif
{
p->static_prio = NICE_TO_PRIO(nice);
set_load_weight(p);
}
old_prio = p->prio;
p->prio = effective_prio(p);
delta = p->prio - old_prio;
@ -3900,6 +3968,29 @@ int task_prio(const struct task_struct *p)
return p->prio - MAX_RT_PRIO;
}
/**
* task_nice - return the nice value of a given task.
* @p: the task in question.
*
* Return: The nice value [ -20 ... 0 ... 19 ].
*/
inline int task_nice(const struct task_struct *p)
{
#ifdef CONFIG_BT_SCHED
int task_nice = 0;
if(bt_prio(p->static_prio))
task_nice = TASK_BT_NICE(p);
else
task_nice = PRIO_TO_NICE((p)->static_prio);
return task_nice;
#else
return PRIO_TO_NICE((p)->static_prio);
#endif
}
EXPORT_SYMBOL(task_nice);
/**
* idle_cpu - is a given CPU idle currently?
* @cpu: the processor in question.
@ -3967,6 +4058,15 @@ static void __setscheduler_params(struct task_struct *p,
else if (fair_policy(policy))
p->static_prio = NICE_TO_PRIO(attr->sched_nice);
#ifdef CONFIG_BT_SCHED
if (unlikely(policy == SCHED_BT)) {
bt_prio_adjust_pos(&p->static_prio);
set_bt_load_weight(p);
} else if (policy == SCHED_NORMAL || policy == SCHED_BATCH ||
policy == SCHED_IDLE) {
bt_prio_adjust_neg(&p->static_prio);
}
#endif
/*
* __sched_setscheduler() ensures attr->sched_priority == 0 when
* !rt_policy. Always setting this ensures that things like
@ -3995,6 +4095,10 @@ static void __setscheduler(struct rq *rq, struct task_struct *p,
p->sched_class = &dl_sched_class;
else if (rt_prio(p->prio))
p->sched_class = &rt_sched_class;
#ifdef CONFIG_BT_SCHED
else if (bt_prio(p->prio))
p->sched_class = &bt_sched_class;
#endif
else
p->sched_class = &fair_sched_class;
}
@ -4031,6 +4135,10 @@ static int __sched_setscheduler(struct task_struct *p,
/* The pi code expects interrupts enabled */
BUG_ON(pi && in_interrupt());
if(!sched_bt_on && SCHED_BT == policy)
return -EINVAL;
recheck:
/* Double check policy once rq lock held: */
if (policy < 0) {
@ -5257,6 +5365,12 @@ void init_idle(struct task_struct *idle, int cpu)
__sched_fork(0, idle);
idle->state = TASK_RUNNING;
idle->se.exec_start = sched_clock();
#ifdef CONFIG_BT_SCHED
idle->bt.exec_start = idle->se.exec_start;
#if defined(CONFIG_SCHEDSTATS) || defined(CONFIG_LATENCYTOP)
idle->bt.bt_statistics = &idle->se.statistics;
#endif
#endif
idle->flags |= PF_IDLE;
kasan_unpoison_task_stack(idle);
@ -5881,6 +5995,10 @@ void __init sched_init(void)
init_cfs_rq(&rq->cfs);
init_rt_rq(&rq->rt);
init_dl_rq(&rq->dl);
#ifdef CONFIG_BT_SCHED
rq->bt_nr_running = 0;
init_bt_rq(&rq->bt);
#endif
#ifdef CONFIG_FAIR_GROUP_SCHED
root_task_group.shares = ROOT_TASK_GROUP_LOAD;
INIT_LIST_HEAD(&rq->leaf_cfs_rq_list);
@ -6069,6 +6187,9 @@ void normalize_rt_tasks(void)
continue;
p->se.exec_start = 0;
#ifdef CONFIG_BT_SCHED
p->bt.exec_start = 0;
#endif
schedstat_set(p->se.statistics.wait_start, 0);
schedstat_set(p->se.statistics.sleep_start, 0);
schedstat_set(p->se.statistics.block_start, 0);

5
kernel/sched/cpupri.c
View File

@ -30,6 +30,7 @@
#include <linux/gfp.h>
#include <linux/sched.h>
#include <linux/sched/rt.h>
#include <linux/sched/batch.h>
#include <linux/slab.h>
#include "cpupri.h"
@ -40,7 +41,11 @@ static int convert_prio(int prio)
if (prio == CPUPRI_INVALID)
cpupri = CPUPRI_INVALID;
#ifdef CONFIG_BT_SCHED
else if (prio >= MAX_PRIO - BT_PRIO_WIDTH)
#else
else if (prio == MAX_PRIO)
#endif
cpupri = CPUPRI_IDLE;
else if (prio >= MAX_RT_PRIO)
cpupri = CPUPRI_NORMAL;

11
kernel/sched/cputime.c
View File

@ -108,6 +108,11 @@ static inline void task_group_account_field(struct task_struct *p, int index,
*
*/
__this_cpu_add(kernel_cpustat.cpustat[index], tmp);
#ifdef CONFIG_BT_SCHED
if(p->sched_class == &bt_sched_class) {
__this_cpu_add(kernel_cpustat.cpustat[CPUTIME_BT], tmp);
}
#endif
cpuacct_account_field(p, index, tmp);
}
@ -273,7 +278,7 @@ static inline u64 account_other_time(u64 max)
#ifdef CONFIG_64BIT
static inline u64 read_sum_exec_runtime(struct task_struct *t)
{
return t->se.sum_exec_runtime;
return TASK_SUM_EXEC_RUNTIME(t);
}
#else
static u64 read_sum_exec_runtime(struct task_struct *t)
@ -283,7 +288,7 @@ static u64 read_sum_exec_runtime(struct task_struct *t)
struct rq *rq;
rq = task_rq_lock(t, &rf);
ns = t->se.sum_exec_runtime;
ns = TASK_SUM_EXEC_RUNTIME(t);
task_rq_unlock(rq, t, &rf);
return ns;
@ -661,7 +666,7 @@ out:
void task_cputime_adjusted(struct task_struct *p, u64 *ut, u64 *st)
{
struct task_cputime cputime = {
.sum_exec_runtime = p->se.sum_exec_runtime,
.sum_exec_runtime = TASK_SUM_EXEC_RUNTIME(p),
};
task_cputime(p, &cputime.utime, &cputime.stime);

25
kernel/sched/debug.c
View File

@ -20,25 +20,14 @@
#include <linux/debugfs.h>
#include "sched.h"
#include "bt_debug.h"
static DEFINE_SPINLOCK(sched_debug_lock);
/*
* This allows printing both to /proc/sched_debug and
* to the console
*/
#define SEQ_printf(m, x...) \
do { \
if (m) \
seq_printf(m, x); \
else \
printk(x); \
} while (0)
/*
* Ease the printing of nsec fields:
*/
static long long nsec_high(unsigned long long nsec)
long long nsec_high(unsigned long long nsec)
{
if ((long long)nsec < 0) {
nsec = -nsec;
@ -50,7 +39,7 @@ static long long nsec_high(unsigned long long nsec)
return nsec;
}
static unsigned long nsec_low(unsigned long long nsec)
unsigned long nsec_low(unsigned long long nsec)
{
if ((long long)nsec < 0)
nsec = -nsec;
@ -58,8 +47,6 @@ static unsigned long nsec_low(unsigned long long nsec)
return do_div(nsec, 1000000);
}
#define SPLIT_NS(x) nsec_high(x), nsec_low(x)
#define SCHED_FEAT(name, enabled) \
#name ,
@ -705,6 +692,9 @@ do { \
print_cfs_stats(m, cpu);
print_rt_stats(m, cpu);
print_dl_stats(m, cpu);
#ifdef CONFIG_BT_SCHED
print_bt_stats(m, cpu);
#endif
print_rq(m, rq, cpu);
spin_unlock_irqrestore(&sched_debug_lock, flags);
@ -1004,6 +994,9 @@ void proc_sched_show_task(struct task_struct *p, struct pid_namespace *ns,
"nr_involuntary_switches", (long long)p->nivcsw);
P(se.load.weight);
#ifdef CONFIG_BT_SCHED
SEQ_printf(m, "%-45s:%21Ld\n", "bt.load.weight", (long long)scale_load_down(p->bt.load.weight));
#endif
#ifdef CONFIG_SMP
P(se.avg.load_sum);
P(se.avg.util_sum);

88
kernel/sched/fair.c
View File

@ -37,6 +37,10 @@
#include <trace/events/sched.h>
#include "sched.h"
#include "fair.h"
#ifdef CONFIG_BT_SCHED
#include "batch.h"
#endif
/*
* Targeted preemption latency for CPU-bound tasks:
@ -78,7 +82,7 @@ unsigned int normalized_sysctl_sched_min_granularity = 750000ULL;
/*
* This value is kept at sysctl_sched_latency/sysctl_sched_min_granularity
*/
static unsigned int sched_nr_latency = 8;
unsigned int sched_nr_latency = 8;
/*
* After fork, child runs first. If set to 0 (default) then
@ -132,19 +136,19 @@ unsigned int sysctl_sched_cfs_bandwidth_slice = 5000UL;
*/
unsigned int capacity_margin = 1280;
static inline void update_load_add(struct load_weight *lw, unsigned long inc)
inline void update_load_add(struct load_weight *lw, unsigned long inc)
{
lw->weight += inc;
lw->inv_weight = 0;
}
static inline void update_load_sub(struct load_weight *lw, unsigned long dec)
inline void update_load_sub(struct load_weight *lw, unsigned long dec)
{
lw->weight -= dec;
lw->inv_weight = 0;
}
static inline void update_load_set(struct load_weight *lw, unsigned long w)
inline void update_load_set(struct load_weight *lw, unsigned long w)
{
lw->weight = w;
lw->inv_weight = 0;
@ -180,7 +184,7 @@ static unsigned int get_update_sysctl_factor(void)
return factor;
}
static void update_sysctl(void)
void update_sysctl(void)
{
unsigned int factor = get_update_sysctl_factor();
@ -229,7 +233,7 @@ static void __update_inv_weight(struct load_weight *lw)
* Or, weight =< lw.weight (because lw.weight is the runqueue weight), thus
* weight/lw.weight <= 1, and therefore our shift will also be positive.
*/
static u64 __calc_delta(u64 delta_exec, unsigned long weight, struct load_weight *lw)
u64 __calc_delta(u64 delta_exec, unsigned long weight, struct load_weight *lw)
{
u64 fact = scale_load_down(weight);
int shift = WMULT_SHIFT;
@ -490,25 +494,6 @@ void account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec);
/**************************************************************
* Scheduling class tree data structure manipulation methods:
*/
static inline u64 max_vruntime(u64 max_vruntime, u64 vruntime)
{
s64 delta = (s64)(vruntime - max_vruntime);
if (delta > 0)
max_vruntime = vruntime;
return max_vruntime;
}
static inline u64 min_vruntime(u64 min_vruntime, u64 vruntime)
{
s64 delta = (s64)(vruntime - min_vruntime);
if (delta < 0)
min_vruntime = vruntime;
return min_vruntime;
}
static inline int entity_before(struct sched_entity *a,
struct sched_entity *b)
{
@ -663,7 +648,7 @@ static inline u64 calc_delta_fair(u64 delta, struct sched_entity *se)
*
* p = (nr <= nl) ? l : l*nr/nl
*/
static u64 __sched_period(unsigned long nr_running)
u64 __sched_period(unsigned long nr_running)
{
if (unlikely(nr_running > sched_nr_latency))
return nr_running * sysctl_sched_min_granularity;
@ -2637,7 +2622,7 @@ void task_tick_numa(struct rq *rq, struct task_struct *curr)
* task needs to have done some actual work before we bother with
* NUMA placement.
*/
now = curr->se.sum_exec_runtime;
now = TASK_SUM_EXEC_RUNTIME(curr);
period = (u64)curr->numa_scan_period * NSEC_PER_MSEC;
if (now > curr->node_stamp + period) {
@ -2653,7 +2638,7 @@ void task_tick_numa(struct rq *rq, struct task_struct *curr)
}
#else
static void task_tick_numa(struct rq *rq, struct task_struct *curr)
void task_tick_numa(struct rq *rq, struct task_struct *curr)
{
}
@ -3547,7 +3532,7 @@ static inline unsigned long cfs_rq_load_avg(struct cfs_rq *cfs_rq)
return cfs_rq->avg.load_avg;
}
static int idle_balance(struct rq *this_rq, struct rq_flags *rf);
int idle_balance(struct rq *this_rq, struct rq_flags *rf);
#else /* CONFIG_SMP */
@ -3576,7 +3561,7 @@ attach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) {}
static inline void
detach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) {}
static inline int idle_balance(struct rq *rq, struct rq_flags *rf)
inline int idle_balance(struct rq *rq, struct rq_flags *rf)
{
return 0;
}
@ -5327,7 +5312,11 @@ static unsigned long cpu_avg_load_per_task(int cpu)
struct rq *rq = cpu_rq(cpu);
unsigned long nr_running = READ_ONCE(rq->cfs.h_nr_running);
unsigned long load_avg = weighted_cpuload(rq);
#ifdef CONFIG_BT_SCHED
unsigned long bt_running = READ_ONCE(rq->bt.h_nr_running);
nr_running -= bt_running;
#endif
if (nr_running)
return load_avg / nr_running;
@ -6230,7 +6219,11 @@ static void check_preempt_wakeup(struct rq *rq, struct task_struct *p, int wake_
/* Idle tasks are by definition preempted by non-idle tasks. */
if (unlikely(curr->policy == SCHED_IDLE) &&
#ifdef CONFIG_BT_SCHED
likely(p->policy != SCHED_IDLE && p->policy != SCHED_BT))
#else
likely(p->policy != SCHED_IDLE))
#endif
goto preempt;
/*
@ -6279,11 +6272,17 @@ pick_next_task_fair(struct rq *rq, struct task_struct *prev, struct rq_flags *rf
struct cfs_rq *cfs_rq = &rq->cfs;
struct sched_entity *se;
struct task_struct *p;
#ifndef CONFIG_BT_SCHED
int new_tasks;
again:
#endif
if (!cfs_rq->nr_running)
#ifdef CONFIG_BT_SCHED
return NULL;
#else
goto idle;
#endif
#ifdef CONFIG_FAIR_GROUP_SCHED
if (prev->sched_class != &fair_sched_class)
@ -6322,7 +6321,11 @@ again:
cfs_rq = &rq->cfs;
if (!cfs_rq->nr_running)
#ifdef CONFIG_BT_SCHED
return NULL;
#else
goto idle;
#endif
goto simple;
}
@ -6381,7 +6384,7 @@ simple:
hrtick_start_fair(rq, p);
return p;
#ifndef CONFIG_BT_SCHED
idle:
new_tasks = idle_balance(rq, rf);
@ -6397,6 +6400,7 @@ idle:
goto again;
return NULL;
#endif
}
/*
@ -6427,7 +6431,7 @@ static void yield_task_fair(struct rq *rq)
/*
* Are we the only task in the tree?
*/
if (unlikely(rq->nr_running == 1))
if (unlikely(RQ_CFS_NR_RUNNING(rq) == 1))
return;
clear_buddies(cfs_rq, se);
@ -7430,7 +7434,7 @@ static inline void update_sg_lb_stats(struct lb_env *env,
sgs->group_util += cpu_util(i);
sgs->sum_nr_running += rq->cfs.h_nr_running;
nr_running = rq->nr_running;
nr_running = RQ_CFS_NR_RUNNING(rq);
if (nr_running > 1)
*overload = true;
@ -7975,7 +7979,7 @@ static struct rq *find_busiest_queue(struct lb_env *env,
* which is not scaled with the cpu capacity.
*/
if (rq->nr_running == 1 && wl > env->imbalance &&
if (RQ_CFS_NR_RUNNING(rq) == 1 && wl > env->imbalance &&
!check_cpu_capacity(rq, env->sd))
continue;
@ -8135,7 +8139,7 @@ redo:
env.src_rq = busiest;
ld_moved = 0;
if (busiest->nr_running > 1) {
if (RQ_CFS_NR_RUNNING(busiest) > 1) {
/*
* Attempt to move tasks. If find_busiest_group has found
* an imbalance but busiest->nr_running <= 1, the group is
@ -8143,7 +8147,7 @@ redo:
* correctly treated as an imbalance.
*/
env.flags |= LBF_ALL_PINNED;
env.loop_max = min(sysctl_sched_nr_migrate, busiest->nr_running);
env.loop_max = min(sysctl_sched_nr_migrate, RQ_CFS_NR_RUNNING(busiest));
more_balance:
rq_lock_irqsave(busiest, &rf);
@ -8377,7 +8381,7 @@ update_next_balance(struct sched_domain *sd, unsigned long *next_balance)
* idle_balance is called by schedule() if this_cpu is about to become
* idle. Attempts to pull tasks from other CPUs.
*/
static int idle_balance(struct rq *this_rq, struct rq_flags *rf)
int idle_balance(struct rq *this_rq, struct rq_flags *rf)
{
unsigned long next_balance = jiffies + HZ;
int this_cpu = this_rq->cpu;
@ -8518,7 +8522,7 @@ static int active_load_balance_cpu_stop(void *data)
goto out_unlock;
/* Is there any task to move? */
if (busiest_rq->nr_running <= 1)
if (RQ_CFS_NR_RUNNING(busiest_rq) <= 1)
goto out_unlock;
/*
@ -8916,7 +8920,11 @@ static inline bool nohz_kick_needed(struct rq *rq)
int nr_busy, i, cpu = rq->cpu;
bool kick = false;
#ifdef CONFIG_BT_SCHED
if (unlikely(rq->idle_balance && !rq->nr_running))
#else
if (unlikely(rq->idle_balance))
#endif
return false;
/*
@ -9521,7 +9529,11 @@ static unsigned int get_rr_interval_fair(struct rq *rq, struct task_struct *task
* All the scheduling class methods:
*/
const struct sched_class fair_sched_class = {
#ifdef CONFIG_BT_SCHED
.next = &bt_sched_class,
#else
.next = &idle_sched_class,
#endif
.enqueue_task = enqueue_task_fair,
.dequeue_task = dequeue_task_fair,
.yield_task = yield_task_fair,

47
kernel/sched/fair.h Normal file
View File

@ -0,0 +1,47 @@
/*
* Copyright (C) 2019 Tencent Ltd. All rights reserved.
*
* File Name ：fair.h
* Author ：
* Date ：2019-12-26
* Descriptor：
*/
#ifndef _FAIR_H
#define _FAIR_H
extern unsigned int sched_nr_latency;
unsigned long calc_delta_mine(unsigned long delta_exec,
unsigned long weight, struct load_weight *lw);
extern u64
__calc_delta(u64 delta_exec, unsigned long weight, struct load_weight *lw);
extern int idle_balance(struct rq *this_rq, struct rq_flags *rf);
static inline u64 max_vruntime(u64 max_vruntime, u64 vruntime)
{
s64 delta = (s64)(vruntime - max_vruntime);
if (delta > 0)
max_vruntime = vruntime;
return max_vruntime;
}
static inline u64 min_vruntime(u64 min_vruntime, u64 vruntime)
{
s64 delta = (s64)(vruntime - min_vruntime);
if (delta < 0)
min_vruntime = vruntime;
return min_vruntime;
}
u64 __sched_period(unsigned long nr_running);
void task_tick_numa(struct rq *rq, struct task_struct *curr);
void update_sysctl(void);
#endif

4
kernel/sched/loadavg.c
View File

@ -84,8 +84,8 @@ long calc_load_fold_active(struct rq *this_rq, long adjust)
{
long nr_active, delta = 0;
nr_active = this_rq->nr_running - adjust;
nr_active += (long)this_rq->nr_uninterruptible;
nr_active = RQ_CFS_NR_RUNNING(this_rq) - adjust;
nr_active += (long)RQ_CFS_NR_UNINTERRUPTIBLE(this_rq);
if (nr_active != this_rq->calc_load_active) {
delta = nr_active - this_rq->calc_load_active;

40
kernel/sched/sched.h
View File

@ -39,6 +39,7 @@
#include "cpupri.h"
#include "cpudeadline.h"
#include "cpuacct.h"
#include "batch.h"
#ifdef CONFIG_SCHED_DEBUG
# define SCHED_WARN_ON(x) WARN_ONCE(x, #x)
@ -114,6 +115,23 @@ static inline void cpu_load_update_active(struct rq *this_rq) { }
*/
#define DL_SCALE (10)
#ifdef CONFIG_BT_SCHED
#define RQ_CFS_NR_UNINTERRUPTIBLE(rq) \
((rq)->nr_uninterruptible - (rq)->bt.nr_uninterruptible)
#define RQ_CFS_NR_RUNNING(rq) \
((rq)->nr_running - (rq)->bt_nr_running)
#else
#define RQ_CFS_NR_UNINTERRUPTIBLE(rq) \
((rq)->nr_uninterruptible)
#define RQ_CFS_NR_RUNNING(rq) \
((rq)->nr_running)
#endif
/*
* These are the 'tuning knobs' of the scheduler:
*/
@ -141,9 +159,13 @@ static inline int dl_policy(int policy)
{
return policy == SCHED_DEADLINE;
}
static inline bool valid_policy(int policy)
{
return idle_policy(policy) || fair_policy(policy) ||
#ifdef CONFIG_BT_SCHED
bt_policy(policy) ||
#endif
rt_policy(policy) || dl_policy(policy);
}
@ -690,6 +712,9 @@ struct rq {
* remote CPUs use both these fields when doing load calculation.
*/
unsigned int nr_running;
#ifdef CONFIG_BT_SCHED
unsigned int bt_nr_running;
#endif
#ifdef CONFIG_NUMA_BALANCING
unsigned int nr_numa_running;
unsigned int nr_preferred_running;
@ -713,6 +738,9 @@ struct rq {
struct cfs_rq cfs;
struct rt_rq rt;
struct dl_rq dl;
#ifdef CONFIG_BT_SCHED
struct bt_rq bt;
#endif
#ifdef CONFIG_FAIR_GROUP_SCHED
/* list of leaf cfs_rq on this cpu: */
@ -1106,6 +1134,13 @@ struct sched_group {
unsigned long cpumask[0];
};
#define tsk_cpus_allowed(tsk) (&(tsk)->cpus_allowed)
static inline struct cpumask *sched_group_cpus(struct sched_group *sg)
{
return to_cpumask(sg->cpumask);
}
static inline struct cpumask *sched_group_span(struct sched_group *sg)
{
return to_cpumask(sg->cpumask);
@ -1357,6 +1392,9 @@ static inline void finish_lock_switch(struct rq *rq, struct task_struct *prev)
#define WF_FORK 0x02 /* child wakeup after fork */
#define WF_MIGRATED 0x4 /* internal use, task got migrated */
extern inline void update_load_add(struct load_weight *lw, unsigned long inc);
extern inline void update_load_sub(struct load_weight *lw, unsigned long dec);
extern inline void update_load_set(struct load_weight *lw, unsigned long w);
/*
* To aid in avoiding the subversion of "niceness" due to uneven distribution
* of tasks with abnormal "nice" values across CPUs the contribution that
@ -1601,7 +1639,7 @@ static inline void add_nr_running(struct rq *rq, unsigned count)
if (prev_nr < 2 && rq->nr_running >= 2) {
#ifdef CONFIG_SMP
if (!rq->rd->overload)
if (!rq->rd->overload && RQ_CFS_NR_RUNNING(rq) >= 2)
rq->rd->overload = true;
#endif
}

4
kernel/time/posix-cpu-timers.c
View File

@ -819,7 +819,7 @@ static void check_thread_timers(struct task_struct *tsk,
tsk_expires->virt_exp = expires;
tsk_expires->sched_exp = check_timers_list(++timers, firing,
tsk->se.sum_exec_runtime);
TASK_SUM_EXEC_RUNTIME(tsk));
/*
* Check for the special case thread timers.
@ -1082,7 +1082,7 @@ static inline int fastpath_timer_check(struct task_struct *tsk)
struct task_cputime task_sample;
task_cputime(tsk, &task_sample.utime, &task_sample.stime);
task_sample.sum_exec_runtime = tsk->se.sum_exec_runtime;
task_sample.sum_exec_runtime = TASK_SUM_EXEC_RUNTIME(tsk);
if (task_cputime_expired(&task_sample, &tsk->cputime_expires))
return 1;
}

1
package/default/config.default
View File

@ -184,6 +184,7 @@ CONFIG_IPC_NS=y
CONFIG_USER_NS=y
CONFIG_PID_NS=y
CONFIG_NET_NS=y
CONFIG_BT_SCHED=y
CONFIG_SCHED_AUTOGROUP=y
# CONFIG_SYSFS_DEPRECATED is not set
CONFIG_RELAY=y