From f7860a5c8675bc3da5916960fc58e0e1235126ff Mon Sep 17 00:00:00 2001
From: charliecgxu <charliecgxu@tencent.com>
Date: Fri, 16 Oct 2020 11:55:22 +0800
Subject: [PATCH] softirq: enhance network latency

Enhance network tail latency by directly handling softirq
when detecting long scheduling latency of ksoftirqd.

Signed-off-by: yongduan <yongduan@tencent.com>
Signed-off-by: charliecgxu <charliecgxu@tencent.com>
---
 kernel/softirq.c | 30 ++++++++++++++++++++++++++++++
 kernel/sysctl.c  | 20 ++++++++++++++++++++
 2 files changed, 50 insertions(+)

diff --git a/kernel/softirq.c b/kernel/softirq.c
index a4c87cf27..cd9c197d1 100644
--- a/kernel/softirq.c
+++ b/kernel/softirq.c
@@ -26,6 +26,8 @@
 #include <linux/smpboot.h>
 #include <linux/tick.h>
 #include <linux/irq.h>
+#include <linux/sched/stat.h>
+#include <linux/sched/clock.h>
 
 #define CREATE_TRACE_POINTS
 #include <trace/events/irq.h>
@@ -77,6 +79,32 @@ static void wakeup_softirqd(void)
 		wake_up_process(tsk);
 }
 
+unsigned int sysctl_softirq_accel_target = 2 * 1000 * 1000; //2ms
+int sysctl_softirq_accel_mask;
+int min_softirq_accel_mask;
+int max_softirq_accel_mask = (1 << NR_SOFTIRQS) - 1;
+static bool need_softirq_accel(struct task_struct *tsk, unsigned long pending)
+{
+#ifdef CONFIG_SCHED_INFO
+	if (tsk && tsk->state == TASK_RUNNING &&
+	    (pending & sysctl_softirq_accel_mask)) {
+		u64 delta = sched_clock_cpu(smp_processor_id());
+
+		if (!sched_info_on() || current == tsk ||
+		    !tsk->sched_info.last_queued ||
+		    delta <= tsk->sched_info.last_queued)
+			return false;
+
+		delta -= tsk->sched_info.last_queued;
+		if (delta >= sysctl_softirq_accel_target) {
+			tsk->sched_info.last_queued += delta;
+			return true;
+		}
+	}
+#endif
+	return false;
+}
+
 /*
  * If ksoftirqd is scheduled, we do not want to process pending softirqs
  * right now. Let ksoftirqd handle this at its own rate, to get fairness,
@@ -89,6 +117,8 @@ static bool ksoftirqd_running(unsigned long pending)
 
 	if (pending & SOFTIRQ_NOW_MASK)
 		return false;
+	if (sysctl_softirq_accel_mask && need_softirq_accel(tsk, pending))
+		return false;
 	return tsk && (tsk->state == TASK_RUNNING);
 }
 
diff --git a/kernel/sysctl.c b/kernel/sysctl.c
index 8389c0d75..1b1bfd04c 100644
--- a/kernel/sysctl.c
+++ b/kernel/sysctl.c
@@ -334,6 +334,10 @@ static int sysrq_use_leftctrl_sysctl_handler(struct ctl_table * table ,int write
 extern int sysctl_min_epoll_wait_time;
 extern int sysctl_clocksource_switch_unstable_cs;
 extern int sysctl_clocksource_unstable_cnt;
+extern unsigned int sysctl_softirq_accel_target;
+extern int sysctl_softirq_accel_mask;
+extern int min_softirq_accel_mask;
+extern int max_softirq_accel_mask;
 extern unsigned int sysctl_memcg_stat_show_subtree;
 
 unsigned int sysctl_cgroup_stats_isolated = 0;
@@ -365,6 +369,22 @@ static struct ctl_table kern_table[] = {
 		.mode		= 0644,
 		.proc_handler	= proc_dointvec,
 	},
+	{
+		.procname	= "softirq_accel_target_us",
+		.data		= &sysctl_softirq_accel_target,
+		.maxlen		= sizeof(unsigned int),
+		.mode		= 0644,
+		.proc_handler	= proc_dointvec,
+	},
+	{
+		.procname	= "softirq_accel_mask",
+		.data		= &sysctl_softirq_accel_mask,
+		.maxlen		= sizeof(int),
+		.mode		= 0644,
+		.proc_handler	= proc_dointvec_minmax,
+		.extra1		= &min_softirq_accel_mask,
+		.extra2		= &max_softirq_accel_mask,
+	},
 	{
 		.procname	= "memcg_stat_show_subtree",
 		.data		= &sysctl_memcg_stat_show_subtree,