diff --git a/init/Kconfig b/init/Kconfig index b33a595f2..d1ef575e5 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -1053,6 +1053,17 @@ config SOCK_CGROUP_DATA bool default n +config CGROUP_SLI + bool "Service level indicator" + depends on MEMCG + default n + help + This option enables the SLI feature for cgroups. SLI is a + lightweight and high efficient mechanism that could help + users to monitor the cgroup's performance. And we can + combine SLI with mbuf, then we can acquire the detail + stack information that figure out who cause this interference. + endif # CGROUPS menuconfig NAMESPACES diff --git a/kernel/cgroup/Makefile b/kernel/cgroup/Makefile index cc3765e92..f5106c82b 100644 --- a/kernel/cgroup/Makefile +++ b/kernel/cgroup/Makefile @@ -6,4 +6,5 @@ obj-$(CONFIG_CGROUP_PIDS) += pids.o obj-$(CONFIG_CGROUP_RDMA) += rdma.o obj-$(CONFIG_CPUSETS) += cpuset.o obj-$(CONFIG_CGROUP_DEBUG) += debug.o -obj-y += mbuf.o sli.o +obj-$(CONFIG_CGROUP_SLI) += sli.o +obj-y += mbuf.o diff --git a/kernel/cgroup/cgroup.c b/kernel/cgroup/cgroup.c index e921fc309..66b2ad931 100644 --- a/kernel/cgroup/cgroup.c +++ b/kernel/cgroup/cgroup.c @@ -3702,6 +3702,7 @@ static void cgroup_pressure_release(struct kernfs_open_file *of) } #endif /* CONFIG_PSI */ +#ifdef CONFIG_CGROUP_SLI static int cgroup_sli_memory_show(struct seq_file *seq, void *v) { struct cgroup *cgroup = seq_css(seq)->cgroup; @@ -3724,6 +3725,7 @@ static int cgroup_sli_max_show(struct seq_file *seq, void *v) sli_schedlat_max_show(seq, cgroup); return sli_memlat_max_show(seq, cgroup); } +#endif void *cgroup_mbuf_start(struct seq_file *s, loff_t *pos) { @@ -5046,6 +5048,7 @@ out_unlock: return ret ?: nbytes; } +#ifdef CONFIG_CGROUP_SLI int cgroup_sli_monitor_open(struct kernfs_open_file *of) { return sli_monitor_open(of); @@ -5075,6 +5078,7 @@ __poll_t cgroup_sli_monitor_poll(struct kernfs_open_file *of, poll_table *pt) { return sli_monitor_poll(of, pt); } +#endif /* cgroup core interface files for the default hierarchy */ static struct cftype cgroup_base_files[] = { @@ -5175,6 +5179,7 @@ static struct cftype cgroup_base_files[] = { .seq_next = cgroup_mbuf_next, .seq_stop = cgroup_mbuf_stop, }, +#ifdef CONFIG_CGROUP_SLI { .name = "sli.memory", .flags = CFTYPE_NOT_ON_ROOT, @@ -5205,6 +5210,7 @@ static struct cftype cgroup_base_files[] = { .seq_stop = cgroup_sli_monitor_stop, .poll = cgroup_sli_monitor_poll, }, +#endif { } /* terminate */ }; @@ -5266,7 +5272,9 @@ static void css_free_rwork_fn(struct work_struct *work) cgroup_put(cgroup_parent(cgrp)); kernfs_put(cgrp->kn); psi_cgroup_free(cgrp); +#ifdef CONFIG_CGROUP_SLI sli_cgroup_free(cgrp); +#endif if (cgroup_on_dfl(cgrp)) cgroup_rstat_exit(cgrp); kfree(cgrp); @@ -5712,15 +5720,19 @@ int cgroup_mkdir(struct kernfs_node *parent_kn, const char *name, umode_t mode) if (ret) goto out_destroy; +#ifdef CONFIG_CGROUP_SLI ret = sli_cgroup_alloc(cgrp); if (ret) goto out_destroy; +#endif if(sysctl_qos_mbuf_enable && cgroup_need_mbuf(cgrp)) cgrp->mbuf = mbuf_slot_alloc(cgrp); - if (cgroup_need_mbuf(cgrp)) +#ifdef CONFIG_CGROUP_SLI + if (cgroup_need_sli(cgrp)) cgrp->sctx = sctx_alloc(); +#endif TRACE_CGROUP_PATH(mkdir, cgrp); @@ -5907,8 +5919,10 @@ static int cgroup_destroy_locked(struct cgroup *cgrp) if (cgrp->mbuf) mbuf_free(cgrp); +#ifdef CONFIG_CGROUP_SLI if (cgrp->sctx) sctx_free(cgrp); +#endif /* put the base reference */ percpu_ref_kill(&cgrp->self.refcnt); diff --git a/kernel/cgroup/sli.c b/kernel/cgroup/sli.c index cd1efde52..42f7a2af8 100755 --- a/kernel/cgroup/sli.c +++ b/kernel/cgroup/sli.c @@ -642,13 +642,16 @@ void sli_update_tick(struct task_struct *tsk) { struct cgroup *cgrp; - if (!static_branch_likely(&sli_monitor_enabled)) + if (!static_branch_likely(&sli_enabled)) return; #ifdef CONFIG_SCHED_INFO sli_check_longsys(tsk); #endif + if (!static_branch_likely(&sli_monitor_enabled)) + return; + rcu_read_lock(); cgrp = get_cgroup_from_task(tsk); diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 6aa406d37..28699c824 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -3705,7 +3705,9 @@ void scheduler_tick(void) rq->idle_balance = idle_cpu(cpu); trigger_load_balance(rq); #endif +#ifdef CONFIG_CGROUP_SLI sli_update_tick(curr); +#endif } #ifdef CONFIG_NO_HZ_FULL @@ -3799,7 +3801,9 @@ out_requeue: if (os == TICK_SCHED_REMOTE_RUNNING) queue_delayed_work(system_unbound_wq, dwork, HZ); +#ifdef CONFIG_CGROUP_SLI sli_update_tick(curr); +#endif } static void sched_tick_start(int cpu) diff --git a/kernel/sched/cpuacct.c b/kernel/sched/cpuacct.c index 88975bef1..62aee7848 100644 --- a/kernel/sched/cpuacct.c +++ b/kernel/sched/cpuacct.c @@ -323,6 +323,7 @@ static int cpuacct_uptime_show_comm(struct seq_file *sf, void *v, struct cpuacct return 0; } +#ifdef CONFIG_CGROUP_SLI static int cpuacct_sli_show(struct seq_file *sf, void *v) { struct cgroup *cgrp = seq_css(sf)->cgroup; @@ -336,6 +337,7 @@ static int cpuacct_sli_max_show(struct seq_file *sf, void *v) return sli_schedlat_max_show(sf, cgrp); } +#endif int cpuacct_cgroupfs_uptime_show(struct seq_file *m, void *v) { @@ -411,6 +413,7 @@ static struct cftype files[] = { .seq_next = cgroup_mbuf_next, .seq_stop = cgroup_mbuf_stop, }, +#ifdef CONFIG_CGROUP_SLI { .name = "sli", .flags = CFTYPE_NOT_ON_ROOT, @@ -436,6 +439,7 @@ static struct cftype files[] = { .seq_stop = cgroup_sli_monitor_stop, .poll = cgroup_sli_monitor_poll, }, +#endif { } /* terminate */ }; diff --git a/kernel/sched/cputime.c b/kernel/sched/cputime.c index 97de71864..a4a7e0394 100644 --- a/kernel/sched/cputime.c +++ b/kernel/sched/cputime.c @@ -109,9 +109,11 @@ static inline void task_group_account_field(struct task_struct *p, int index, cgroup_account_cputime_field(p, index, tmp); +#ifdef CONFIG_CGROUP_SLI /* Collect the irq cputime for cgroup(used for sli) */ if (index == CPUTIME_SOFTIRQ || index == CPUTIME_IRQ) sli_schedlat_stat(p, SCHEDLAT_IRQTIME, tmp); +#endif } /* diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 8d3377d76..f3118abaf 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -917,7 +917,9 @@ update_stats_wait_end(struct cfs_rq *cfs_rq, struct sched_entity *se) return; } trace_sched_stat_wait(p, delta); +#ifdef CONFIG_CGROUP_SLI sli_schedlat_stat(p,SCHEDLAT_WAIT,delta); +#endif } __schedstat_set(se->statistics.wait_max, @@ -955,7 +957,9 @@ update_stats_enqueue_sleeper(struct cfs_rq *cfs_rq, struct sched_entity *se) __schedstat_add(se->statistics.sum_sleep_runtime, delta); if (tsk) { +#ifdef CONFIG_CGROUP_SLI sli_schedlat_stat(tsk,SCHEDLAT_SLEEP,delta); +#endif account_scheduler_latency(tsk, delta >> 10, 1); trace_sched_stat_sleep(tsk, delta); } @@ -974,13 +978,19 @@ update_stats_enqueue_sleeper(struct cfs_rq *cfs_rq, struct sched_entity *se) if (tsk) { if (tsk->in_iowait) { +#ifdef CONFIG_CGROUP_SLI sli_schedlat_stat(tsk,SCHEDLAT_IOBLOCK,delta); +#endif __schedstat_add(se->statistics.iowait_sum, delta); __schedstat_inc(se->statistics.iowait_count); trace_sched_stat_iowait(tsk, delta); +#ifdef CONFIG_CGROUP_SLI } else { sli_schedlat_stat(tsk,SCHEDLAT_BLOCK,delta); } +#else + } +#endif trace_sched_stat_blocked(tsk, delta); diff --git a/kernel/sched/stats.h b/kernel/sched/stats.h index 4543af7b1..24ffe7b8a 100644 --- a/kernel/sched/stats.h +++ b/kernel/sched/stats.h @@ -184,7 +184,9 @@ static void sched_info_arrive(struct rq *rq, struct task_struct *t, struct task_ t->sched_info.last_arrival = now; t->sched_info.pcount++; rq_sched_info_arrive(rq, delta); +#ifdef CONFIG_CGROUP_SLI sli_schedlat_rundelay(t, prev, delta); +#endif } /* diff --git a/mm/memcontrol.c b/mm/memcontrol.c index ae8de0572..d0d026f61 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -2565,12 +2565,16 @@ void mem_cgroup_handle_over_high(void) unsigned long pflags; unsigned int nr_pages = current->memcg_nr_pages_over_high; struct mem_cgroup *memcg; +#ifdef CONFIG_CGROUP_SLI u64 start; +#endif if (likely(!nr_pages)) return; +#ifdef CONFIG_CGROUP_SLI sli_memlat_stat_start(&start); +#endif memcg = get_mem_cgroup_from_mm(current->mm); reclaim_high(memcg, nr_pages, GFP_KERNEL); current->memcg_nr_pages_over_high = 0; @@ -2600,7 +2604,9 @@ void mem_cgroup_handle_over_high(void) psi_memstall_leave(&pflags); out: +#ifdef CONFIG_CGROUP_SLI sli_memlat_stat_end(MEM_LAT_MEMCG_DIRECT_RECLAIM, start); +#endif css_put(&memcg->css); } @@ -2615,7 +2621,9 @@ static int try_charge(struct mem_cgroup *memcg, gfp_t gfp_mask, bool may_swap = true; bool drained = false; enum oom_status oom_status; +#ifdef CONFIG_CGROUP_SLI u64 start; +#endif if (mem_cgroup_is_root(memcg)) return 0; @@ -2675,11 +2683,15 @@ retry: memcg_memory_event(mem_over_limit, MEMCG_MAX); +#ifdef CONFIG_CGROUP_SLI sli_memlat_stat_start(&start); +#endif nr_reclaimed = try_to_free_mem_cgroup_pages(mem_over_limit, nr_pages, gfp_mask, may_swap); +#ifdef CONFIG_CGROUP_SLI sli_memlat_stat_end(MEM_LAT_MEMCG_DIRECT_RECLAIM, start); +#endif if (mem_cgroup_margin(mem_over_limit) >= nr_pages) goto retry; @@ -5468,6 +5480,7 @@ static int mem_cgroup_bind_blkio_show(struct seq_file *m, void *v) return 0; } +#ifdef CONFIG_CGROUP_SLI static int mem_cgroup_sli_max_show(struct seq_file *m, void *v) { struct mem_cgroup *memcg = mem_cgroup_from_seq(m); @@ -5485,6 +5498,7 @@ static int mem_cgroup_sli_show(struct seq_file *m, void *v) return sli_memlat_stat_show(m, cgrp); } +#endif int mem_cgroupfs_vmstat_show(struct seq_file *m, void *v) { @@ -5664,6 +5678,7 @@ static struct cftype mem_cgroup_legacy_files[] = { .write = mem_cgroup_reset, .read_u64 = mem_cgroup_read_u64, }, +#ifdef CONFIG_CGROUP_SLI { .name = "sli", .flags = CFTYPE_NOT_ON_ROOT, @@ -5674,6 +5689,7 @@ static struct cftype mem_cgroup_legacy_files[] = { .flags = CFTYPE_NOT_ON_ROOT, .seq_show = mem_cgroup_sli_max_show, }, +#endif { .name = "bind_blkio", .flags = CFTYPE_NOT_ON_ROOT, diff --git a/mm/memory.c b/mm/memory.c index 465c6b6bb..c5e09882f 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -3963,9 +3963,13 @@ static vm_fault_t handle_pte_fault(struct vm_fault *vmf) vm_fault_t retval; u64 start; +#ifdef CONFIG_CGROUP_SLI sli_memlat_stat_start(&start); +#endif retval = do_swap_page(vmf); +#ifdef CONFIG_CGROUP_SLI sli_memlat_stat_end(MEM_LAT_DIRECT_SWAPIN, start); +#endif return retval; } diff --git a/mm/page_alloc.c b/mm/page_alloc.c index e48e8c69c..912f053a9 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -3909,19 +3909,25 @@ __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order, struct page *page = NULL; unsigned long pflags; unsigned int noreclaim_flag; +#ifdef CONFIG_CGROUP_SLI u64 start; +#endif if (!order) return NULL; psi_memstall_enter(&pflags); +#ifdef CONFIG_CGROUP_SLI sli_memlat_stat_start(&start); +#endif noreclaim_flag = memalloc_noreclaim_save(); *compact_result = try_to_compact_pages(gfp_mask, order, alloc_flags, ac, prio, &page); memalloc_noreclaim_restore(noreclaim_flag); +#ifdef CONFIG_CGROUP_SLI sli_memlat_stat_end(MEM_LAT_DIRECT_COMPACT, start); +#endif psi_memstall_leave(&pflags); /* @@ -4132,14 +4138,18 @@ __perform_reclaim(gfp_t gfp_mask, unsigned int order, int progress; unsigned int noreclaim_flag; unsigned long pflags; +#ifdef CONFIG_CGROUP_SLI u64 start; +#endif cond_resched(); /* We now go into synchronous reclaim */ cpuset_memory_pressure_bump(); psi_memstall_enter(&pflags); +#ifdef CONFIG_CGROUP_SLI sli_memlat_stat_start(&start); +#endif fs_reclaim_acquire(gfp_mask); noreclaim_flag = memalloc_noreclaim_save(); @@ -4148,7 +4158,9 @@ __perform_reclaim(gfp_t gfp_mask, unsigned int order, memalloc_noreclaim_restore(noreclaim_flag); fs_reclaim_release(gfp_mask); +#ifdef CONFIG_CGROUP_SLI sli_memlat_stat_end(MEM_LAT_GLOBAL_DIRECT_RECLAIM, start); +#endif psi_memstall_leave(&pflags); cond_resched(); diff --git a/mm/shmem.c b/mm/shmem.c index 9699ecd1a..4df06bfdc 100644 --- a/mm/shmem.c +++ b/mm/shmem.c @@ -1650,7 +1650,9 @@ static int shmem_swapin_page(struct inode *inode, pgoff_t index, struct page *page; swp_entry_t swap; int error; +#ifdef CONFIG_CGROUP_SLI u64 start; +#endif VM_BUG_ON(!*pagep || !xa_is_value(*pagep)); swap = radix_to_swp_entry(*pagep); @@ -1666,9 +1668,13 @@ static int shmem_swapin_page(struct inode *inode, pgoff_t index, count_memcg_event_mm(charge_mm, PGMAJFAULT); } /* Here we actually start the io */ +#ifdef CONFIG_CGROUP_SLI sli_memlat_stat_start(&start); +#endif page = shmem_swapin(swap, gfp, info, index); +#ifdef CONFIG_CGROUP_SLI sli_memlat_stat_end(MEM_LAT_DIRECT_SWAPIN, start); +#endif if (!page) { error = -ENOMEM; goto failed; diff --git a/mm/vmscan.c b/mm/vmscan.c index 284ba8834..b8e4e8213 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -840,7 +840,9 @@ typedef enum { static pageout_t pageout(struct page *page, struct address_space *mapping, struct scan_control *sc) { +#ifdef CONFIG_CGROUP_SLI u64 start; +#endif /* * If the page is dirty, only perform writeback if that write * will be non-blocking. To prevent this allocation from being @@ -889,14 +891,18 @@ static pageout_t pageout(struct page *page, struct address_space *mapping, }; SetPageReclaim(page); +#ifdef CONFIG_CGROUP_SLI if (!current_is_kswapd()) sli_memlat_stat_start(&start); +#endif res = mapping->a_ops->writepage(page, &wbc); +#ifdef CONFIG_CGROUP_SLI if (!current_is_kswapd()) sli_memlat_stat_end(global_reclaim(sc) ? MEM_LAT_GLOBAL_DIRECT_SWAPOUT : MEM_LAT_MEMCG_DIRECT_SWAPOUT, start); +#endif if (res < 0) handle_write_error(mapping, page, res); if (res == AOP_WRITEPAGE_ACTIVATE) { diff --git a/package/arm/config.default b/package/arm/config.default index 155a915c3..95cb5fdcd 100644 --- a/package/arm/config.default +++ b/package/arm/config.default @@ -95,6 +95,7 @@ CONFIG_TASK_XACCT=y CONFIG_TASK_IO_ACCOUNTING=y CONFIG_PSI=y CONFIG_PSI_DEFAULT_DISABLED=y +CONFIG_CGROUP_SLI=y # end of CPU/Task time and stats accounting CONFIG_CPU_ISOLATION=y diff --git a/package/default/config.default b/package/default/config.default index 9b1b5feb8..504771275 100644 --- a/package/default/config.default +++ b/package/default/config.default @@ -109,6 +109,7 @@ CONFIG_TASK_XACCT=y CONFIG_TASK_IO_ACCOUNTING=y CONFIG_PSI=y CONFIG_PSI_DEFAULT_DISABLED=y +CONFIG_CGROUP_SLI=y # end of CPU/Task time and stats accounting CONFIG_CPU_ISOLATION=y