From 9de8b674e21cab164ac71d0966a5b15ccfe1c8fa Mon Sep 17 00:00:00 2001 From: Xiaoming Gao Date: Tue, 21 Apr 2020 00:15:09 +0800 Subject: [PATCH] slub: add fast_slub_nr_free to track free objects use lightweight track method to count free objects in slub, remove heavy logic in spin_lock_irqsave. Signed-off-by: Xiaoming Gao --- include/linux/slub_def.h | 3 +++ kernel/sysctl.c | 9 +++++++++ mm/slub.c | 24 ++++++++++++++++++++++-- 3 files changed, 34 insertions(+), 2 deletions(-) diff --git a/include/linux/slub_def.h b/include/linux/slub_def.h index f8ced87a2..bb4cc628e 100644 --- a/include/linux/slub_def.h +++ b/include/linux/slub_def.h @@ -136,6 +136,9 @@ struct kmem_cache { struct kasan_cache kasan_info; #endif +#ifdef CONFIG_SLUB_DEBUG + atomic_long_t total_objects_free; +#endif struct kmem_cache_node *node[MAX_NUMNODES]; }; diff --git a/kernel/sysctl.c b/kernel/sysctl.c index b812bb89f..a7518dbc4 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -309,6 +309,8 @@ static int min_extfrag_threshold; static int max_extfrag_threshold = 1000; #endif +extern unsigned long sysctl_fast_slub_nr_free; + static int sysrq_use_leftctrl_sysctl_handler(struct ctl_table * table ,int write, void __user *buffer,size_t *lenp, loff_t *ppos) @@ -1810,6 +1812,13 @@ static struct ctl_table vm_table[] = { .extra2 = (void *)&mmap_rnd_compat_bits_max, }, #endif + { + .procname = "fast_slub_nr_free", + .data = &sysctl_fast_slub_nr_free, + .maxlen = sizeof(sysctl_fast_slub_nr_free), + .mode = 0644, + .proc_handler = proc_doulongvec_minmax, + }, { } }; diff --git a/mm/slub.c b/mm/slub.c index 220d42e59..3458aa9b4 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -1635,6 +1635,10 @@ out: inc_slabs_node(s, page_to_nid(page), page->objects); +#ifdef CONFIG_SLUB_DEBUG + atomic_long_add(page->objects, &s->total_objects_free); +#endif + return page; } @@ -1718,6 +1722,9 @@ static void free_slab(struct kmem_cache *s, struct page *page) static void discard_slab(struct kmem_cache *s, struct page *page) { +#ifdef CONFIG_SLUB_DEBUG + atomic_long_sub(page->objects, &s->total_objects_free); +#endif dec_slabs_node(s, page_to_nid(page), page->objects); free_slab(s, page); } @@ -2722,6 +2729,9 @@ redo: slab_post_alloc_hook(s, gfpflags, 1, &object); +#ifdef CONFIG_SLUB_DEBUG + atomic_long_dec(&s->total_objects_free); +#endif return object; } @@ -2957,6 +2967,9 @@ redo: } else __slab_free(s, page, head, tail_obj, cnt, addr); +#ifdef CONFIG_SLUB_DEBUG + atomic_long_add(cnt, &s->total_objects_free); +#endif } static __always_inline void slab_free(struct kmem_cache *s, struct page *page, @@ -3385,6 +3398,9 @@ static int init_kmem_cache_nodes(struct kmem_cache *s) { int node; +#ifdef CONFIG_SLUB_DEBUG + atomic_long_set(&s->total_objects_free, 0); +#endif for_each_node_state(node, N_NORMAL_MEMORY) { struct kmem_cache_node *n; @@ -5837,6 +5853,7 @@ __initcall(slab_sysfs_init); * The /proc/slabinfo ABI */ #ifdef CONFIG_SLABINFO +unsigned long sysctl_fast_slub_nr_free = 0; void get_slabinfo(struct kmem_cache *s, struct slabinfo *sinfo) { unsigned long nr_slabs = 0; @@ -5848,9 +5865,12 @@ void get_slabinfo(struct kmem_cache *s, struct slabinfo *sinfo) for_each_kmem_cache_node(s, node, n) { nr_slabs += node_nr_slabs(n); nr_objs += node_nr_objs(n); - nr_free += count_partial(n, count_free); + if (!sysctl_fast_slub_nr_free) + nr_free += count_partial(n, count_free); } - + if (sysctl_fast_slub_nr_free) + nr_free = atomic_long_read(&s->total_objects_free); + nr_free = min(nr_objs, nr_free); sinfo->active_objs = nr_objs - nr_free; sinfo->num_objs = nr_objs; sinfo->active_slabs = nr_slabs;