diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index 79ed5c442..09dea065b 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -47,6 +47,8 @@ enum memcg_memory_event { MEMCG_OOM_KILL, MEMCG_SWAP_MAX, MEMCG_SWAP_FAIL, + MEMCG_PAGECACHE_MAX, + MEMCG_PAGECACHE_OOM, MEMCG_NR_MEMORY_EVENTS, }; @@ -214,6 +216,9 @@ struct mem_cgroup { /* Accounted resources */ struct page_counter memory; struct page_counter swap; + struct page_counter pagecache; + u64 pagecache_reclaim_ratio; + u32 pagecache_max_ratio; /* Legacy consumer-oriented counters */ struct page_counter memsw; @@ -345,6 +350,21 @@ struct mem_cgroup { */ #define MEMCG_CHARGE_BATCH 32U +/* + * Iteration constructs for visiting all cgroups (under a tree). If + * loops are exited prematurely (break), mem_cgroup_iter_break() must + * be used for reference counting. + */ +#define for_each_mem_cgroup_tree(iter, root) \ + for (iter = mem_cgroup_iter(root, NULL, NULL); \ + iter != NULL; \ + iter = mem_cgroup_iter(root, iter, NULL)) + +#define for_each_mem_cgroup(iter) \ + for (iter = mem_cgroup_iter(NULL, NULL, NULL); \ + iter != NULL; \ + iter = mem_cgroup_iter(NULL, iter, NULL)) + extern struct mem_cgroup *root_mem_cgroup; static inline bool mem_cgroup_is_root(struct mem_cgroup *memcg) @@ -1337,6 +1357,10 @@ struct sock; bool mem_cgroup_charge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages); void mem_cgroup_uncharge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages); #ifdef CONFIG_MEMCG + +extern unsigned int vm_pagecache_limit_retry_times __read_mostly; +extern void mem_cgroup_shrink_pagecache(struct mem_cgroup *memcg, gfp_t gfp_mask); + extern struct static_key_false memcg_sockets_enabled_key; #define mem_cgroup_sockets_enabled static_branch_unlikely(&memcg_sockets_enabled_key) void mem_cgroup_sk_alloc(struct sock *sk); diff --git a/include/linux/swap.h b/include/linux/swap.h index 115e437c0..b9561372e 100644 --- a/include/linux/swap.h +++ b/include/linux/swap.h @@ -370,6 +370,7 @@ extern int vm_swappiness; #define ADDITIONAL_RECLAIM_RATIO 2 extern unsigned long pagecache_over_limit(void); extern void shrink_page_cache(gfp_t mask, struct page *page); +extern unsigned long shrink_page_cache_memcg(gfp_t mask, struct mem_cgroup *memcg, unsigned long nr_pages); extern unsigned long vm_pagecache_limit_pages; extern unsigned long vm_pagecache_limit_reclaim_pages; extern int vm_pagecache_limit_ratio; @@ -379,6 +380,7 @@ extern unsigned int vm_pagecache_limit_async; extern unsigned int vm_pagecache_ignore_slab; extern int kpagecache_limitd_run(void); extern void kpagecache_limitd_stop(void); +extern unsigned int vm_pagecache_limit_global; extern int remove_mapping(struct address_space *mapping, struct page *page); extern unsigned long vm_total_pages; diff --git a/kernel/sysctl.c b/kernel/sysctl.c index 505fea4e2..de69ac536 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -1620,6 +1620,24 @@ static struct ctl_table vm_table[] = { .mode = 0644, .proc_handler = &proc_dointvec, }, + { + .procname = "pagecache_limit_global", + .data = &vm_pagecache_limit_global, + .maxlen = sizeof(vm_pagecache_limit_global), + .mode = 0644, + .proc_handler = &proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE, + }, + { + .procname = "pagecache_limit_retry_times", + .data = &vm_pagecache_limit_retry_times, + .maxlen = sizeof(vm_pagecache_limit_retry_times), + .mode = 0644, + .proc_handler = &proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = &one_hundred, + }, #ifdef CONFIG_HUGETLB_PAGE { .procname = "nr_hugepages", diff --git a/mm/filemap.c b/mm/filemap.c index 424bb59a8..4a830bc94 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -198,7 +198,6 @@ static void unaccount_page_cache_page(struct address_space *mapping, return; nr = hpage_nr_pages(page); - __mod_node_page_state(page_pgdat(page), NR_FILE_PAGES, -nr); if (PageSwapBacked(page)) { __mod_node_page_state(page_pgdat(page), NR_SHMEM, -nr); @@ -867,6 +866,7 @@ noinline int __add_to_page_cache_locked(struct page *page, gfp_mask, &memcg, false); if (error) return error; + mem_cgroup_shrink_pagecache(memcg, gfp_mask); } get_page(page); diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 5f26dd945..505548053 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -74,6 +74,7 @@ EXPORT_SYMBOL(memory_cgrp_subsys); struct mem_cgroup *root_mem_cgroup __read_mostly; #define MEM_CGROUP_RECLAIM_RETRIES 5 +#define DEFAULT_PAGE_RECLAIM_RATIO 5 /* Socket memory accounting disabled? */ static bool cgroup_memory_nosocket; @@ -231,21 +232,6 @@ enum res_type { /* Used for OOM nofiier */ #define OOM_CONTROL (0) -/* - * Iteration constructs for visiting all cgroups (under a tree). If - * loops are exited prematurely (break), mem_cgroup_iter_break() must - * be used for reference counting. - */ -#define for_each_mem_cgroup_tree(iter, root) \ - for (iter = mem_cgroup_iter(root, NULL, NULL); \ - iter != NULL; \ - iter = mem_cgroup_iter(root, iter, NULL)) - -#define for_each_mem_cgroup(iter) \ - for (iter = mem_cgroup_iter(NULL, NULL, NULL); \ - iter != NULL; \ - iter = mem_cgroup_iter(NULL, iter, NULL)) - static inline bool should_force_charge(void) { return tsk_is_oom_victim(current) || fatal_signal_pending(current) || @@ -709,6 +695,13 @@ void __mod_memcg_state(struct mem_cgroup *memcg, int idx, int val) x = 0; } __this_cpu_write(memcg->vmstats_percpu->stat[idx], x); + + if (idx == MEMCG_CACHE) { + if (val > 0) + page_counter_charge(&memcg->pagecache, val); + else + page_counter_uncharge(&memcg->pagecache, -val); + } } static struct mem_cgroup_per_node * @@ -3172,6 +3165,8 @@ static inline int mem_cgroup_move_swap_account(swp_entry_t entry, } #endif +static void pagecache_set_limit(struct mem_cgroup *memcg); + static DEFINE_MUTEX(memcg_max_mutex); static int mem_cgroup_resize_max(struct mem_cgroup *memcg, @@ -3222,8 +3217,11 @@ static int mem_cgroup_resize_max(struct mem_cgroup *memcg, } } while (true); - if (!ret && enlarge) - memcg_oom_recover(memcg); + if (!ret) { + if (enlarge) + memcg_oom_recover(memcg); + pagecache_set_limit(memcg); + } return ret; } @@ -3408,6 +3406,215 @@ static int mem_cgroup_hierarchy_write(struct cgroup_subsys_state *css, return retval; } +#define MIN_PAGECACHE_PAGES 16 + +unsigned int vm_pagecache_limit_retry_times; +void mem_cgroup_shrink_pagecache(struct mem_cgroup *memcg, gfp_t gfp_mask) +{ + unsigned long pages_used, pages_max, pages_reclaimed, goal_pages_used, pre_used; + unsigned int retry_times = 0; + unsigned int limit_retry_times; + + if (!memcg || mem_cgroup_is_root(memcg)) + return; + + pages_max = READ_ONCE(memcg->pagecache.max); + if (pages_max == PAGE_COUNTER_MAX || vm_pagecache_limit_global) + return; + + if (gfp_mask & __GFP_ATOMIC) + return; + + if (unlikely(should_force_charge())) + return; + + if (unlikely(current->flags & PF_MEMALLOC)) + return; + + if (unlikely(task_in_memcg_oom(current))) + return; + + if (!gfpflags_allow_blocking(gfp_mask)) + return; + + pages_used = page_counter_read(&memcg->pagecache); + limit_retry_times = READ_ONCE(vm_pagecache_limit_retry_times); + goal_pages_used = (100 - READ_ONCE(memcg->pagecache_reclaim_ratio)) * pages_max / 100; + goal_pages_used = max_t(unsigned long, MIN_PAGECACHE_PAGES, goal_pages_used); + + if (pages_used > pages_max) { + memcg_memory_event(memcg, MEMCG_PAGECACHE_MAX); + while (pages_used > goal_pages_used) { + if (fatal_signal_pending(current)) + break; + + pre_used = pages_used; + pages_reclaimed = shrink_page_cache_memcg(gfp_mask, memcg, pages_used - goal_pages_used); + + if (limit_retry_times == 0) + goto next_shrink; + + if (pages_reclaimed == 0) { + congestion_wait(BLK_RW_ASYNC, HZ/10); + retry_times++; + } else + retry_times = 0; + + if (retry_times > limit_retry_times) { + memcg_memory_event(memcg, MEMCG_PAGECACHE_OOM); + mem_cgroup_out_of_memory(memcg, GFP_KERNEL, 0); + break; + } + +next_shrink: + pages_used = page_counter_read(&memcg->pagecache); + cond_resched(); + } + } +} + +static u64 pagecache_reclaim_ratio_read(struct cgroup_subsys_state *css, + struct cftype *cft) +{ + struct mem_cgroup *memcg = mem_cgroup_from_css(css); + + return memcg->pagecache_reclaim_ratio; +} + +static ssize_t pagecache_reclaim_ratio_write(struct kernfs_open_file *of, + char *buf, size_t nbytes, loff_t off) +{ + struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); + u64 reclaim_ratio; + int ret; + unsigned long nr_pages; + + buf = strstrip(buf); + if (!buf) + return -EINVAL; + + ret = kstrtou64(buf, 0, &reclaim_ratio); + if (ret) + return ret; + + if ((reclaim_ratio > 0) && (reclaim_ratio < 100)) { + memcg->pagecache_reclaim_ratio = reclaim_ratio; + return nbytes; + } else if (reclaim_ratio == 100) { + nr_pages = page_counter_read(&memcg->pagecache); + shrink_page_cache_memcg(GFP_KERNEL, memcg, nr_pages); + return nbytes; + } + + return -EINVAL; +} + +static u64 pagecache_current_read(struct cgroup_subsys_state *css, + struct cftype *cft) +{ + struct mem_cgroup *memcg = mem_cgroup_from_css(css); + + return (u64)page_counter_read(&memcg->pagecache) * PAGE_SIZE; +} + +static u64 memory_pagecache_max_read(struct cgroup_subsys_state *css, + struct cftype *cft) +{ + struct mem_cgroup *memcg = mem_cgroup_from_css(css); + + return memcg->pagecache_max_ratio; +} + +unsigned long mem_cgroup_pagecache_get_reclaim_pages(struct mem_cgroup *memcg) +{ + unsigned long goal_pages_used, pages_used, pages_max; + + if ((!memcg) || (mem_cgroup_is_root(memcg))) + return 0; + + pages_max = READ_ONCE(memcg->pagecache.max); + if (pages_max == PAGE_COUNTER_MAX) + return 0; + + goal_pages_used = (100 - READ_ONCE(memcg->pagecache_reclaim_ratio)) * pages_max / 100; + goal_pages_used = max_t(unsigned long, MIN_PAGECACHE_PAGES, goal_pages_used); + pages_used = page_counter_read(&memcg->pagecache); + + return pages_used > pages_max ? pages_used - goal_pages_used : 0; +} + +#define PAGECACHE_MAX_RATIO_MIN 5 +#define PAGECACHE_MAX_RATIO_MAX 100 + +static void pagecache_set_limit(struct mem_cgroup *memcg) +{ + unsigned long max, pre, pages_max; + u32 max_ratio; + + pages_max = READ_ONCE(memcg->memory.max); + max_ratio = READ_ONCE(memcg->pagecache_max_ratio); + max = ((pages_max * max_ratio) / 100); + xchg(&memcg->pagecache.max, max); +} + +static ssize_t memory_pagecache_max_write(struct kernfs_open_file *of, + char *buf, size_t nbytes, loff_t off) +{ + struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); + unsigned int nr_reclaims = MEM_CGROUP_RECLAIM_RETRIES; + unsigned long max, pages_reclaimed; + int err, ret = 0; + u64 max_ratio; + + if (!buf) + return -EINVAL; + + xchg(&memcg->pagecache.max, max); + ret = kstrtou64(buf, 0, &max_ratio); + if (ret) + return ret; + + if (max_ratio > PAGECACHE_MAX_RATIO_MAX || + max_ratio < PAGECACHE_MAX_RATIO_MIN) + return -EINVAL; + + if (READ_ONCE(memcg->memory.max) == PAGE_COUNTER_MAX) { + printk(KERN_WARNING "pagecache limit not allowed for cgroup without memory limit set\n"); + return -EPERM; + } + + memcg->pagecache_max_ratio = max_ratio; + pagecache_set_limit(memcg); + max = memcg->pagecache.max; + + for (;;) { + unsigned long pages_used = page_counter_read(&memcg->pagecache); + + if (pages_used <= max) + break; + + if (fatal_signal_pending(current)) { + ret = -EINTR; + break; + } + + if (nr_reclaims) { + pages_reclaimed = shrink_page_cache_memcg(GFP_KERNEL, memcg, mem_cgroup_pagecache_get_reclaim_pages(memcg)); + if (pages_reclaimed == 0) { + congestion_wait(BLK_RW_ASYNC, HZ/10); + nr_reclaims--; + } + continue; + } + + memcg_memory_event(memcg, MEMCG_OOM); + if (!mem_cgroup_out_of_memory(memcg, GFP_KERNEL, 0)) + return -EINVAL; + } + + return ret ? : nbytes; +} + static unsigned long mem_cgroup_usage(struct mem_cgroup *memcg, bool swap) { unsigned long val; @@ -5133,6 +5340,23 @@ static int mem_cgroup_vmstat_read(struct seq_file *m, void *vv) } static struct cftype mem_cgroup_legacy_files[] = { + { + .name = "pagecache.reclaim_ratio", + .flags = CFTYPE_NOT_ON_ROOT, + .read_u64 = pagecache_reclaim_ratio_read, + .write = pagecache_reclaim_ratio_write, + }, + { + .name = "pagecache.max_ratio", + .flags = CFTYPE_NOT_ON_ROOT, + .read_u64 = memory_pagecache_max_read, + .write = memory_pagecache_max_write, + }, + { + .name = "pagecache.current", + .flags = CFTYPE_NOT_ON_ROOT, + .read_u64 = pagecache_current_read, + }, { .name = "usage_in_bytes", .private = MEMFILE_PRIVATE(_MEM, RES_USAGE), @@ -5499,6 +5723,8 @@ mem_cgroup_css_alloc(struct cgroup_subsys_state *parent_css) memcg->high = PAGE_COUNTER_MAX; memcg->soft_limit = PAGE_COUNTER_MAX; + memcg->pagecache_reclaim_ratio = DEFAULT_PAGE_RECLAIM_RATIO; + memcg->pagecache_max_ratio = PAGECACHE_MAX_RATIO_MAX; if (parent) { memcg->swappiness = mem_cgroup_swappiness(parent); memcg->oom_kill_disable = parent->oom_kill_disable; @@ -5510,12 +5736,14 @@ mem_cgroup_css_alloc(struct cgroup_subsys_state *parent_css) page_counter_init(&memcg->memsw, &parent->memsw); page_counter_init(&memcg->kmem, &parent->kmem); page_counter_init(&memcg->tcpmem, &parent->tcpmem); + page_counter_init(&memcg->pagecache, &parent->pagecache); } else { page_counter_init(&memcg->memory, NULL); page_counter_init(&memcg->swap, NULL); page_counter_init(&memcg->memsw, NULL); page_counter_init(&memcg->kmem, NULL); page_counter_init(&memcg->tcpmem, NULL); + page_counter_init(&memcg->pagecache, NULL); /* * Deeper hierachy with use_hierarchy == false doesn't make * much sense so let cgroup subsystem know about this @@ -5648,6 +5876,7 @@ static void mem_cgroup_css_reset(struct cgroup_subsys_state *css) page_counter_set_max(&memcg->memsw, PAGE_COUNTER_MAX); page_counter_set_max(&memcg->kmem, PAGE_COUNTER_MAX); page_counter_set_max(&memcg->tcpmem, PAGE_COUNTER_MAX); + page_counter_set_max(&memcg->pagecache, PAGE_COUNTER_MAX); page_counter_set_min(&memcg->memory, 0); page_counter_set_low(&memcg->memory, 0); memcg->high = PAGE_COUNTER_MAX; @@ -6532,6 +6761,8 @@ static ssize_t memory_max_write(struct kernfs_open_file *of, break; } + pagecache_set_limit(memcg); + memcg_wb_domain_size_changed(memcg); return nbytes; } @@ -6544,6 +6775,8 @@ static void __memory_events_show(struct seq_file *m, atomic_long_t *events) seq_printf(m, "oom %lu\n", atomic_long_read(&events[MEMCG_OOM])); seq_printf(m, "oom_kill %lu\n", atomic_long_read(&events[MEMCG_OOM_KILL])); + seq_printf(m, "pagecache_max:%lu\n", atomic_long_read(&events[MEMCG_PAGECACHE_MAX])); + seq_printf(m, "pagecache_oom:%lu\n", atomic_long_read(&events[MEMCG_PAGECACHE_OOM])); } static int memory_events_show(struct seq_file *m, void *v) @@ -6607,6 +6840,23 @@ static ssize_t memory_oom_group_write(struct kernfs_open_file *of, } static struct cftype memory_files[] = { + { + .name = "pagecache.reclaim_ratio", + .flags = CFTYPE_NOT_ON_ROOT, + .read_u64 = pagecache_reclaim_ratio_read, + .write = pagecache_reclaim_ratio_write, + }, + { + .name = "pagecache.max_ratio", + .flags = CFTYPE_NOT_ON_ROOT, + .read_u64 = memory_pagecache_max_read, + .write = memory_pagecache_max_write, + }, + { + .name = "pagecache.current", + .flags = CFTYPE_NOT_ON_ROOT, + .read_u64 = pagecache_current_read, + }, { .name = "current", .flags = CFTYPE_NOT_ON_ROOT, diff --git a/mm/vmscan.c b/mm/vmscan.c index 81564f6e5..36cff58c3 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -174,6 +174,7 @@ unsigned int vm_pagecache_limit_async __read_mostly = 0; unsigned int vm_pagecache_ignore_slab __read_mostly = 1; static struct task_struct *kpclimitd = NULL; static bool kpclimitd_context = false; +unsigned int vm_pagecache_limit_global __read_mostly = 1; /* * The total number of pages which are beyond the high watermark within all * zones. @@ -3583,7 +3584,7 @@ static bool kswapd_shrink_node(pg_data_t *pgdat, return sc->nr_scanned >= sc->nr_to_reclaim; } -static void __shrink_page_cache(gfp_t mask); +static unsigned long __shrink_page_cache(gfp_t mask, struct mem_cgroup *memcg, unsigned long nr_pages); /* * For kswapd, balance_pgdat() will reclaim pages across a node from zones @@ -3613,6 +3614,7 @@ static int balance_pgdat(pg_data_t *pgdat, int order, int classzone_idx) .order = order, .may_unmap = 1, }; + unsigned long nr_pages; set_task_reclaim_state(current, &sc.reclaim_state); psi_memstall_enter(&pflags); @@ -3621,8 +3623,11 @@ static int balance_pgdat(pg_data_t *pgdat, int order, int classzone_idx) count_vm_event(PAGEOUTRUN); /* this reclaims from all zones so don't count to sc.nr_reclaimed */ - if (unlikely(vm_pagecache_limit_pages) && pagecache_over_limit() > 0) - __shrink_page_cache(GFP_KERNEL); + if (unlikely(vm_pagecache_limit_pages)) { + nr_pages = pagecache_over_limit(); + if (nr_pages) + __shrink_page_cache(GFP_KERNEL, NULL, nr_pages); + } /* * Account for the reclaim boost. Note that the zone boost is left in @@ -4276,7 +4281,7 @@ out: * This function is similar to shrink_all_memory, except that it may never * swap out mapped pages and only does four passes. */ -static void __shrink_page_cache(gfp_t mask) +static unsigned long __shrink_page_cache(gfp_t mask, struct mem_cgroup *memcg, unsigned long nr_pages) { unsigned long ret = 0; int pass = 0; @@ -4287,11 +4292,10 @@ static void __shrink_page_cache(gfp_t mask) .priority = DEF_PRIORITY, .may_unmap = 0, .may_writepage = 0, - .target_mem_cgroup = NULL, + .target_mem_cgroup = memcg, .reclaim_idx = MAX_NR_ZONES, }; struct reclaim_state *old_rs = current->reclaim_state; - long nr_pages; /* We might sleep during direct reclaim so make atomic context * is certainly a bug. @@ -4300,7 +4304,7 @@ static void __shrink_page_cache(gfp_t mask) retry: /* How many pages are we over the limit?*/ - nr_pages = pagecache_over_limit(); +// nr_pages = pagecache_get_reclaim_pages(memcg); /* * Return early if there's no work to do. @@ -4309,7 +4313,7 @@ retry: * This makes sure that no sleeping reclaimer will stay behind. * Allow breaching the limit if the task is on the way out. */ - if (nr_pages <= 0 || fatal_signal_pending(current)) { + if (nr_pages == 0 || fatal_signal_pending(current)) { wake_up_interruptible(&pagecache_reclaim_wq); goto out; } @@ -4347,11 +4351,11 @@ retry: reclaim_state.reclaimed_slab = 0; for_each_online_node(nid) { - struct mem_cgroup *memcg = mem_cgroup_iter(NULL, NULL, NULL); + struct mem_cgroup *iter; - do { - shrink_slab(mask, nid, memcg, sc.priority); - } while ((memcg = mem_cgroup_iter(NULL, memcg, NULL)) != NULL); + for_each_mem_cgroup_tree(iter, memcg) { + shrink_slab(mask, nid, iter, sc.priority); + } } ret += reclaim_state.reclaimed_slab; @@ -4370,6 +4374,8 @@ retry: out: current->reclaim_state = old_rs; + + return sc.nr_reclaimed; } static int kpagecache_limitd(void *data) @@ -4384,7 +4390,7 @@ static int kpagecache_limitd(void *data) wake_up_interruptible(&pagecache_reclaim_wq); for ( ; ; ) { - __shrink_page_cache(GFP_KERNEL); + __shrink_page_cache(GFP_KERNEL, NULL, pagecache_over_limit()); prepare_to_wait(&kpagecache_limitd_wq, &wait, TASK_INTERRUPTIBLE); if (!kthread_should_stop()) @@ -4408,10 +4414,20 @@ static void wakeup_kpclimitd(gfp_t mask) void shrink_page_cache(gfp_t mask, struct page *page) { - if (0 == vm_pagecache_limit_async) - __shrink_page_cache(mask); - else - wakeup_kpclimitd(mask); + if (vm_pagecache_limit_global) { + if (0 == vm_pagecache_limit_async) + __shrink_page_cache(mask, NULL, pagecache_over_limit()); + else + wakeup_kpclimitd(mask); + } +} + +unsigned long shrink_page_cache_memcg(gfp_t mask, struct mem_cgroup *memcg, unsigned long nr_pages) +{ + if (!vm_pagecache_limit_global) + return __shrink_page_cache(mask, memcg, nr_pages); + + return 0; } /* It's optimal to keep kswapds on the same CPUs as their memory, but