forked from TencentOS/TencentOS-kernel
mm: pagecache limit per cgroup support
Signed-off-by: Chen Xiaoguang <xiaoggchen@tencent.com> Signed-off-by: Zeng Jingxiang <linuszeng@tencent.com> Reviewed-by: Bin Lai <robinlai@tencent.com> Reviewed-by: bauerchen <bauerchen@tencent.com>
This commit is contained in:
linuszeng
Chen Lei
parent
a3472379b7
commit
06d37efeef
|
|
@ -47,6 +47,8 @@ enum memcg_memory_event {
|
|||
MEMCG_OOM_KILL,
|
||||
MEMCG_SWAP_MAX,
|
||||
MEMCG_SWAP_FAIL,
|
||||
MEMCG_PAGECACHE_MAX,
|
||||
MEMCG_PAGECACHE_OOM,
|
||||
MEMCG_NR_MEMORY_EVENTS,
|
||||
};
|
||||
|
||||
|
|
@ -214,6 +216,9 @@ struct mem_cgroup {
|
|||
/* Accounted resources */
|
||||
struct page_counter memory;
|
||||
struct page_counter swap;
|
||||
struct page_counter pagecache;
|
||||
u64 pagecache_reclaim_ratio;
|
||||
u32 pagecache_max_ratio;
|
||||
|
||||
/* Legacy consumer-oriented counters */
|
||||
struct page_counter memsw;
|
||||
|
|
@ -345,6 +350,21 @@ struct mem_cgroup {
|
|||
*/
|
||||
#define MEMCG_CHARGE_BATCH 32U
|
||||
|
||||
/*
|
||||
* Iteration constructs for visiting all cgroups (under a tree). If
|
||||
* loops are exited prematurely (break), mem_cgroup_iter_break() must
|
||||
* be used for reference counting.
|
||||
*/
|
||||
#define for_each_mem_cgroup_tree(iter, root) \
|
||||
for (iter = mem_cgroup_iter(root, NULL, NULL); \
|
||||
iter != NULL; \
|
||||
iter = mem_cgroup_iter(root, iter, NULL))
|
||||
|
||||
#define for_each_mem_cgroup(iter) \
|
||||
for (iter = mem_cgroup_iter(NULL, NULL, NULL); \
|
||||
iter != NULL; \
|
||||
iter = mem_cgroup_iter(NULL, iter, NULL))
|
||||
|
||||
extern struct mem_cgroup *root_mem_cgroup;
|
||||
|
||||
static inline bool mem_cgroup_is_root(struct mem_cgroup *memcg)
|
||||
|
|
@ -1337,6 +1357,10 @@ struct sock;
|
|||
bool mem_cgroup_charge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages);
|
||||
void mem_cgroup_uncharge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages);
|
||||
#ifdef CONFIG_MEMCG
|
||||
|
||||
extern unsigned int vm_pagecache_limit_retry_times __read_mostly;
|
||||
extern void mem_cgroup_shrink_pagecache(struct mem_cgroup *memcg, gfp_t gfp_mask);
|
||||
|
||||
extern struct static_key_false memcg_sockets_enabled_key;
|
||||
#define mem_cgroup_sockets_enabled static_branch_unlikely(&memcg_sockets_enabled_key)
|
||||
void mem_cgroup_sk_alloc(struct sock *sk);
|
||||
|
|
|
|||
|
|
@ -370,6 +370,7 @@ extern int vm_swappiness;
|
|||
#define ADDITIONAL_RECLAIM_RATIO 2
|
||||
extern unsigned long pagecache_over_limit(void);
|
||||
extern void shrink_page_cache(gfp_t mask, struct page *page);
|
||||
extern unsigned long shrink_page_cache_memcg(gfp_t mask, struct mem_cgroup *memcg, unsigned long nr_pages);
|
||||
extern unsigned long vm_pagecache_limit_pages;
|
||||
extern unsigned long vm_pagecache_limit_reclaim_pages;
|
||||
extern int vm_pagecache_limit_ratio;
|
||||
|
|
@ -379,6 +380,7 @@ extern unsigned int vm_pagecache_limit_async;
|
|||
extern unsigned int vm_pagecache_ignore_slab;
|
||||
extern int kpagecache_limitd_run(void);
|
||||
extern void kpagecache_limitd_stop(void);
|
||||
extern unsigned int vm_pagecache_limit_global;
|
||||
extern int remove_mapping(struct address_space *mapping, struct page *page);
|
||||
extern unsigned long vm_total_pages;
|
||||
|
||||
|
|
|
|||
|
|
@ -1620,6 +1620,24 @@ static struct ctl_table vm_table[] = {
|
|||
.mode = 0644,
|
||||
.proc_handler = &proc_dointvec,
|
||||
},
|
||||
{
|
||||
.procname = "pagecache_limit_global",
|
||||
.data = &vm_pagecache_limit_global,
|
||||
.maxlen = sizeof(vm_pagecache_limit_global),
|
||||
.mode = 0644,
|
||||
.proc_handler = &proc_dointvec_minmax,
|
||||
.extra1 = SYSCTL_ZERO,
|
||||
.extra2 = SYSCTL_ONE,
|
||||
},
|
||||
{
|
||||
.procname = "pagecache_limit_retry_times",
|
||||
.data = &vm_pagecache_limit_retry_times,
|
||||
.maxlen = sizeof(vm_pagecache_limit_retry_times),
|
||||
.mode = 0644,
|
||||
.proc_handler = &proc_dointvec_minmax,
|
||||
.extra1 = SYSCTL_ZERO,
|
||||
.extra2 = &one_hundred,
|
||||
},
|
||||
#ifdef CONFIG_HUGETLB_PAGE
|
||||
{
|
||||
.procname = "nr_hugepages",
|
||||
|
|
|
|||
|
|
@ -198,7 +198,6 @@ static void unaccount_page_cache_page(struct address_space *mapping,
|
|||
return;
|
||||
|
||||
nr = hpage_nr_pages(page);
|
||||
|
||||
__mod_node_page_state(page_pgdat(page), NR_FILE_PAGES, -nr);
|
||||
if (PageSwapBacked(page)) {
|
||||
__mod_node_page_state(page_pgdat(page), NR_SHMEM, -nr);
|
||||
|
|
@ -867,6 +866,7 @@ noinline int __add_to_page_cache_locked(struct page *page,
|
|||
gfp_mask, &memcg, false);
|
||||
if (error)
|
||||
return error;
|
||||
mem_cgroup_shrink_pagecache(memcg, gfp_mask);
|
||||
}
|
||||
|
||||
get_page(page);
|
||||
|
|
|
|||
284
mm/memcontrol.c
284
mm/memcontrol.c
|
|
@ -74,6 +74,7 @@ EXPORT_SYMBOL(memory_cgrp_subsys);
|
|||
struct mem_cgroup *root_mem_cgroup __read_mostly;
|
||||
|
||||
#define MEM_CGROUP_RECLAIM_RETRIES 5
|
||||
#define DEFAULT_PAGE_RECLAIM_RATIO 5
|
||||
|
||||
/* Socket memory accounting disabled? */
|
||||
static bool cgroup_memory_nosocket;
|
||||
|
|
@ -231,21 +232,6 @@ enum res_type {
|
|||
/* Used for OOM nofiier */
|
||||
#define OOM_CONTROL (0)
|
||||
|
||||
/*
|
||||
* Iteration constructs for visiting all cgroups (under a tree). If
|
||||
* loops are exited prematurely (break), mem_cgroup_iter_break() must
|
||||
* be used for reference counting.
|
||||
*/
|
||||
#define for_each_mem_cgroup_tree(iter, root) \
|
||||
for (iter = mem_cgroup_iter(root, NULL, NULL); \
|
||||
iter != NULL; \
|
||||
iter = mem_cgroup_iter(root, iter, NULL))
|
||||
|
||||
#define for_each_mem_cgroup(iter) \
|
||||
for (iter = mem_cgroup_iter(NULL, NULL, NULL); \
|
||||
iter != NULL; \
|
||||
iter = mem_cgroup_iter(NULL, iter, NULL))
|
||||
|
||||
static inline bool should_force_charge(void)
|
||||
{
|
||||
return tsk_is_oom_victim(current) || fatal_signal_pending(current) ||
|
||||
|
|
@ -709,6 +695,13 @@ void __mod_memcg_state(struct mem_cgroup *memcg, int idx, int val)
|
|||
x = 0;
|
||||
}
|
||||
__this_cpu_write(memcg->vmstats_percpu->stat[idx], x);
|
||||
|
||||
if (idx == MEMCG_CACHE) {
|
||||
if (val > 0)
|
||||
page_counter_charge(&memcg->pagecache, val);
|
||||
else
|
||||
page_counter_uncharge(&memcg->pagecache, -val);
|
||||
}
|
||||
}
|
||||
|
||||
static struct mem_cgroup_per_node *
|
||||
|
|
@ -3172,6 +3165,8 @@ static inline int mem_cgroup_move_swap_account(swp_entry_t entry,
|
|||
}
|
||||
#endif
|
||||
|
||||
static void pagecache_set_limit(struct mem_cgroup *memcg);
|
||||
|
||||
static DEFINE_MUTEX(memcg_max_mutex);
|
||||
|
||||
static int mem_cgroup_resize_max(struct mem_cgroup *memcg,
|
||||
|
|
@ -3222,8 +3217,11 @@ static int mem_cgroup_resize_max(struct mem_cgroup *memcg,
|
|||
}
|
||||
} while (true);
|
||||
|
||||
if (!ret && enlarge)
|
||||
memcg_oom_recover(memcg);
|
||||
if (!ret) {
|
||||
if (enlarge)
|
||||
memcg_oom_recover(memcg);
|
||||
pagecache_set_limit(memcg);
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
|
@ -3408,6 +3406,215 @@ static int mem_cgroup_hierarchy_write(struct cgroup_subsys_state *css,
|
|||
return retval;
|
||||
}
|
||||
|
||||
#define MIN_PAGECACHE_PAGES 16
|
||||
|
||||
unsigned int vm_pagecache_limit_retry_times;
|
||||
void mem_cgroup_shrink_pagecache(struct mem_cgroup *memcg, gfp_t gfp_mask)
|
||||
{
|
||||
unsigned long pages_used, pages_max, pages_reclaimed, goal_pages_used, pre_used;
|
||||
unsigned int retry_times = 0;
|
||||
unsigned int limit_retry_times;
|
||||
|
||||
if (!memcg || mem_cgroup_is_root(memcg))
|
||||
return;
|
||||
|
||||
pages_max = READ_ONCE(memcg->pagecache.max);
|
||||
if (pages_max == PAGE_COUNTER_MAX || vm_pagecache_limit_global)
|
||||
return;
|
||||
|
||||
if (gfp_mask & __GFP_ATOMIC)
|
||||
return;
|
||||
|
||||
if (unlikely(should_force_charge()))
|
||||
return;
|
||||
|
||||
if (unlikely(current->flags & PF_MEMALLOC))
|
||||
return;
|
||||
|
||||
if (unlikely(task_in_memcg_oom(current)))
|
||||
return;
|
||||
|
||||
if (!gfpflags_allow_blocking(gfp_mask))
|
||||
return;
|
||||
|
||||
pages_used = page_counter_read(&memcg->pagecache);
|
||||
limit_retry_times = READ_ONCE(vm_pagecache_limit_retry_times);
|
||||
goal_pages_used = (100 - READ_ONCE(memcg->pagecache_reclaim_ratio)) * pages_max / 100;
|
||||
goal_pages_used = max_t(unsigned long, MIN_PAGECACHE_PAGES, goal_pages_used);
|
||||
|
||||
if (pages_used > pages_max) {
|
||||
memcg_memory_event(memcg, MEMCG_PAGECACHE_MAX);
|
||||
while (pages_used > goal_pages_used) {
|
||||
if (fatal_signal_pending(current))
|
||||
break;
|
||||
|
||||
pre_used = pages_used;
|
||||
pages_reclaimed = shrink_page_cache_memcg(gfp_mask, memcg, pages_used - goal_pages_used);
|
||||
|
||||
if (limit_retry_times == 0)
|
||||
goto next_shrink;
|
||||
|
||||
if (pages_reclaimed == 0) {
|
||||
congestion_wait(BLK_RW_ASYNC, HZ/10);
|
||||
retry_times++;
|
||||
} else
|
||||
retry_times = 0;
|
||||
|
||||
if (retry_times > limit_retry_times) {
|
||||
memcg_memory_event(memcg, MEMCG_PAGECACHE_OOM);
|
||||
mem_cgroup_out_of_memory(memcg, GFP_KERNEL, 0);
|
||||
break;
|
||||
}
|
||||
|
||||
next_shrink:
|
||||
pages_used = page_counter_read(&memcg->pagecache);
|
||||
cond_resched();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static u64 pagecache_reclaim_ratio_read(struct cgroup_subsys_state *css,
|
||||
struct cftype *cft)
|
||||
{
|
||||
struct mem_cgroup *memcg = mem_cgroup_from_css(css);
|
||||
|
||||
return memcg->pagecache_reclaim_ratio;
|
||||
}
|
||||
|
||||
static ssize_t pagecache_reclaim_ratio_write(struct kernfs_open_file *of,
|
||||
char *buf, size_t nbytes, loff_t off)
|
||||
{
|
||||
struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
|
||||
u64 reclaim_ratio;
|
||||
int ret;
|
||||
unsigned long nr_pages;
|
||||
|
||||
buf = strstrip(buf);
|
||||
if (!buf)
|
||||
return -EINVAL;
|
||||
|
||||
ret = kstrtou64(buf, 0, &reclaim_ratio);
|
||||
if (ret)
|
||||
return ret;
|
||||
|
||||
if ((reclaim_ratio > 0) && (reclaim_ratio < 100)) {
|
||||
memcg->pagecache_reclaim_ratio = reclaim_ratio;
|
||||
return nbytes;
|
||||
} else if (reclaim_ratio == 100) {
|
||||
nr_pages = page_counter_read(&memcg->pagecache);
|
||||
shrink_page_cache_memcg(GFP_KERNEL, memcg, nr_pages);
|
||||
return nbytes;
|
||||
}
|
||||
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
static u64 pagecache_current_read(struct cgroup_subsys_state *css,
|
||||
struct cftype *cft)
|
||||
{
|
||||
struct mem_cgroup *memcg = mem_cgroup_from_css(css);
|
||||
|
||||
return (u64)page_counter_read(&memcg->pagecache) * PAGE_SIZE;
|
||||
}
|
||||
|
||||
static u64 memory_pagecache_max_read(struct cgroup_subsys_state *css,
|
||||
struct cftype *cft)
|
||||
{
|
||||
struct mem_cgroup *memcg = mem_cgroup_from_css(css);
|
||||
|
||||
return memcg->pagecache_max_ratio;
|
||||
}
|
||||
|
||||
unsigned long mem_cgroup_pagecache_get_reclaim_pages(struct mem_cgroup *memcg)
|
||||
{
|
||||
unsigned long goal_pages_used, pages_used, pages_max;
|
||||
|
||||
if ((!memcg) || (mem_cgroup_is_root(memcg)))
|
||||
return 0;
|
||||
|
||||
pages_max = READ_ONCE(memcg->pagecache.max);
|
||||
if (pages_max == PAGE_COUNTER_MAX)
|
||||
return 0;
|
||||
|
||||
goal_pages_used = (100 - READ_ONCE(memcg->pagecache_reclaim_ratio)) * pages_max / 100;
|
||||
goal_pages_used = max_t(unsigned long, MIN_PAGECACHE_PAGES, goal_pages_used);
|
||||
pages_used = page_counter_read(&memcg->pagecache);
|
||||
|
||||
return pages_used > pages_max ? pages_used - goal_pages_used : 0;
|
||||
}
|
||||
|
||||
#define PAGECACHE_MAX_RATIO_MIN 5
|
||||
#define PAGECACHE_MAX_RATIO_MAX 100
|
||||
|
||||
static void pagecache_set_limit(struct mem_cgroup *memcg)
|
||||
{
|
||||
unsigned long max, pre, pages_max;
|
||||
u32 max_ratio;
|
||||
|
||||
pages_max = READ_ONCE(memcg->memory.max);
|
||||
max_ratio = READ_ONCE(memcg->pagecache_max_ratio);
|
||||
max = ((pages_max * max_ratio) / 100);
|
||||
xchg(&memcg->pagecache.max, max);
|
||||
}
|
||||
|
||||
static ssize_t memory_pagecache_max_write(struct kernfs_open_file *of,
|
||||
char *buf, size_t nbytes, loff_t off)
|
||||
{
|
||||
struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
|
||||
unsigned int nr_reclaims = MEM_CGROUP_RECLAIM_RETRIES;
|
||||
unsigned long max, pages_reclaimed;
|
||||
int err, ret = 0;
|
||||
u64 max_ratio;
|
||||
|
||||
if (!buf)
|
||||
return -EINVAL;
|
||||
|
||||
xchg(&memcg->pagecache.max, max);
|
||||
ret = kstrtou64(buf, 0, &max_ratio);
|
||||
if (ret)
|
||||
return ret;
|
||||
|
||||
if (max_ratio > PAGECACHE_MAX_RATIO_MAX ||
|
||||
max_ratio < PAGECACHE_MAX_RATIO_MIN)
|
||||
return -EINVAL;
|
||||
|
||||
if (READ_ONCE(memcg->memory.max) == PAGE_COUNTER_MAX) {
|
||||
printk(KERN_WARNING "pagecache limit not allowed for cgroup without memory limit set\n");
|
||||
return -EPERM;
|
||||
}
|
||||
|
||||
memcg->pagecache_max_ratio = max_ratio;
|
||||
pagecache_set_limit(memcg);
|
||||
max = memcg->pagecache.max;
|
||||
|
||||
for (;;) {
|
||||
unsigned long pages_used = page_counter_read(&memcg->pagecache);
|
||||
|
||||
if (pages_used <= max)
|
||||
break;
|
||||
|
||||
if (fatal_signal_pending(current)) {
|
||||
ret = -EINTR;
|
||||
break;
|
||||
}
|
||||
|
||||
if (nr_reclaims) {
|
||||
pages_reclaimed = shrink_page_cache_memcg(GFP_KERNEL, memcg, mem_cgroup_pagecache_get_reclaim_pages(memcg));
|
||||
if (pages_reclaimed == 0) {
|
||||
congestion_wait(BLK_RW_ASYNC, HZ/10);
|
||||
nr_reclaims--;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
memcg_memory_event(memcg, MEMCG_OOM);
|
||||
if (!mem_cgroup_out_of_memory(memcg, GFP_KERNEL, 0))
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
return ret ? : nbytes;
|
||||
}
|
||||
|
||||
static unsigned long mem_cgroup_usage(struct mem_cgroup *memcg, bool swap)
|
||||
{
|
||||
unsigned long val;
|
||||
|
|
@ -5133,6 +5340,23 @@ static int mem_cgroup_vmstat_read(struct seq_file *m, void *vv)
|
|||
}
|
||||
|
||||
static struct cftype mem_cgroup_legacy_files[] = {
|
||||
{
|
||||
.name = "pagecache.reclaim_ratio",
|
||||
.flags = CFTYPE_NOT_ON_ROOT,
|
||||
.read_u64 = pagecache_reclaim_ratio_read,
|
||||
.write = pagecache_reclaim_ratio_write,
|
||||
},
|
||||
{
|
||||
.name = "pagecache.max_ratio",
|
||||
.flags = CFTYPE_NOT_ON_ROOT,
|
||||
.read_u64 = memory_pagecache_max_read,
|
||||
.write = memory_pagecache_max_write,
|
||||
},
|
||||
{
|
||||
.name = "pagecache.current",
|
||||
.flags = CFTYPE_NOT_ON_ROOT,
|
||||
.read_u64 = pagecache_current_read,
|
||||
},
|
||||
{
|
||||
.name = "usage_in_bytes",
|
||||
.private = MEMFILE_PRIVATE(_MEM, RES_USAGE),
|
||||
|
|
@ -5499,6 +5723,8 @@ mem_cgroup_css_alloc(struct cgroup_subsys_state *parent_css)
|
|||
|
||||
memcg->high = PAGE_COUNTER_MAX;
|
||||
memcg->soft_limit = PAGE_COUNTER_MAX;
|
||||
memcg->pagecache_reclaim_ratio = DEFAULT_PAGE_RECLAIM_RATIO;
|
||||
memcg->pagecache_max_ratio = PAGECACHE_MAX_RATIO_MAX;
|
||||
if (parent) {
|
||||
memcg->swappiness = mem_cgroup_swappiness(parent);
|
||||
memcg->oom_kill_disable = parent->oom_kill_disable;
|
||||
|
|
@ -5510,12 +5736,14 @@ mem_cgroup_css_alloc(struct cgroup_subsys_state *parent_css)
|
|||
page_counter_init(&memcg->memsw, &parent->memsw);
|
||||
page_counter_init(&memcg->kmem, &parent->kmem);
|
||||
page_counter_init(&memcg->tcpmem, &parent->tcpmem);
|
||||
page_counter_init(&memcg->pagecache, &parent->pagecache);
|
||||
} else {
|
||||
page_counter_init(&memcg->memory, NULL);
|
||||
page_counter_init(&memcg->swap, NULL);
|
||||
page_counter_init(&memcg->memsw, NULL);
|
||||
page_counter_init(&memcg->kmem, NULL);
|
||||
page_counter_init(&memcg->tcpmem, NULL);
|
||||
page_counter_init(&memcg->pagecache, NULL);
|
||||
/*
|
||||
* Deeper hierachy with use_hierarchy == false doesn't make
|
||||
* much sense so let cgroup subsystem know about this
|
||||
|
|
@ -5648,6 +5876,7 @@ static void mem_cgroup_css_reset(struct cgroup_subsys_state *css)
|
|||
page_counter_set_max(&memcg->memsw, PAGE_COUNTER_MAX);
|
||||
page_counter_set_max(&memcg->kmem, PAGE_COUNTER_MAX);
|
||||
page_counter_set_max(&memcg->tcpmem, PAGE_COUNTER_MAX);
|
||||
page_counter_set_max(&memcg->pagecache, PAGE_COUNTER_MAX);
|
||||
page_counter_set_min(&memcg->memory, 0);
|
||||
page_counter_set_low(&memcg->memory, 0);
|
||||
memcg->high = PAGE_COUNTER_MAX;
|
||||
|
|
@ -6532,6 +6761,8 @@ static ssize_t memory_max_write(struct kernfs_open_file *of,
|
|||
break;
|
||||
}
|
||||
|
||||
pagecache_set_limit(memcg);
|
||||
|
||||
memcg_wb_domain_size_changed(memcg);
|
||||
return nbytes;
|
||||
}
|
||||
|
|
@ -6544,6 +6775,8 @@ static void __memory_events_show(struct seq_file *m, atomic_long_t *events)
|
|||
seq_printf(m, "oom %lu\n", atomic_long_read(&events[MEMCG_OOM]));
|
||||
seq_printf(m, "oom_kill %lu\n",
|
||||
atomic_long_read(&events[MEMCG_OOM_KILL]));
|
||||
seq_printf(m, "pagecache_max:%lu\n", atomic_long_read(&events[MEMCG_PAGECACHE_MAX]));
|
||||
seq_printf(m, "pagecache_oom:%lu\n", atomic_long_read(&events[MEMCG_PAGECACHE_OOM]));
|
||||
}
|
||||
|
||||
static int memory_events_show(struct seq_file *m, void *v)
|
||||
|
|
@ -6607,6 +6840,23 @@ static ssize_t memory_oom_group_write(struct kernfs_open_file *of,
|
|||
}
|
||||
|
||||
static struct cftype memory_files[] = {
|
||||
{
|
||||
.name = "pagecache.reclaim_ratio",
|
||||
.flags = CFTYPE_NOT_ON_ROOT,
|
||||
.read_u64 = pagecache_reclaim_ratio_read,
|
||||
.write = pagecache_reclaim_ratio_write,
|
||||
},
|
||||
{
|
||||
.name = "pagecache.max_ratio",
|
||||
.flags = CFTYPE_NOT_ON_ROOT,
|
||||
.read_u64 = memory_pagecache_max_read,
|
||||
.write = memory_pagecache_max_write,
|
||||
},
|
||||
{
|
||||
.name = "pagecache.current",
|
||||
.flags = CFTYPE_NOT_ON_ROOT,
|
||||
.read_u64 = pagecache_current_read,
|
||||
},
|
||||
{
|
||||
.name = "current",
|
||||
.flags = CFTYPE_NOT_ON_ROOT,
|
||||
|
|
|
|||
50
mm/vmscan.c
50
mm/vmscan.c
|
|
@ -174,6 +174,7 @@ unsigned int vm_pagecache_limit_async __read_mostly = 0;
|
|||
unsigned int vm_pagecache_ignore_slab __read_mostly = 1;
|
||||
static struct task_struct *kpclimitd = NULL;
|
||||
static bool kpclimitd_context = false;
|
||||
unsigned int vm_pagecache_limit_global __read_mostly = 1;
|
||||
/*
|
||||
* The total number of pages which are beyond the high watermark within all
|
||||
* zones.
|
||||
|
|
@ -3583,7 +3584,7 @@ static bool kswapd_shrink_node(pg_data_t *pgdat,
|
|||
return sc->nr_scanned >= sc->nr_to_reclaim;
|
||||
}
|
||||
|
||||
static void __shrink_page_cache(gfp_t mask);
|
||||
static unsigned long __shrink_page_cache(gfp_t mask, struct mem_cgroup *memcg, unsigned long nr_pages);
|
||||
|
||||
/*
|
||||
* For kswapd, balance_pgdat() will reclaim pages across a node from zones
|
||||
|
|
@ -3613,6 +3614,7 @@ static int balance_pgdat(pg_data_t *pgdat, int order, int classzone_idx)
|
|||
.order = order,
|
||||
.may_unmap = 1,
|
||||
};
|
||||
unsigned long nr_pages;
|
||||
|
||||
set_task_reclaim_state(current, &sc.reclaim_state);
|
||||
psi_memstall_enter(&pflags);
|
||||
|
|
@ -3621,8 +3623,11 @@ static int balance_pgdat(pg_data_t *pgdat, int order, int classzone_idx)
|
|||
count_vm_event(PAGEOUTRUN);
|
||||
|
||||
/* this reclaims from all zones so don't count to sc.nr_reclaimed */
|
||||
if (unlikely(vm_pagecache_limit_pages) && pagecache_over_limit() > 0)
|
||||
__shrink_page_cache(GFP_KERNEL);
|
||||
if (unlikely(vm_pagecache_limit_pages)) {
|
||||
nr_pages = pagecache_over_limit();
|
||||
if (nr_pages)
|
||||
__shrink_page_cache(GFP_KERNEL, NULL, nr_pages);
|
||||
}
|
||||
|
||||
/*
|
||||
* Account for the reclaim boost. Note that the zone boost is left in
|
||||
|
|
@ -4276,7 +4281,7 @@ out:
|
|||
* This function is similar to shrink_all_memory, except that it may never
|
||||
* swap out mapped pages and only does four passes.
|
||||
*/
|
||||
static void __shrink_page_cache(gfp_t mask)
|
||||
static unsigned long __shrink_page_cache(gfp_t mask, struct mem_cgroup *memcg, unsigned long nr_pages)
|
||||
{
|
||||
unsigned long ret = 0;
|
||||
int pass = 0;
|
||||
|
|
@ -4287,11 +4292,10 @@ static void __shrink_page_cache(gfp_t mask)
|
|||
.priority = DEF_PRIORITY,
|
||||
.may_unmap = 0,
|
||||
.may_writepage = 0,
|
||||
.target_mem_cgroup = NULL,
|
||||
.target_mem_cgroup = memcg,
|
||||
.reclaim_idx = MAX_NR_ZONES,
|
||||
};
|
||||
struct reclaim_state *old_rs = current->reclaim_state;
|
||||
long nr_pages;
|
||||
|
||||
/* We might sleep during direct reclaim so make atomic context
|
||||
* is certainly a bug.
|
||||
|
|
@ -4300,7 +4304,7 @@ static void __shrink_page_cache(gfp_t mask)
|
|||
|
||||
retry:
|
||||
/* How many pages are we over the limit?*/
|
||||
nr_pages = pagecache_over_limit();
|
||||
// nr_pages = pagecache_get_reclaim_pages(memcg);
|
||||
|
||||
/*
|
||||
* Return early if there's no work to do.
|
||||
|
|
@ -4309,7 +4313,7 @@ retry:
|
|||
* This makes sure that no sleeping reclaimer will stay behind.
|
||||
* Allow breaching the limit if the task is on the way out.
|
||||
*/
|
||||
if (nr_pages <= 0 || fatal_signal_pending(current)) {
|
||||
if (nr_pages == 0 || fatal_signal_pending(current)) {
|
||||
wake_up_interruptible(&pagecache_reclaim_wq);
|
||||
goto out;
|
||||
}
|
||||
|
|
@ -4347,11 +4351,11 @@ retry:
|
|||
|
||||
reclaim_state.reclaimed_slab = 0;
|
||||
for_each_online_node(nid) {
|
||||
struct mem_cgroup *memcg = mem_cgroup_iter(NULL, NULL, NULL);
|
||||
struct mem_cgroup *iter;
|
||||
|
||||
do {
|
||||
shrink_slab(mask, nid, memcg, sc.priority);
|
||||
} while ((memcg = mem_cgroup_iter(NULL, memcg, NULL)) != NULL);
|
||||
for_each_mem_cgroup_tree(iter, memcg) {
|
||||
shrink_slab(mask, nid, iter, sc.priority);
|
||||
}
|
||||
}
|
||||
ret += reclaim_state.reclaimed_slab;
|
||||
|
||||
|
|
@ -4370,6 +4374,8 @@ retry:
|
|||
|
||||
out:
|
||||
current->reclaim_state = old_rs;
|
||||
|
||||
return sc.nr_reclaimed;
|
||||
}
|
||||
|
||||
static int kpagecache_limitd(void *data)
|
||||
|
|
@ -4384,7 +4390,7 @@ static int kpagecache_limitd(void *data)
|
|||
wake_up_interruptible(&pagecache_reclaim_wq);
|
||||
|
||||
for ( ; ; ) {
|
||||
__shrink_page_cache(GFP_KERNEL);
|
||||
__shrink_page_cache(GFP_KERNEL, NULL, pagecache_over_limit());
|
||||
prepare_to_wait(&kpagecache_limitd_wq, &wait, TASK_INTERRUPTIBLE);
|
||||
|
||||
if (!kthread_should_stop())
|
||||
|
|
@ -4408,10 +4414,20 @@ static void wakeup_kpclimitd(gfp_t mask)
|
|||
|
||||
void shrink_page_cache(gfp_t mask, struct page *page)
|
||||
{
|
||||
if (0 == vm_pagecache_limit_async)
|
||||
__shrink_page_cache(mask);
|
||||
else
|
||||
wakeup_kpclimitd(mask);
|
||||
if (vm_pagecache_limit_global) {
|
||||
if (0 == vm_pagecache_limit_async)
|
||||
__shrink_page_cache(mask, NULL, pagecache_over_limit());
|
||||
else
|
||||
wakeup_kpclimitd(mask);
|
||||
}
|
||||
}
|
||||
|
||||
unsigned long shrink_page_cache_memcg(gfp_t mask, struct mem_cgroup *memcg, unsigned long nr_pages)
|
||||
{
|
||||
if (!vm_pagecache_limit_global)
|
||||
return __shrink_page_cache(mask, memcg, nr_pages);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* It's optimal to keep kswapds on the same CPUs as their memory, but
|
||||
|
|
|
|||
Loading…
Reference in New Issue