diff --git a/include/linux/swap.h b/include/linux/swap.h index 95ce03875..9ec0f68c0 100644 --- a/include/linux/swap.h +++ b/include/linux/swap.h @@ -364,6 +364,9 @@ extern unsigned long vm_pagecache_limit_reclaim_pages; extern int vm_pagecache_limit_ratio; extern int vm_pagecache_limit_reclaim_ratio; extern unsigned int vm_pagecache_ignore_dirty; +extern unsigned int vm_pagecache_limit_async; +extern int kpagecache_limitd_run(void); +extern void kpagecache_limitd_stop(void); extern int remove_mapping(struct address_space *mapping, struct page *page); extern unsigned long vm_total_pages; diff --git a/kernel/sysctl.c b/kernel/sysctl.c index 27ea5d92a..45e22d850 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -1280,12 +1280,15 @@ static struct ctl_table kern_table[] = { { } }; -int pc_limit_proc_dointvec(struct ctl_table *table, int write, +static int pc_limit_proc_dointvec(struct ctl_table *table, int write, void __user *buffer, size_t *lenp, loff_t *ppos); -int pc_reclaim_limit_proc_dointvec(struct ctl_table *table, int write, +static int pc_reclaim_limit_proc_dointvec(struct ctl_table *table, int write, void __user *buffer, size_t *lenp, loff_t *ppos); +static int pc_limit_async_handler(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, loff_t *ppos); + static struct ctl_table vm_table[] = { { .procname = "overcommit_memory", @@ -1437,6 +1440,13 @@ static struct ctl_table vm_table[] = { .mode = 0644, .proc_handler = &proc_dointvec, }, + { + .procname = "pagecache_limit_async", + .data = &vm_pagecache_limit_async, + .maxlen = sizeof(vm_pagecache_limit_async), + .mode = 0644, + .proc_handler = &pc_limit_async_handler, + }, #ifdef CONFIG_HUGETLB_PAGE { .procname = "nr_hugepages", @@ -2401,7 +2411,7 @@ int setup_pagecache_limit(void) return 0; } -int pc_limit_proc_dointvec(struct ctl_table *table, int write, +static int pc_limit_proc_dointvec(struct ctl_table *table, int write, void __user *buffer, size_t *lenp, loff_t *ppos) { int ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); @@ -2410,14 +2420,15 @@ int pc_limit_proc_dointvec(struct ctl_table *table, int write, return ret; } -int pc_reclaim_limit_proc_dointvec(struct ctl_table *table, int write, +static int pc_reclaim_limit_proc_dointvec(struct ctl_table *table, int write, void __user *buffer, size_t *lenp, loff_t *ppos) { + int pre_reclaim_ratio = vm_pagecache_limit_reclaim_ratio; + int ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); + if (write && vm_pagecache_limit_ratio == 0) return -EINVAL; - int pre_reclaim_ratio = vm_pagecache_limit_reclaim_ratio; - int ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); if (write && !ret) { if (vm_pagecache_limit_reclaim_ratio - vm_pagecache_limit_ratio < ADDITIONAL_RECLAIM_RATIO) { vm_pagecache_limit_reclaim_ratio = pre_reclaim_ratio; @@ -2428,6 +2439,28 @@ int pc_reclaim_limit_proc_dointvec(struct ctl_table *table, int write, return ret; } +static int pc_limit_async_handler(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, loff_t *ppos) +{ + int ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); + + if (write && vm_pagecache_limit_ratio == 0) + return -EINVAL; + + if (write && !ret) { + if (vm_pagecache_limit_async > 0) { + if (kpagecache_limitd_run()) { + vm_pagecache_limit_async = 0; + return -EINVAL; + } + } + else { + kpagecache_limitd_stop(); + } + } + return ret; +} + static int do_proc_douintvec_w(unsigned int *tbl_data, struct ctl_table *table, void __user *buffer, diff --git a/mm/vmscan.c b/mm/vmscan.c index 41519c69c..d4d7afe2f 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -155,6 +155,9 @@ unsigned long vm_pagecache_limit_reclaim_pages = 0; int vm_pagecache_limit_ratio __read_mostly = 0; int vm_pagecache_limit_reclaim_ratio __read_mostly = 0; unsigned int vm_pagecache_ignore_dirty __read_mostly = 1; +unsigned int vm_pagecache_limit_async __read_mostly = 0; +static struct task_struct *kpclimitd = NULL; +static bool kpclimitd_context = false; /* * The total number of pages which are beyond the high watermark within all * zones. @@ -3762,6 +3765,7 @@ static void pagecache_reclaim_unlock_zone(struct zone *zone) * So this is basically a congestion wait queue for them. */ DECLARE_WAIT_QUEUE_HEAD(pagecache_reclaim_wq); +DECLARE_WAIT_QUEUE_HEAD(kpagecache_limitd_wq); /* * Similar to shrink_zone but it has a different consumer - pagecache limit @@ -3898,7 +3902,8 @@ static int shrink_all_zones(unsigned long nr_pages, int pass, * do it if there is nothing to be done. */ if (!nr_locked_zones) { - schedule(); + if (!kpclimitd_context) + schedule(); finish_wait(&pagecache_reclaim_wq, &wait); goto out; } @@ -4018,9 +4023,46 @@ out: current->reclaim_state = old_rs; } +static int kpagecache_limitd(void *data) +{ + DEFINE_WAIT(wait); + kpclimitd_context = true; + + /* + * make sure all work threads woken up, when switch to async mode + */ + if (waitqueue_active(&pagecache_reclaim_wq)) + wake_up_interruptible(&pagecache_reclaim_wq); + + for ( ; ; ) { + __shrink_page_cache(GFP_KERNEL); + prepare_to_wait(&kpagecache_limitd_wq, &wait, TASK_INTERRUPTIBLE); + + if (!kthread_should_stop()) + schedule(); + else { + finish_wait(&kpagecache_limitd_wq, &wait); + break; + } + finish_wait(&kpagecache_limitd_wq, &wait); + } + kpclimitd_context = false; + return 0; +} + +static void wakeup_kpclimitd(gfp_t mask) +{ + if (!waitqueue_active(&kpagecache_limitd_wq)) + return; + wake_up_interruptible(&kpagecache_limitd_wq); +} + void shrink_page_cache(gfp_t mask, struct page *page) { - __shrink_page_cache(mask); + if (0 == vm_pagecache_limit_async) + __shrink_page_cache(mask); + else + wakeup_kpclimitd(mask); } /* It's optimal to keep kswapds on the same CPUs as their memory, but @@ -4097,6 +4139,31 @@ static int __init kswapd_init(void) module_init(kswapd_init) +int kpagecache_limitd_run(void) +{ + int ret = 0; + + if (kpclimitd) + return 0; + + kpclimitd = kthread_run(kpagecache_limitd, NULL, "kpclimitd"); + if (IS_ERR(kpclimitd)) { + pr_err("Failed to start kpagecache_limitd thread\n"); + ret = PTR_ERR(kpclimitd); + kpclimitd = NULL; + } + return ret; + +} + +void kpagecache_limitd_stop(void) +{ + if (kpclimitd) { + kthread_stop(kpclimitd); + kpclimitd = NULL; + } +} + #ifdef CONFIG_NUMA /* * Node reclaim mode