From 6ef3bc24ee656a3d7b289c2bb364d1822da83895 Mon Sep 17 00:00:00 2001 From: Bin Lai Date: Mon, 5 Sep 2022 20:56:02 +0800 Subject: [PATCH] Revert " mm/lru: replace pgdat lru_lock with lruvec lock" This reverts commit 1b7478d8d01db85b9b5ec87bea0009f0d97f2eb5. --- include/linux/memcontrol.h | 61 +------------------- include/linux/mmzone.h | 3 +- mm/compaction.c | 55 ++++++++---------- mm/huge_memory.c | 9 ++- mm/memcontrol.c | 111 +++++++++++++++---------------------- mm/mlock.c | 21 ++----- mm/mmzone.c | 2 +- mm/page_alloc.c | 2 +- mm/swap.c | 99 ++++++++++++++++++--------------- mm/swap_state.c | 2 +- mm/vmscan.c | 55 +++++++++--------- 11 files changed, 171 insertions(+), 249 deletions(-) diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index 8f37c46f8..6c7c2c5f6 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -403,7 +403,8 @@ void mem_cgroup_commit_charge(struct page *page, struct mem_cgroup *memcg, bool lrucare, bool compound); void mem_cgroup_cancel_charge(struct page *page, struct mem_cgroup *memcg, bool compound); -int mem_cgroup_charge(struct page *page, struct mm_struct *mm, gfp_t gfp_mask); +int mem_cgroup_charge(struct page *page, struct mm_struct *mm, gfp_t gfp_mask, + bool lrucare); void mem_cgroup_uncharge(struct page *page); void mem_cgroup_uncharge_list(struct list_head *page_list); @@ -456,19 +457,6 @@ struct mem_cgroup *get_mem_cgroup_from_mm(struct mm_struct *mm); struct mem_cgroup *get_mem_cgroup_from_page(struct page *page); -struct lruvec *lock_page_lruvec(struct page *page); -struct lruvec *lock_page_lruvec_irq(struct page *page); -struct lruvec *lock_page_lruvec_irqsave(struct page *page, - unsigned long *flags); - -#ifdef CONFIG_DEBUG_VM -void lruvec_memcg_debug(struct lruvec *lruvec, struct page *page); -#else -static inline void lruvec_memcg_debug(struct lruvec *lruvec, struct page *page) -{ -} -#endif - static inline struct mem_cgroup *mem_cgroup_from_css(struct cgroup_subsys_state *css){ return css ? container_of(css, struct mem_cgroup, css) : NULL; @@ -975,31 +963,6 @@ static inline void mem_cgroup_put(struct mem_cgroup *memcg) { } -static inline struct lruvec *lock_page_lruvec(struct page *page) -{ - struct pglist_data *pgdat = page_pgdat(page); - - spin_lock(&pgdat->__lruvec.lru_lock); - return &pgdat->__lruvec; -} - -static inline struct lruvec *lock_page_lruvec_irq(struct page *page) -{ - struct pglist_data *pgdat = page_pgdat(page); - - spin_lock_irq(&pgdat->__lruvec.lru_lock); - return &pgdat->__lruvec; -} - -static inline struct lruvec *lock_page_lruvec_irqsave(struct page *page, - unsigned long *flagsp) -{ - struct pglist_data *pgdat = page_pgdat(page); - - spin_lock_irqsave(&pgdat->__lruvec.lru_lock, *flagsp); - return &pgdat->__lruvec; -} - static inline struct mem_cgroup * mem_cgroup_iter(struct mem_cgroup *root, struct mem_cgroup *prev, @@ -1234,10 +1197,6 @@ static inline void count_memcg_event_mm(struct mm_struct *mm, enum vm_event_item idx) { } - -static inline void lruvec_memcg_debug(struct lruvec *lruvec, struct page *page) -{ -} #endif /* CONFIG_MEMCG */ /* idx can be of type enum memcg_stat_item or node_stat_item */ @@ -1354,22 +1313,6 @@ static inline void dec_lruvec_page_state(struct page *page, mod_lruvec_page_state(page, idx, -1); } -static inline void unlock_page_lruvec(struct lruvec *lruvec) -{ - spin_unlock(&lruvec->lru_lock); -} - -static inline void unlock_page_lruvec_irq(struct lruvec *lruvec) -{ - spin_unlock_irq(&lruvec->lru_lock); -} - -static inline void unlock_page_lruvec_irqrestore(struct lruvec *lruvec, - unsigned long flags) -{ - spin_unlock_irqrestore(&lruvec->lru_lock, flags); -} - #ifdef CONFIG_CGROUP_WRITEBACK struct wb_domain *mem_cgroup_wb_domain(struct bdi_writeback *wb); diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index d93f9bef6..9598f80ee 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -297,8 +297,6 @@ struct zone_reclaim_stat { }; struct lruvec { - /* per lruvec lru_lock for memcg */ - spinlock_t lru_lock; struct list_head lists[NR_LRU_LISTS]; struct zone_reclaim_stat reclaim_stat; /* Evictions & activations on the inactive file list */ @@ -770,6 +768,7 @@ typedef struct pglist_data { /* Write-intensive fields used by page reclaim */ ZONE_PADDING(_pad1_) + spinlock_t lru_lock; #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT /* diff --git a/mm/compaction.c b/mm/compaction.c index 873094586..3da2952d1 100644 --- a/mm/compaction.c +++ b/mm/compaction.c @@ -786,7 +786,7 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn, unsigned long nr_scanned = 0, nr_isolated = 0; struct lruvec *lruvec; unsigned long flags = 0; - struct lruvec *locked = NULL; + bool locked = false; struct page *page = NULL, *valid_page = NULL; unsigned long start_pfn = low_pfn; bool skip_on_failure = false; @@ -846,17 +846,11 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn, * contention, to give chance to IRQs. Abort completely if * a fatal signal is pending. */ - if (!(low_pfn % SWAP_CLUSTER_MAX)) { - if (locked) { - unlock_page_lruvec_irqrestore(locked, flags); - locked = NULL; - } - if (fatal_signal_pending(current)) { - cc->contended = true; - low_pfn = 0; - goto fatal_pending; - } - cond_resched(); + if (!(low_pfn % SWAP_CLUSTER_MAX) + && compact_unlock_should_abort(&pgdat->lru_lock, + flags, &locked, cc)) { + low_pfn = 0; + goto fatal_pending; } if (!pfn_valid_within(low_pfn)) @@ -927,8 +921,9 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn, if (unlikely(__PageMovable(page)) && !PageIsolated(page)) { if (locked) { - unlock_page_lruvec_irqrestore(locked, flags); - locked = NULL; + spin_unlock_irqrestore(&pgdat->lru_lock, + flags); + locked = false; } if (!isolate_movable_page(page, isolate_mode)) @@ -969,18 +964,10 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn, if (!TestClearPageLRU(page)) goto isolate_fail_put; - rcu_read_lock(); - lruvec = mem_cgroup_page_lruvec(page, pgdat); - /* If we already hold the lock, we can skip some rechecking */ - if (lruvec != locked) { - if (locked) - unlock_page_lruvec_irqrestore(locked, flags); - compact_lock_irqsave(&lruvec->lru_lock, &flags, cc); - locked = lruvec; - rcu_read_unlock(); - - lruvec_memcg_debug(lruvec, page); + if (!locked) { + locked = compact_lock_irqsave(&pgdat->lru_lock, + &flags, cc); /* Try get exclusive access under lock */ if (!skip_updated) { @@ -999,8 +986,9 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn, SetPageLRU(page); goto isolate_fail_put; } - } else - rcu_read_unlock(); + } + + lruvec = mem_cgroup_page_lruvec(page, pgdat); VM_BUG_ON_PAGE(PageCompound(page), page); @@ -1030,8 +1018,8 @@ isolate_success: isolate_fail_put: /* Avoid potential deadlock in freeing page under lru_lock */ if (locked) { - unlock_page_lruvec_irqrestore(locked, flags); - locked = NULL; + spin_unlock_irqrestore(&pgdat->lru_lock, flags); + locked = false; } put_page(page); @@ -1046,8 +1034,8 @@ isolate_fail: */ if (nr_isolated) { if (locked) { - unlock_page_lruvec_irqrestore(locked, flags); - locked = NULL; + spin_unlock_irqrestore(&pgdat->lru_lock, flags); + locked = false; } putback_movable_pages(&cc->migratepages); cc->nr_migratepages = 0; @@ -1074,7 +1062,7 @@ isolate_fail: isolate_abort: if (locked) - unlock_page_lruvec_irqrestore(locked, flags); + spin_unlock_irqrestore(&pgdat->lru_lock, flags); if (page) { SetPageLRU(page); put_page(page); @@ -1114,7 +1102,8 @@ fatal_pending: * Otherwise, function returns one-past-the-last PFN of isolated page * (which may be greater than end_pfn if end fell in a middle of a THP page). */ -unsigned long isolate_migratepages_range(struct compact_control *cc, unsigned long start_pfn, +unsigned long +isolate_migratepages_range(struct compact_control *cc, unsigned long start_pfn, unsigned long end_pfn) { unsigned long pfn, block_start_pfn, block_end_pfn; diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 2d902ce12..77b507021 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -2542,6 +2542,7 @@ static void __split_huge_page(struct page *page, struct list_head *list, pgoff_t end) { struct page *head = compound_head(page); + pg_data_t *pgdat = page_pgdat(head); struct lruvec *lruvec; struct address_space *swap_cache = NULL; unsigned long offset = 0; @@ -2557,8 +2558,10 @@ static void __split_huge_page(struct page *page, struct list_head *list, swap_cache = swap_address_space(entry); xa_lock(&swap_cache->i_pages); } - /* lock lru list/PageCompound, ref freezed by page_ref_freeze */ - lruvec = lock_page_lruvec(head); + /* prevent PageLRU to go away from under us, and freeze lru stats */ + spin_lock(&pgdat->lru_lock); + + lruvec = mem_cgroup_page_lruvec(head, pgdat); for (i = HPAGE_PMD_NR - 1; i >= 1; i--) { __split_huge_page_tail(head, i, lruvec, list); @@ -2579,7 +2582,7 @@ static void __split_huge_page(struct page *page, struct list_head *list, } ClearPageCompound(head); - unlock_page_lruvec(lruvec); + spin_unlock(&pgdat->lru_lock); /* Caller disabled irqs, so they are still disabled here */ split_page_owner(head, HPAGE_PMD_NR); diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 58862b280..865589fb0 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -1236,19 +1236,6 @@ int mem_cgroup_scan_tasks(struct mem_cgroup *memcg, return ret; } -#ifdef CONFIG_DEBUG_VM -void lruvec_memcg_debug(struct lruvec *lruvec, struct page *page) -{ - if (mem_cgroup_disabled()) - return; - - if (!page->mem_cgroup) - VM_BUG_ON_PAGE(lruvec_memcg(lruvec) != root_mem_cgroup, page); - else - VM_BUG_ON_PAGE(lruvec_memcg(lruvec) != page->mem_cgroup, page); -} -#endif - /** * mem_cgroup_page_lruvec - return lruvec for isolating/putting an LRU page * @page: the page @@ -1287,51 +1274,6 @@ out: return lruvec; } -struct lruvec *lock_page_lruvec(struct page *page) -{ - struct lruvec *lruvec; - struct pglist_data *pgdat = page_pgdat(page); - - rcu_read_lock(); - lruvec = mem_cgroup_page_lruvec(page, pgdat); - spin_lock(&lruvec->lru_lock); - rcu_read_unlock(); - - lruvec_memcg_debug(lruvec, page); - - return lruvec; -} - -struct lruvec *lock_page_lruvec_irq(struct page *page) -{ - struct lruvec *lruvec; - struct pglist_data *pgdat = page_pgdat(page); - - rcu_read_lock(); - lruvec = mem_cgroup_page_lruvec(page, pgdat); - spin_lock_irq(&lruvec->lru_lock); - rcu_read_unlock(); - - lruvec_memcg_debug(lruvec, page); - - return lruvec; -} - -struct lruvec *lock_page_lruvec_irqsave(struct page *page, unsigned long *flags) -{ - struct lruvec *lruvec; - struct pglist_data *pgdat = page_pgdat(page); - - rcu_read_lock(); - lruvec = mem_cgroup_page_lruvec(page, pgdat); - spin_lock_irqsave(&lruvec->lru_lock, *flags); - rcu_read_unlock(); - - lruvec_memcg_debug(lruvec, page); - - return lruvec; -} - /** * mem_cgroup_update_lru_size - account for adding or removing an lru page * @lruvec: mem_cgroup per zone lru vector @@ -2794,14 +2736,50 @@ static void cancel_charge(struct mem_cgroup *memcg, unsigned int nr_pages) css_put_many(&memcg->css, nr_pages); } -static void commit_charge(struct page *page, struct mem_cgroup *memcg) +static void lock_page_lru(struct page *page, int *isolated) { + pg_data_t *pgdat = page_pgdat(page); + + spin_lock_irq(&pgdat->lru_lock); + if (PageLRU(page)) { + struct lruvec *lruvec; + + lruvec = mem_cgroup_page_lruvec(page, pgdat); + ClearPageLRU(page); + del_page_from_lru_list(page, lruvec, page_lru(page)); + *isolated = 1; + } else + *isolated = 0; +} + +static void unlock_page_lru(struct page *page, int isolated) +{ + pg_data_t *pgdat = page_pgdat(page); + + if (isolated) { + struct lruvec *lruvec; + + lruvec = mem_cgroup_page_lruvec(page, pgdat); + VM_BUG_ON_PAGE(PageLRU(page), page); + SetPageLRU(page); + add_page_to_lru_list(page, lruvec, page_lru(page)); + } + spin_unlock_irq(&pgdat->lru_lock); +} + +static void commit_charge(struct page *page, struct mem_cgroup *memcg, + bool lrucare) +{ + int isolated; + VM_BUG_ON_PAGE(page->mem_cgroup, page); /* * In some cases, SwapCache and FUSE(splice_buf->radixtree), the page * may already be on some other mem_cgroup's LRU. Take care of it. */ + if (lrucare) + lock_page_lru(page, &isolated); /* * Nobody should be changing or seriously looking at @@ -2818,6 +2796,9 @@ static void commit_charge(struct page *page, struct mem_cgroup *memcg) * have the page locked */ page->mem_cgroup = memcg; + + if (lrucare) + unlock_page_lru(page, isolated); } #ifdef CONFIG_MEMCG_KMEM @@ -7251,12 +7232,12 @@ int mem_cgroup_try_charge_delay(struct page *page, struct mm_struct *mm, * Use mem_cgroup_cancel_charge() to cancel the transaction instead. */ void mem_cgroup_commit_charge(struct page *page, struct mem_cgroup *memcg, - bool lruvec, bool compound) + bool lrucare, bool compound) { unsigned int nr_pages = compound ? hpage_nr_pages(page) : 1; VM_BUG_ON_PAGE(!page->mapping, page); - VM_BUG_ON_PAGE(PageLRU(page), page); + VM_BUG_ON_PAGE(PageLRU(page) && !lrucare, page); if (mem_cgroup_disabled()) return; @@ -7268,7 +7249,7 @@ void mem_cgroup_commit_charge(struct page *page, struct mem_cgroup *memcg, if (!memcg) return; - commit_charge(page, memcg); + commit_charge(page, memcg, lrucare); local_irq_disable(); mem_cgroup_charge_statistics(memcg, page, compound, nr_pages); @@ -7312,7 +7293,7 @@ void mem_cgroup_cancel_charge(struct page *page, struct mem_cgroup *memcg, cancel_charge(memcg, nr_pages); } -int mem_cgroup_charge(struct page *page, struct mm_struct *mm, gfp_t gfp_mask) +int mem_cgroup_charge(struct page *page, struct mm_struct *mm, gfp_t gfp_mask, bool lruvec) { struct mem_cgroup *memcg; int ret; @@ -7322,7 +7303,7 @@ int mem_cgroup_charge(struct page *page, struct mm_struct *mm, gfp_t gfp_mask) ret = mem_cgroup_try_charge(page, mm, gfp_mask, &memcg, false); if (ret) return ret; - mem_cgroup_commit_charge(page, memcg, false, false); + mem_cgroup_commit_charge(page, memcg, lruvec, false); return 0; } @@ -7525,7 +7506,7 @@ void mem_cgroup_migrate(struct page *oldpage, struct page *newpage) page_counter_charge(&memcg->memsw, nr_pages); css_get_many(&memcg->css, nr_pages); - commit_charge(newpage, memcg); + commit_charge(newpage, memcg, false); local_irq_save(flags); mem_cgroup_charge_statistics(memcg, newpage, compound, nr_pages); diff --git a/mm/mlock.c b/mm/mlock.c index 6b1c2c229..ba8d7e63c 100644 --- a/mm/mlock.c +++ b/mm/mlock.c @@ -258,12 +258,12 @@ static void __munlock_pagevec(struct pagevec *pvec, struct zone *zone) int nr = pagevec_count(pvec); int delta_munlocked = -nr; struct pagevec pvec_putback; - struct lruvec *lruvec = NULL; int pgrescued = 0; pagevec_init(&pvec_putback); /* Phase 1: page isolation */ + spin_lock_irq(&zone->zone_pgdat->lru_lock); for (i = 0; i < nr; i++) { struct page *page = pvec->pages[i]; @@ -273,15 +273,10 @@ static void __munlock_pagevec(struct pagevec *pvec, struct zone *zone) * so we can spare the get_page() here. */ if (TestClearPageLRU(page)) { - struct lruvec *new_lruvec; + struct lruvec *lruvec; - new_lruvec = mem_cgroup_page_lruvec(page, - page_pgdat(page)); - if (new_lruvec != lruvec) { - if (lruvec) - unlock_page_lruvec_irq(lruvec); - lruvec = lock_page_lruvec_irq(page); - } + lruvec = mem_cgroup_page_lruvec(page, + page_pgdat(page)); del_page_from_lru_list(page, lruvec, page_lru(page)); continue; @@ -300,12 +295,8 @@ static void __munlock_pagevec(struct pagevec *pvec, struct zone *zone) pagevec_add(&pvec_putback, pvec->pages[i]); pvec->pages[i] = NULL; } - if (lruvec) { - __mod_zone_page_state(zone, NR_MLOCK, delta_munlocked); - unlock_page_lruvec_irq(lruvec); - } else if (delta_munlocked) { - mod_zone_page_state(zone, NR_MLOCK, delta_munlocked); - } + __mod_zone_page_state(zone, NR_MLOCK, delta_munlocked); + spin_unlock_irq(&zone->zone_pgdat->lru_lock); /* Now we can release pins of pages that we are not munlocking */ pagevec_release(&pvec_putback); diff --git a/mm/mmzone.c b/mm/mmzone.c index b3c4a4e44..4686fdc23 100644 --- a/mm/mmzone.c +++ b/mm/mmzone.c @@ -91,7 +91,7 @@ void lruvec_init(struct lruvec *lruvec) enum lru_list lru; memset(lruvec, 0, sizeof(struct lruvec)); - spin_lock_init(&lruvec->lru_lock); + for_each_lru(lru) INIT_LIST_HEAD(&lruvec->lists[lru]); } diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 57f9ed21e..f0ec43034 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -6741,7 +6741,7 @@ static void __meminit pgdat_init_internals(struct pglist_data *pgdat) init_waitqueue_head(&pgdat->pfmemalloc_wait); pgdat_page_ext_init(pgdat); - + spin_lock_init(&pgdat->lru_lock); lruvec_init(node_lruvec(pgdat)); } diff --git a/mm/swap.c b/mm/swap.c index e39fd3ef6..53ca6f15f 100644 --- a/mm/swap.c +++ b/mm/swap.c @@ -60,14 +60,16 @@ static DEFINE_PER_CPU(struct pagevec, activate_page_pvecs); static void __page_cache_release(struct page *page) { if (PageLRU(page)) { + pg_data_t *pgdat = page_pgdat(page); struct lruvec *lruvec; unsigned long flags; - lruvec = lock_page_lruvec_irqsave(page, &flags); + spin_lock_irqsave(&pgdat->lru_lock, flags); + lruvec = mem_cgroup_page_lruvec(page, pgdat); VM_BUG_ON_PAGE(!PageLRU(page), page); __ClearPageLRU(page); del_page_from_lru_list(page, lruvec, page_off_lru(page)); - unlock_page_lruvec_irqrestore(lruvec, flags); + spin_unlock_irqrestore(&pgdat->lru_lock, flags); } __ClearPageWaiters(page); } @@ -189,28 +191,30 @@ static void pagevec_lru_move_fn(struct pagevec *pvec, void (*move_fn)(struct page *page, struct lruvec *lruvec)) { int i; - struct lruvec *lruvec = NULL; + struct pglist_data *pgdat = NULL; + struct lruvec *lruvec; unsigned long flags = 0; for (i = 0; i < pagevec_count(pvec); i++) { struct page *page = pvec->pages[i]; - struct lruvec *new_lruvec; + struct pglist_data *pagepgdat = page_pgdat(page); + if (pagepgdat != pgdat) { + if (pgdat) + spin_unlock_irqrestore(&pgdat->lru_lock, flags); + pgdat = pagepgdat; + spin_lock_irqsave(&pgdat->lru_lock, flags); + } /* block memcg migration during page moving between lru */ if (!TestClearPageLRU(page)) continue; - new_lruvec = mem_cgroup_page_lruvec(page, page_pgdat(page)); - if (lruvec != new_lruvec) { - if (lruvec) - unlock_page_lruvec_irqrestore(lruvec, flags); - lruvec = lock_page_lruvec_irqsave(page, &flags); - } + lruvec = mem_cgroup_page_lruvec(page, pgdat); (*move_fn)(page, lruvec); SetPageLRU(page); } - if (lruvec) - unlock_page_lruvec_irqrestore(lruvec, flags); + if (pgdat) + spin_unlock_irqrestore(&pgdat->lru_lock, flags); release_pages(pvec->pages, pvec->nr); pagevec_reinit(pvec); } @@ -308,13 +312,13 @@ static inline void activate_page_drain(int cpu) void activate_page(struct page *page) { - struct lruvec *lruvec; + pg_data_t *pgdat = page_pgdat(page); page = compound_head(page); - lruvec = lock_page_lruvec_irq(page); + spin_lock_irq(&pgdat->lru_lock); if (PageLRU(page)) - __activate_page(page, lruvec); - unlock_page_lruvec_irq(lruvec); + __activate_page(page, mem_cgroup_page_lruvec(page, pgdat)); + spin_unlock_irq(&pgdat->lru_lock); } #endif @@ -741,7 +745,7 @@ void release_pages(struct page **pages, int nr) int i; LIST_HEAD(pages_to_free); struct pglist_data *locked_pgdat = NULL; - struct lruvec *lruvec = NULL; + struct lruvec *lruvec; unsigned long uninitialized_var(flags); unsigned int uninitialized_var(lock_batch); @@ -753,18 +757,19 @@ void release_pages(struct page **pages, int nr) * excessive with a continuous string of pages from the * same pgdat. The lock is held only if pgdat != NULL. */ - if (lruvec && ++lock_batch == SWAP_CLUSTER_MAX) { - unlock_page_lruvec_irqrestore(lruvec, flags); - lruvec = NULL; + if (locked_pgdat && ++lock_batch == SWAP_CLUSTER_MAX) { + spin_unlock_irqrestore(&locked_pgdat->lru_lock, flags); + locked_pgdat = NULL; } if (is_huge_zero_page(page)) continue; if (is_zone_device_page(page)) { - if (lruvec) { - unlock_page_lruvec_irqrestore(lruvec, flags); - lruvec = NULL; + if (locked_pgdat) { + spin_unlock_irqrestore(&locked_pgdat->lru_lock, + flags); + locked_pgdat = NULL; } /* * ZONE_DEVICE pages that return 'false' from @@ -781,25 +786,27 @@ void release_pages(struct page **pages, int nr) continue; if (PageCompound(page)) { - if (lruvec) { - unlock_page_lruvec_irqrestore(lruvec, flags); - lruvec = NULL; + if (locked_pgdat) { + spin_unlock_irqrestore(&locked_pgdat->lru_lock, flags); + locked_pgdat = NULL; } __put_compound_page(page); continue; } if (PageLRU(page)) { - struct lruvec *new_lruvec; + struct pglist_data *pgdat = page_pgdat(page); - new_lruvec = mem_cgroup_page_lruvec(page, page_pgdat(page)); - if (new_lruvec != lruvec) { - if (lruvec) - unlock_page_lruvec_irqrestore(lruvec, flags); + if (pgdat != locked_pgdat) { + if (locked_pgdat) + spin_unlock_irqrestore(&locked_pgdat->lru_lock, + flags); lock_batch = 0; - lruvec = lock_page_lruvec_irqsave(page, &flags); + locked_pgdat = pgdat; + spin_lock_irqsave(&locked_pgdat->lru_lock, flags); } + lruvec = mem_cgroup_page_lruvec(page, locked_pgdat); VM_BUG_ON_PAGE(!PageLRU(page), page); __ClearPageLRU(page); del_page_from_lru_list(page, lruvec, page_off_lru(page)); @@ -811,12 +818,14 @@ void release_pages(struct page **pages, int nr) list_add(&page->lru, &pages_to_free); } - if (lruvec) - unlock_page_lruvec_irqrestore(lruvec, flags); + if (locked_pgdat) + spin_unlock_irqrestore(&locked_pgdat->lru_lock, flags); + mem_cgroup_uncharge_list(&pages_to_free); free_unref_page_list(&pages_to_free); } EXPORT_SYMBOL(release_pages); + /* * The pages which we're about to release may be in the deferred lru-addition * queues. That would prevent them from really being freed right now. That's @@ -848,7 +857,7 @@ void lru_add_page_tail(struct page *page, struct page *page_tail, VM_BUG_ON_PAGE(!PageHead(page), page); VM_BUG_ON_PAGE(PageCompound(page_tail), page); VM_BUG_ON_PAGE(PageLRU(page_tail), page); - lockdep_assert_held(&lruvec->lru_lock); + lockdep_assert_held(&lruvec_pgdat(lruvec)->lru_lock); if (list) { /* page reclaim is reclaiming a huge page */ @@ -928,24 +937,26 @@ static void __pagevec_lru_add_fn(struct page *page, struct lruvec *lruvec) void __pagevec_lru_add(struct pagevec *pvec) { int i; - struct lruvec *lruvec = NULL; + struct pglist_data *pgdat = NULL; + struct lruvec *lruvec; unsigned long flags = 0; for (i = 0; i < pagevec_count(pvec); i++) { struct page *page = pvec->pages[i]; - struct lruvec *new_lruvec; + struct pglist_data *pagepgdat = page_pgdat(page); - new_lruvec = mem_cgroup_page_lruvec(page, page_pgdat(page)); - if (lruvec != new_lruvec) { - if (lruvec) - unlock_page_lruvec_irqrestore(lruvec, flags); - lruvec = lock_page_lruvec_irqsave(page, &flags); + if (pagepgdat != pgdat) { + if (pgdat) + spin_unlock_irqrestore(&pgdat->lru_lock, flags); + pgdat = pagepgdat; + spin_lock_irqsave(&pgdat->lru_lock, flags); } + lruvec = mem_cgroup_page_lruvec(page, pgdat); __pagevec_lru_add_fn(page, lruvec); } - if (lruvec) - unlock_page_lruvec_irqrestore(lruvec, flags); + if (pgdat) + spin_unlock_irqrestore(&pgdat->lru_lock, flags); release_pages(pvec->pages, pvec->nr); pagevec_reinit(pvec); } diff --git a/mm/swap_state.c b/mm/swap_state.c index b3f7587d5..8e8d9b06e 100644 --- a/mm/swap_state.c +++ b/mm/swap_state.c @@ -435,7 +435,7 @@ struct page *__read_swap_cache_async(swp_entry_t entry, gfp_t gfp_mask, goto fail_unlock; } - if (mem_cgroup_charge(page, NULL, gfp_mask)) { + if (mem_cgroup_charge(page, NULL, gfp_mask, false)) { delete_from_swap_cache(page); goto fail_unlock; } diff --git a/mm/vmscan.c b/mm/vmscan.c index 3adf652af..cb18fe439 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -1824,13 +1824,15 @@ int isolate_lru_page(struct page *page) WARN_RATELIMIT(PageTail(page), "trying to isolate tail page"); if (TestClearPageLRU(page)) { + pg_data_t *pgdat = page_pgdat(page); struct lruvec *lruvec; get_page(page); + lruvec = mem_cgroup_page_lruvec(page, pgdat); - lruvec = lock_page_lruvec_irq(page); + spin_lock_irq(&pgdat->lru_lock); del_page_from_lru_list(page, lruvec, page_lru(page)); - unlock_page_lruvec_irq(lruvec); + spin_unlock_irq(&pgdat->lru_lock); ret = 0; } return ret; @@ -1896,6 +1898,7 @@ static int too_many_isolated(struct pglist_data *pgdat, int file, static unsigned noinline_for_stack move_pages_to_lru(struct lruvec *lruvec, struct list_head *list) { + struct pglist_data *pgdat = lruvec_pgdat(lruvec); int nr_pages, nr_moved = 0; LIST_HEAD(pages_to_free); struct page *page; @@ -1906,9 +1909,9 @@ static unsigned noinline_for_stack move_pages_to_lru(struct lruvec *lruvec, VM_BUG_ON_PAGE(PageLRU(page), page); list_del(&page->lru); if (unlikely(!page_evictable(page))) { - spin_unlock_irq(&lruvec->lru_lock); + spin_unlock_irq(&pgdat->lru_lock); putback_lru_page(page); - spin_lock_irq(&lruvec->lru_lock); + spin_lock_irq(&pgdat->lru_lock); continue; } /* The SetPageLRU needs to be kept here for list integrity. @@ -1929,9 +1932,9 @@ static unsigned noinline_for_stack move_pages_to_lru(struct lruvec *lruvec, __ClearPageActive(page); if (unlikely(PageCompound(page))) { - spin_unlock_irq(&lruvec->lru_lock); + spin_unlock_irq(&pgdat->lru_lock); (*get_compound_page_dtor(page))(page); - spin_lock_irq(&lruvec->lru_lock); + spin_lock_irq(&pgdat->lru_lock); } else list_add(&page->lru, &pages_to_free); continue; @@ -2001,7 +2004,7 @@ shrink_inactive_list(unsigned long nr_to_scan, struct lruvec *lruvec, lru_add_drain(); - spin_lock_irq(&lruvec->lru_lock); + spin_lock_irq(&pgdat->lru_lock); nr_taken = isolate_lru_pages(nr_to_scan, lruvec, &page_list, &nr_scanned, sc, lru); @@ -2013,7 +2016,7 @@ shrink_inactive_list(unsigned long nr_to_scan, struct lruvec *lruvec, if (global_reclaim(sc)) __count_vm_events(item, nr_scanned); __count_memcg_events(lruvec_memcg(lruvec), item, nr_scanned); - spin_unlock_irq(&lruvec->lru_lock); + spin_unlock_irq(&pgdat->lru_lock); if (nr_taken == 0) return 0; @@ -2021,7 +2024,7 @@ shrink_inactive_list(unsigned long nr_to_scan, struct lruvec *lruvec, nr_reclaimed = shrink_page_list(&page_list, pgdat, sc, 0, &stat, false); - spin_lock_irq(&lruvec->lru_lock); + spin_lock_irq(&pgdat->lru_lock); item = current_is_kswapd() ? PGSTEAL_KSWAPD : PGSTEAL_DIRECT; if (global_reclaim(sc)) @@ -2034,7 +2037,7 @@ shrink_inactive_list(unsigned long nr_to_scan, struct lruvec *lruvec, __mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, -nr_taken); - spin_unlock_irq(&lruvec->lru_lock); + spin_unlock_irq(&pgdat->lru_lock); mem_cgroup_uncharge_list(&page_list); free_unref_page_list(&page_list); @@ -2087,7 +2090,7 @@ static void shrink_active_list(unsigned long nr_to_scan, lru_add_drain(); - spin_lock_irq(&lruvec->lru_lock); + spin_lock_irq(&pgdat->lru_lock); nr_taken = isolate_lru_pages(nr_to_scan, lruvec, &l_hold, &nr_scanned, sc, lru); @@ -2098,7 +2101,7 @@ static void shrink_active_list(unsigned long nr_to_scan, __count_vm_events(PGREFILL, nr_scanned); __count_memcg_events(lruvec_memcg(lruvec), PGREFILL, nr_scanned); - spin_unlock_irq(&lruvec->lru_lock); + spin_unlock_irq(&pgdat->lru_lock); while (!list_empty(&l_hold)) { cond_resched(); @@ -2144,7 +2147,7 @@ static void shrink_active_list(unsigned long nr_to_scan, /* * Move pages back to the lru list. */ - spin_lock_irq(&lruvec->lru_lock); + spin_lock_irq(&pgdat->lru_lock); /* * Count referenced pages from currently used mappings as rotated, * even though only some of them are actually re-activated. This @@ -2162,7 +2165,7 @@ static void shrink_active_list(unsigned long nr_to_scan, __count_memcg_events(lruvec_memcg(lruvec), PGDEACTIVATE, nr_deactivate); __mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, -nr_taken); - spin_unlock_irq(&lruvec->lru_lock); + spin_unlock_irq(&pgdat->lru_lock); mem_cgroup_uncharge_list(&l_active); free_unref_page_list(&l_active); @@ -2454,7 +2457,7 @@ static void get_scan_count(struct lruvec *lruvec, struct mem_cgroup *memcg, file = lruvec_lru_size(lruvec, LRU_ACTIVE_FILE, MAX_NR_ZONES) + lruvec_lru_size(lruvec, LRU_INACTIVE_FILE, MAX_NR_ZONES); - spin_lock_irq(&lruvec->lru_lock); + spin_lock_irq(&pgdat->lru_lock); if (unlikely(reclaim_stat->recent_scanned[0] > anon / 4)) { reclaim_stat->recent_scanned[0] /= 2; reclaim_stat->recent_rotated[0] /= 2; @@ -2475,7 +2478,7 @@ static void get_scan_count(struct lruvec *lruvec, struct mem_cgroup *memcg, fp = file_prio * (reclaim_stat->recent_scanned[1] + 1); fp /= reclaim_stat->recent_rotated[1] + 1; - spin_unlock_irq(&lruvec->lru_lock); + spin_unlock_irq(&pgdat->lru_lock); fraction[0] = ap; fraction[1] = fp; @@ -4752,14 +4755,15 @@ int page_evictable(struct page *page) */ void check_move_unevictable_pages(struct pagevec *pvec) { - struct lruvec *lruvec = NULL; + struct lruvec *lruvec; + struct pglist_data *pgdat = NULL; int pgscanned = 0; int pgrescued = 0; int i; for (i = 0; i < pvec->nr; i++) { struct page *page = pvec->pages[i]; - struct lruvec *new_lruvec; + struct pglist_data *pagepgdat = page_pgdat(page); pgscanned++; @@ -4767,12 +4771,13 @@ void check_move_unevictable_pages(struct pagevec *pvec) if (!TestClearPageLRU(page)) continue; - new_lruvec = mem_cgroup_page_lruvec(page, page_pgdat(page)); - if (lruvec != new_lruvec) { - if (lruvec) - unlock_page_lruvec_irq(lruvec); - lruvec = lock_page_lruvec_irq(page); + if (pagepgdat != pgdat) { + if (pgdat) + spin_unlock_irq(&pgdat->lru_lock); + pgdat = pagepgdat; + spin_lock_irq(&pgdat->lru_lock); } + lruvec = mem_cgroup_page_lruvec(page, pgdat); if (page_evictable(page) && PageUnevictable(page)) { enum lru_list lru = page_lru_base_type(page); @@ -4786,10 +4791,10 @@ void check_move_unevictable_pages(struct pagevec *pvec) } } - if (lruvec) { + if (pgdat) { __count_vm_events(UNEVICTABLE_PGRESCUED, pgrescued); __count_vm_events(UNEVICTABLE_PGSCANNED, pgscanned); - unlock_page_lruvec_irq(lruvec); + spin_unlock_irq(&pgdat->lru_lock); } else if (pgscanned) { count_vm_events(UNEVICTABLE_PGSCANNED, pgscanned); }