TencentOS
/
TencentOS-kernel
mirror of https://github.com/Tencent/TencentOS-kernel
0
0
Fork 4
Code Issues Projects Releases Wiki Activity

Revert " mm/lru: replace pgdat lru_lock with lruvec lock" 

This reverts commit 1b7478d8d0.
This commit is contained in:
Bin Lai 2022-09-05 20:56:02 +08:00
parent bb589f37a6
commit 6ef3bc24ee
11 changed files with 171 additions and 249 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

61
include/linux/memcontrol.h
View File

@ -403,7 +403,8 @@ void mem_cgroup_commit_charge(struct page *page, struct mem_cgroup *memcg,
bool lrucare, bool compound);
void mem_cgroup_cancel_charge(struct page *page, struct mem_cgroup *memcg,
bool compound);
int mem_cgroup_charge(struct page *page, struct mm_struct *mm, gfp_t gfp_mask);
int mem_cgroup_charge(struct page *page, struct mm_struct *mm, gfp_t gfp_mask,
bool lrucare);
void mem_cgroup_uncharge(struct page *page);
void mem_cgroup_uncharge_list(struct list_head *page_list);
@ -456,19 +457,6 @@ struct mem_cgroup *get_mem_cgroup_from_mm(struct mm_struct *mm);
struct mem_cgroup *get_mem_cgroup_from_page(struct page *page);
struct lruvec *lock_page_lruvec(struct page *page);
struct lruvec *lock_page_lruvec_irq(struct page *page);
struct lruvec *lock_page_lruvec_irqsave(struct page *page,
unsigned long *flags);
#ifdef CONFIG_DEBUG_VM
void lruvec_memcg_debug(struct lruvec *lruvec, struct page *page);
#else
static inline void lruvec_memcg_debug(struct lruvec *lruvec, struct page *page)
{
}
#endif
static inline
struct mem_cgroup *mem_cgroup_from_css(struct cgroup_subsys_state *css){
return css ? container_of(css, struct mem_cgroup, css) : NULL;
@ -975,31 +963,6 @@ static inline void mem_cgroup_put(struct mem_cgroup *memcg)
{
}
static inline struct lruvec *lock_page_lruvec(struct page *page)
{
struct pglist_data *pgdat = page_pgdat(page);
spin_lock(&pgdat->__lruvec.lru_lock);
return &pgdat->__lruvec;
}
static inline struct lruvec *lock_page_lruvec_irq(struct page *page)
{
struct pglist_data *pgdat = page_pgdat(page);
spin_lock_irq(&pgdat->__lruvec.lru_lock);
return &pgdat->__lruvec;
}
static inline struct lruvec *lock_page_lruvec_irqsave(struct page *page,
unsigned long *flagsp)
{
struct pglist_data *pgdat = page_pgdat(page);
spin_lock_irqsave(&pgdat->__lruvec.lru_lock, *flagsp);
return &pgdat->__lruvec;
}
static inline struct mem_cgroup *
mem_cgroup_iter(struct mem_cgroup *root,
struct mem_cgroup *prev,
@ -1234,10 +1197,6 @@ static inline
void count_memcg_event_mm(struct mm_struct *mm, enum vm_event_item idx)
{
}
static inline void lruvec_memcg_debug(struct lruvec *lruvec, struct page *page)
{
}
#endif /* CONFIG_MEMCG */
/* idx can be of type enum memcg_stat_item or node_stat_item */
@ -1354,22 +1313,6 @@ static inline void dec_lruvec_page_state(struct page *page,
mod_lruvec_page_state(page, idx, -1);
}
static inline void unlock_page_lruvec(struct lruvec *lruvec)
{
spin_unlock(&lruvec->lru_lock);
}
static inline void unlock_page_lruvec_irq(struct lruvec *lruvec)
{
spin_unlock_irq(&lruvec->lru_lock);
}
static inline void unlock_page_lruvec_irqrestore(struct lruvec *lruvec,
unsigned long flags)
{
spin_unlock_irqrestore(&lruvec->lru_lock, flags);
}
#ifdef CONFIG_CGROUP_WRITEBACK
struct wb_domain *mem_cgroup_wb_domain(struct bdi_writeback *wb);

3
include/linux/mmzone.h
View File

@ -297,8 +297,6 @@ struct zone_reclaim_stat {
};
struct lruvec {
/* per lruvec lru_lock for memcg */
spinlock_t lru_lock;
struct list_head lists[NR_LRU_LISTS];
struct zone_reclaim_stat reclaim_stat;
/* Evictions & activations on the inactive file list */
@ -770,6 +768,7 @@ typedef struct pglist_data {
/* Write-intensive fields used by page reclaim */
ZONE_PADDING(_pad1_)
spinlock_t lru_lock;
#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
/*

55
mm/compaction.c
View File

@ -786,7 +786,7 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn,
unsigned long nr_scanned = 0, nr_isolated = 0;
struct lruvec *lruvec;
unsigned long flags = 0;
struct lruvec *locked = NULL;
bool locked = false;
struct page *page = NULL, *valid_page = NULL;
unsigned long start_pfn = low_pfn;
bool skip_on_failure = false;
@ -846,17 +846,11 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn,
* contention, to give chance to IRQs. Abort completely if
* a fatal signal is pending.
*/
if (!(low_pfn % SWAP_CLUSTER_MAX)) {
if (locked) {
unlock_page_lruvec_irqrestore(locked, flags);
locked = NULL;
}
if (fatal_signal_pending(current)) {
cc->contended = true;
low_pfn = 0;
goto fatal_pending;
}
cond_resched();
if (!(low_pfn % SWAP_CLUSTER_MAX)
&& compact_unlock_should_abort(&pgdat->lru_lock,
flags, &locked, cc)) {
low_pfn = 0;
goto fatal_pending;
}
if (!pfn_valid_within(low_pfn))
@ -927,8 +921,9 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn,
if (unlikely(__PageMovable(page)) &&
!PageIsolated(page)) {
if (locked) {
unlock_page_lruvec_irqrestore(locked, flags);
locked = NULL;
spin_unlock_irqrestore(&pgdat->lru_lock,
flags);
locked = false;
}
if (!isolate_movable_page(page, isolate_mode))
@ -969,18 +964,10 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn,
if (!TestClearPageLRU(page))
goto isolate_fail_put;
rcu_read_lock();
lruvec = mem_cgroup_page_lruvec(page, pgdat);
/* If we already hold the lock, we can skip some rechecking */
if (lruvec != locked) {
if (locked)
unlock_page_lruvec_irqrestore(locked, flags);
compact_lock_irqsave(&lruvec->lru_lock, &flags, cc);
locked = lruvec;
rcu_read_unlock();
lruvec_memcg_debug(lruvec, page);
if (!locked) {
locked = compact_lock_irqsave(&pgdat->lru_lock,
&flags, cc);
/* Try get exclusive access under lock */
if (!skip_updated) {
@ -999,8 +986,9 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn,
SetPageLRU(page);
goto isolate_fail_put;
}
} else
rcu_read_unlock();
}
lruvec = mem_cgroup_page_lruvec(page, pgdat);
VM_BUG_ON_PAGE(PageCompound(page), page);
@ -1030,8 +1018,8 @@ isolate_success:
isolate_fail_put:
/* Avoid potential deadlock in freeing page under lru_lock */
if (locked) {
unlock_page_lruvec_irqrestore(locked, flags);
locked = NULL;
spin_unlock_irqrestore(&pgdat->lru_lock, flags);
locked = false;
}
put_page(page);
@ -1046,8 +1034,8 @@ isolate_fail:
*/
if (nr_isolated) {
if (locked) {
unlock_page_lruvec_irqrestore(locked, flags);
locked = NULL;
spin_unlock_irqrestore(&pgdat->lru_lock, flags);
locked = false;
}
putback_movable_pages(&cc->migratepages);
cc->nr_migratepages = 0;
@ -1074,7 +1062,7 @@ isolate_fail:
isolate_abort:
if (locked)
unlock_page_lruvec_irqrestore(locked, flags);
spin_unlock_irqrestore(&pgdat->lru_lock, flags);
if (page) {
SetPageLRU(page);
put_page(page);
@ -1114,7 +1102,8 @@ fatal_pending:
* Otherwise, function returns one-past-the-last PFN of isolated page
* (which may be greater than end_pfn if end fell in a middle of a THP page).
*/
unsigned long isolate_migratepages_range(struct compact_control *cc, unsigned long start_pfn,
unsigned long
isolate_migratepages_range(struct compact_control *cc, unsigned long start_pfn,
unsigned long end_pfn)
{
unsigned long pfn, block_start_pfn, block_end_pfn;

9
mm/huge_memory.c
View File

@ -2542,6 +2542,7 @@ static void __split_huge_page(struct page *page, struct list_head *list,
pgoff_t end)
{
struct page *head = compound_head(page);
pg_data_t *pgdat = page_pgdat(head);
struct lruvec *lruvec;
struct address_space *swap_cache = NULL;
unsigned long offset = 0;
@ -2557,8 +2558,10 @@ static void __split_huge_page(struct page *page, struct list_head *list,
swap_cache = swap_address_space(entry);
xa_lock(&swap_cache->i_pages);
}
/* lock lru list/PageCompound, ref freezed by page_ref_freeze */
lruvec = lock_page_lruvec(head);
/* prevent PageLRU to go away from under us, and freeze lru stats */
spin_lock(&pgdat->lru_lock);
lruvec = mem_cgroup_page_lruvec(head, pgdat);
for (i = HPAGE_PMD_NR - 1; i >= 1; i--) {
__split_huge_page_tail(head, i, lruvec, list);
@ -2579,7 +2582,7 @@ static void __split_huge_page(struct page *page, struct list_head *list,
}
ClearPageCompound(head);
unlock_page_lruvec(lruvec);
spin_unlock(&pgdat->lru_lock);
/* Caller disabled irqs, so they are still disabled here */
split_page_owner(head, HPAGE_PMD_NR);

111
mm/memcontrol.c
View File

@ -1236,19 +1236,6 @@ int mem_cgroup_scan_tasks(struct mem_cgroup *memcg,
return ret;
}
#ifdef CONFIG_DEBUG_VM
void lruvec_memcg_debug(struct lruvec *lruvec, struct page *page)
{
if (mem_cgroup_disabled())
return;
if (!page->mem_cgroup)
VM_BUG_ON_PAGE(lruvec_memcg(lruvec) != root_mem_cgroup, page);
else
VM_BUG_ON_PAGE(lruvec_memcg(lruvec) != page->mem_cgroup, page);
}
#endif
/**
* mem_cgroup_page_lruvec - return lruvec for isolating/putting an LRU page
* @page: the page
@ -1287,51 +1274,6 @@ out:
return lruvec;
}
struct lruvec *lock_page_lruvec(struct page *page)
{
struct lruvec *lruvec;
struct pglist_data *pgdat = page_pgdat(page);
rcu_read_lock();
lruvec = mem_cgroup_page_lruvec(page, pgdat);
spin_lock(&lruvec->lru_lock);
rcu_read_unlock();
lruvec_memcg_debug(lruvec, page);
return lruvec;
}
struct lruvec *lock_page_lruvec_irq(struct page *page)
{
struct lruvec *lruvec;
struct pglist_data *pgdat = page_pgdat(page);
rcu_read_lock();
lruvec = mem_cgroup_page_lruvec(page, pgdat);
spin_lock_irq(&lruvec->lru_lock);
rcu_read_unlock();
lruvec_memcg_debug(lruvec, page);
return lruvec;
}
struct lruvec *lock_page_lruvec_irqsave(struct page *page, unsigned long *flags)
{
struct lruvec *lruvec;
struct pglist_data *pgdat = page_pgdat(page);
rcu_read_lock();
lruvec = mem_cgroup_page_lruvec(page, pgdat);
spin_lock_irqsave(&lruvec->lru_lock, *flags);
rcu_read_unlock();
lruvec_memcg_debug(lruvec, page);
return lruvec;
}
/**
* mem_cgroup_update_lru_size - account for adding or removing an lru page
* @lruvec: mem_cgroup per zone lru vector
@ -2794,14 +2736,50 @@ static void cancel_charge(struct mem_cgroup *memcg, unsigned int nr_pages)
css_put_many(&memcg->css, nr_pages);
}
static void commit_charge(struct page *page, struct mem_cgroup *memcg)
static void lock_page_lru(struct page *page, int *isolated)
{
pg_data_t *pgdat = page_pgdat(page);
spin_lock_irq(&pgdat->lru_lock);
if (PageLRU(page)) {
struct lruvec *lruvec;
lruvec = mem_cgroup_page_lruvec(page, pgdat);
ClearPageLRU(page);
del_page_from_lru_list(page, lruvec, page_lru(page));
*isolated = 1;
} else
*isolated = 0;
}
static void unlock_page_lru(struct page *page, int isolated)
{
pg_data_t *pgdat = page_pgdat(page);
if (isolated) {
struct lruvec *lruvec;
lruvec = mem_cgroup_page_lruvec(page, pgdat);
VM_BUG_ON_PAGE(PageLRU(page), page);
SetPageLRU(page);
add_page_to_lru_list(page, lruvec, page_lru(page));
}
spin_unlock_irq(&pgdat->lru_lock);
}
static void commit_charge(struct page *page, struct mem_cgroup *memcg,
bool lrucare)
{
int isolated;
VM_BUG_ON_PAGE(page->mem_cgroup, page);
/*
* In some cases, SwapCache and FUSE(splice_buf->radixtree), the page
* may already be on some other mem_cgroup's LRU. Take care of it.
*/
if (lrucare)
lock_page_lru(page, &isolated);
/*
* Nobody should be changing or seriously looking at
@ -2818,6 +2796,9 @@ static void commit_charge(struct page *page, struct mem_cgroup *memcg)
* have the page locked
*/
page->mem_cgroup = memcg;
if (lrucare)
unlock_page_lru(page, isolated);
}
#ifdef CONFIG_MEMCG_KMEM
@ -7251,12 +7232,12 @@ int mem_cgroup_try_charge_delay(struct page *page, struct mm_struct *mm,
* Use mem_cgroup_cancel_charge() to cancel the transaction instead.
*/
void mem_cgroup_commit_charge(struct page *page, struct mem_cgroup *memcg,
bool lruvec, bool compound)
bool lrucare, bool compound)
{
unsigned int nr_pages = compound ? hpage_nr_pages(page) : 1;
VM_BUG_ON_PAGE(!page->mapping, page);
VM_BUG_ON_PAGE(PageLRU(page), page);
VM_BUG_ON_PAGE(PageLRU(page) && !lrucare, page);
if (mem_cgroup_disabled())
return;
@ -7268,7 +7249,7 @@ void mem_cgroup_commit_charge(struct page *page, struct mem_cgroup *memcg,
if (!memcg)
return;
commit_charge(page, memcg);
commit_charge(page, memcg, lrucare);
local_irq_disable();
mem_cgroup_charge_statistics(memcg, page, compound, nr_pages);
@ -7312,7 +7293,7 @@ void mem_cgroup_cancel_charge(struct page *page, struct mem_cgroup *memcg,
cancel_charge(memcg, nr_pages);
}
int mem_cgroup_charge(struct page *page, struct mm_struct *mm, gfp_t gfp_mask)
int mem_cgroup_charge(struct page *page, struct mm_struct *mm, gfp_t gfp_mask, bool lruvec)
{
struct mem_cgroup *memcg;
int ret;
@ -7322,7 +7303,7 @@ int mem_cgroup_charge(struct page *page, struct mm_struct *mm, gfp_t gfp_mask)
ret = mem_cgroup_try_charge(page, mm, gfp_mask, &memcg, false);
if (ret)
return ret;
mem_cgroup_commit_charge(page, memcg, false, false);
mem_cgroup_commit_charge(page, memcg, lruvec, false);
return 0;
}
@ -7525,7 +7506,7 @@ void mem_cgroup_migrate(struct page *oldpage, struct page *newpage)
page_counter_charge(&memcg->memsw, nr_pages);
css_get_many(&memcg->css, nr_pages);
commit_charge(newpage, memcg);
commit_charge(newpage, memcg, false);
local_irq_save(flags);
mem_cgroup_charge_statistics(memcg, newpage, compound, nr_pages);

21
mm/mlock.c
View File

@ -258,12 +258,12 @@ static void __munlock_pagevec(struct pagevec *pvec, struct zone *zone)
int nr = pagevec_count(pvec);
int delta_munlocked = -nr;
struct pagevec pvec_putback;
struct lruvec *lruvec = NULL;
int pgrescued = 0;
pagevec_init(&pvec_putback);
/* Phase 1: page isolation */
spin_lock_irq(&zone->zone_pgdat->lru_lock);
for (i = 0; i < nr; i++) {
struct page *page = pvec->pages[i];
@ -273,15 +273,10 @@ static void __munlock_pagevec(struct pagevec *pvec, struct zone *zone)
* so we can spare the get_page() here.
*/
if (TestClearPageLRU(page)) {
struct lruvec *new_lruvec;
struct lruvec *lruvec;
new_lruvec = mem_cgroup_page_lruvec(page,
page_pgdat(page));
if (new_lruvec != lruvec) {
if (lruvec)
unlock_page_lruvec_irq(lruvec);
lruvec = lock_page_lruvec_irq(page);
}
lruvec = mem_cgroup_page_lruvec(page,
page_pgdat(page));
del_page_from_lru_list(page, lruvec,
page_lru(page));
continue;
@ -300,12 +295,8 @@ static void __munlock_pagevec(struct pagevec *pvec, struct zone *zone)
pagevec_add(&pvec_putback, pvec->pages[i]);
pvec->pages[i] = NULL;
}
if (lruvec) {
__mod_zone_page_state(zone, NR_MLOCK, delta_munlocked);
unlock_page_lruvec_irq(lruvec);
} else if (delta_munlocked) {
mod_zone_page_state(zone, NR_MLOCK, delta_munlocked);
}
__mod_zone_page_state(zone, NR_MLOCK, delta_munlocked);
spin_unlock_irq(&zone->zone_pgdat->lru_lock);
/* Now we can release pins of pages that we are not munlocking */
pagevec_release(&pvec_putback);

2
mm/mmzone.c
View File

@ -91,7 +91,7 @@ void lruvec_init(struct lruvec *lruvec)
enum lru_list lru;
memset(lruvec, 0, sizeof(struct lruvec));
spin_lock_init(&lruvec->lru_lock);
for_each_lru(lru)
INIT_LIST_HEAD(&lruvec->lists[lru]);
}

2
mm/page_alloc.c
View File

@ -6741,7 +6741,7 @@ static void __meminit pgdat_init_internals(struct pglist_data *pgdat)
init_waitqueue_head(&pgdat->pfmemalloc_wait);
pgdat_page_ext_init(pgdat);
spin_lock_init(&pgdat->lru_lock);
lruvec_init(node_lruvec(pgdat));
}

99
mm/swap.c
View File

@ -60,14 +60,16 @@ static DEFINE_PER_CPU(struct pagevec, activate_page_pvecs);
static void __page_cache_release(struct page *page)
{
if (PageLRU(page)) {
pg_data_t *pgdat = page_pgdat(page);
struct lruvec *lruvec;
unsigned long flags;
lruvec = lock_page_lruvec_irqsave(page, &flags);
spin_lock_irqsave(&pgdat->lru_lock, flags);
lruvec = mem_cgroup_page_lruvec(page, pgdat);
VM_BUG_ON_PAGE(!PageLRU(page), page);
__ClearPageLRU(page);
del_page_from_lru_list(page, lruvec, page_off_lru(page));
unlock_page_lruvec_irqrestore(lruvec, flags);
spin_unlock_irqrestore(&pgdat->lru_lock, flags);
}
__ClearPageWaiters(page);
}
@ -189,28 +191,30 @@ static void pagevec_lru_move_fn(struct pagevec *pvec,
void (*move_fn)(struct page *page, struct lruvec *lruvec))
{
int i;
struct lruvec *lruvec = NULL;
struct pglist_data *pgdat = NULL;
struct lruvec *lruvec;
unsigned long flags = 0;
for (i = 0; i < pagevec_count(pvec); i++) {
struct page *page = pvec->pages[i];
struct lruvec *new_lruvec;
struct pglist_data *pagepgdat = page_pgdat(page);
if (pagepgdat != pgdat) {
if (pgdat)
spin_unlock_irqrestore(&pgdat->lru_lock, flags);
pgdat = pagepgdat;
spin_lock_irqsave(&pgdat->lru_lock, flags);
}
/* block memcg migration during page moving between lru */
if (!TestClearPageLRU(page))
continue;
new_lruvec = mem_cgroup_page_lruvec(page, page_pgdat(page));
if (lruvec != new_lruvec) {
if (lruvec)
unlock_page_lruvec_irqrestore(lruvec, flags);
lruvec = lock_page_lruvec_irqsave(page, &flags);
}
lruvec = mem_cgroup_page_lruvec(page, pgdat);
(*move_fn)(page, lruvec);
SetPageLRU(page);
}
if (lruvec)
unlock_page_lruvec_irqrestore(lruvec, flags);
if (pgdat)
spin_unlock_irqrestore(&pgdat->lru_lock, flags);
release_pages(pvec->pages, pvec->nr);
pagevec_reinit(pvec);
}
@ -308,13 +312,13 @@ static inline void activate_page_drain(int cpu)
void activate_page(struct page *page)
{
struct lruvec *lruvec;
pg_data_t *pgdat = page_pgdat(page);
page = compound_head(page);
lruvec = lock_page_lruvec_irq(page);
spin_lock_irq(&pgdat->lru_lock);
if (PageLRU(page))
__activate_page(page, lruvec);
unlock_page_lruvec_irq(lruvec);
__activate_page(page, mem_cgroup_page_lruvec(page, pgdat));
spin_unlock_irq(&pgdat->lru_lock);
}
#endif
@ -741,7 +745,7 @@ void release_pages(struct page **pages, int nr)
int i;
LIST_HEAD(pages_to_free);
struct pglist_data *locked_pgdat = NULL;
struct lruvec *lruvec = NULL;
struct lruvec *lruvec;
unsigned long uninitialized_var(flags);
unsigned int uninitialized_var(lock_batch);
@ -753,18 +757,19 @@ void release_pages(struct page **pages, int nr)
* excessive with a continuous string of pages from the
* same pgdat. The lock is held only if pgdat != NULL.
*/
if (lruvec && ++lock_batch == SWAP_CLUSTER_MAX) {
unlock_page_lruvec_irqrestore(lruvec, flags);
lruvec = NULL;
if (locked_pgdat && ++lock_batch == SWAP_CLUSTER_MAX) {
spin_unlock_irqrestore(&locked_pgdat->lru_lock, flags);
locked_pgdat = NULL;
}
if (is_huge_zero_page(page))
continue;
if (is_zone_device_page(page)) {
if (lruvec) {
unlock_page_lruvec_irqrestore(lruvec, flags);
lruvec = NULL;
if (locked_pgdat) {
spin_unlock_irqrestore(&locked_pgdat->lru_lock,
flags);
locked_pgdat = NULL;
}
/*
* ZONE_DEVICE pages that return 'false' from
@ -781,25 +786,27 @@ void release_pages(struct page **pages, int nr)
continue;
if (PageCompound(page)) {
if (lruvec) {
unlock_page_lruvec_irqrestore(lruvec, flags);
lruvec = NULL;
if (locked_pgdat) {
spin_unlock_irqrestore(&locked_pgdat->lru_lock, flags);
locked_pgdat = NULL;
}
__put_compound_page(page);
continue;
}
if (PageLRU(page)) {
struct lruvec *new_lruvec;
struct pglist_data *pgdat = page_pgdat(page);
new_lruvec = mem_cgroup_page_lruvec(page, page_pgdat(page));
if (new_lruvec != lruvec) {
if (lruvec)
unlock_page_lruvec_irqrestore(lruvec, flags);
if (pgdat != locked_pgdat) {
if (locked_pgdat)
spin_unlock_irqrestore(&locked_pgdat->lru_lock,
flags);
lock_batch = 0;
lruvec = lock_page_lruvec_irqsave(page, &flags);
locked_pgdat = pgdat;
spin_lock_irqsave(&locked_pgdat->lru_lock, flags);
}
lruvec = mem_cgroup_page_lruvec(page, locked_pgdat);
VM_BUG_ON_PAGE(!PageLRU(page), page);
__ClearPageLRU(page);
del_page_from_lru_list(page, lruvec, page_off_lru(page));
@ -811,12 +818,14 @@ void release_pages(struct page **pages, int nr)
list_add(&page->lru, &pages_to_free);
}
if (lruvec)
unlock_page_lruvec_irqrestore(lruvec, flags);
if (locked_pgdat)
spin_unlock_irqrestore(&locked_pgdat->lru_lock, flags);
mem_cgroup_uncharge_list(&pages_to_free);
free_unref_page_list(&pages_to_free);
}
EXPORT_SYMBOL(release_pages);
/*
* The pages which we're about to release may be in the deferred lru-addition
* queues. That would prevent them from really being freed right now. That's
@ -848,7 +857,7 @@ void lru_add_page_tail(struct page *page, struct page *page_tail,
VM_BUG_ON_PAGE(!PageHead(page), page);
VM_BUG_ON_PAGE(PageCompound(page_tail), page);
VM_BUG_ON_PAGE(PageLRU(page_tail), page);
lockdep_assert_held(&lruvec->lru_lock);
lockdep_assert_held(&lruvec_pgdat(lruvec)->lru_lock);
if (list) {
/* page reclaim is reclaiming a huge page */
@ -928,24 +937,26 @@ static void __pagevec_lru_add_fn(struct page *page, struct lruvec *lruvec)
void __pagevec_lru_add(struct pagevec *pvec)
{
int i;
struct lruvec *lruvec = NULL;
struct pglist_data *pgdat = NULL;
struct lruvec *lruvec;
unsigned long flags = 0;
for (i = 0; i < pagevec_count(pvec); i++) {
struct page *page = pvec->pages[i];
struct lruvec *new_lruvec;
struct pglist_data *pagepgdat = page_pgdat(page);
new_lruvec = mem_cgroup_page_lruvec(page, page_pgdat(page));
if (lruvec != new_lruvec) {
if (lruvec)
unlock_page_lruvec_irqrestore(lruvec, flags);
lruvec = lock_page_lruvec_irqsave(page, &flags);
if (pagepgdat != pgdat) {
if (pgdat)
spin_unlock_irqrestore(&pgdat->lru_lock, flags);
pgdat = pagepgdat;
spin_lock_irqsave(&pgdat->lru_lock, flags);
}
lruvec = mem_cgroup_page_lruvec(page, pgdat);
__pagevec_lru_add_fn(page, lruvec);
}
if (lruvec)
unlock_page_lruvec_irqrestore(lruvec, flags);
if (pgdat)
spin_unlock_irqrestore(&pgdat->lru_lock, flags);
release_pages(pvec->pages, pvec->nr);
pagevec_reinit(pvec);
}

2
mm/swap_state.c
View File

@ -435,7 +435,7 @@ struct page *__read_swap_cache_async(swp_entry_t entry, gfp_t gfp_mask,
goto fail_unlock;
}
if (mem_cgroup_charge(page, NULL, gfp_mask)) {
if (mem_cgroup_charge(page, NULL, gfp_mask, false)) {
delete_from_swap_cache(page);
goto fail_unlock;
}

55
mm/vmscan.c
View File

@ -1824,13 +1824,15 @@ int isolate_lru_page(struct page *page)
WARN_RATELIMIT(PageTail(page), "trying to isolate tail page");
if (TestClearPageLRU(page)) {
pg_data_t *pgdat = page_pgdat(page);
struct lruvec *lruvec;
get_page(page);
lruvec = mem_cgroup_page_lruvec(page, pgdat);
lruvec = lock_page_lruvec_irq(page);
spin_lock_irq(&pgdat->lru_lock);
del_page_from_lru_list(page, lruvec, page_lru(page));
unlock_page_lruvec_irq(lruvec);
spin_unlock_irq(&pgdat->lru_lock);
ret = 0;
}
return ret;
@ -1896,6 +1898,7 @@ static int too_many_isolated(struct pglist_data *pgdat, int file,
static unsigned noinline_for_stack move_pages_to_lru(struct lruvec *lruvec,
struct list_head *list)
{
struct pglist_data *pgdat = lruvec_pgdat(lruvec);
int nr_pages, nr_moved = 0;
LIST_HEAD(pages_to_free);
struct page *page;
@ -1906,9 +1909,9 @@ static unsigned noinline_for_stack move_pages_to_lru(struct lruvec *lruvec,
VM_BUG_ON_PAGE(PageLRU(page), page);
list_del(&page->lru);
if (unlikely(!page_evictable(page))) {
spin_unlock_irq(&lruvec->lru_lock);
spin_unlock_irq(&pgdat->lru_lock);
putback_lru_page(page);
spin_lock_irq(&lruvec->lru_lock);
spin_lock_irq(&pgdat->lru_lock);
continue;
}
/* The SetPageLRU needs to be kept here for list integrity.
@ -1929,9 +1932,9 @@ static unsigned noinline_for_stack move_pages_to_lru(struct lruvec *lruvec,
__ClearPageActive(page);
if (unlikely(PageCompound(page))) {
spin_unlock_irq(&lruvec->lru_lock);
spin_unlock_irq(&pgdat->lru_lock);
(*get_compound_page_dtor(page))(page);
spin_lock_irq(&lruvec->lru_lock);
spin_lock_irq(&pgdat->lru_lock);
} else
list_add(&page->lru, &pages_to_free);
continue;
@ -2001,7 +2004,7 @@ shrink_inactive_list(unsigned long nr_to_scan, struct lruvec *lruvec,
lru_add_drain();
spin_lock_irq(&lruvec->lru_lock);
spin_lock_irq(&pgdat->lru_lock);
nr_taken = isolate_lru_pages(nr_to_scan, lruvec, &page_list,
&nr_scanned, sc, lru);
@ -2013,7 +2016,7 @@ shrink_inactive_list(unsigned long nr_to_scan, struct lruvec *lruvec,
if (global_reclaim(sc))
__count_vm_events(item, nr_scanned);
__count_memcg_events(lruvec_memcg(lruvec), item, nr_scanned);
spin_unlock_irq(&lruvec->lru_lock);
spin_unlock_irq(&pgdat->lru_lock);
if (nr_taken == 0)
return 0;
@ -2021,7 +2024,7 @@ shrink_inactive_list(unsigned long nr_to_scan, struct lruvec *lruvec,
nr_reclaimed = shrink_page_list(&page_list, pgdat, sc, 0,
&stat, false);
spin_lock_irq(&lruvec->lru_lock);
spin_lock_irq(&pgdat->lru_lock);
item = current_is_kswapd() ? PGSTEAL_KSWAPD : PGSTEAL_DIRECT;
if (global_reclaim(sc))
@ -2034,7 +2037,7 @@ shrink_inactive_list(unsigned long nr_to_scan, struct lruvec *lruvec,
__mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, -nr_taken);
spin_unlock_irq(&lruvec->lru_lock);
spin_unlock_irq(&pgdat->lru_lock);
mem_cgroup_uncharge_list(&page_list);
free_unref_page_list(&page_list);
@ -2087,7 +2090,7 @@ static void shrink_active_list(unsigned long nr_to_scan,
lru_add_drain();
spin_lock_irq(&lruvec->lru_lock);
spin_lock_irq(&pgdat->lru_lock);
nr_taken = isolate_lru_pages(nr_to_scan, lruvec, &l_hold,
&nr_scanned, sc, lru);
@ -2098,7 +2101,7 @@ static void shrink_active_list(unsigned long nr_to_scan,
__count_vm_events(PGREFILL, nr_scanned);
__count_memcg_events(lruvec_memcg(lruvec), PGREFILL, nr_scanned);
spin_unlock_irq(&lruvec->lru_lock);
spin_unlock_irq(&pgdat->lru_lock);
while (!list_empty(&l_hold)) {
cond_resched();
@ -2144,7 +2147,7 @@ static void shrink_active_list(unsigned long nr_to_scan,
/*
* Move pages back to the lru list.
*/
spin_lock_irq(&lruvec->lru_lock);
spin_lock_irq(&pgdat->lru_lock);
/*
* Count referenced pages from currently used mappings as rotated,
* even though only some of them are actually re-activated. This
@ -2162,7 +2165,7 @@ static void shrink_active_list(unsigned long nr_to_scan,
__count_memcg_events(lruvec_memcg(lruvec), PGDEACTIVATE, nr_deactivate);
__mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, -nr_taken);
spin_unlock_irq(&lruvec->lru_lock);
spin_unlock_irq(&pgdat->lru_lock);
mem_cgroup_uncharge_list(&l_active);
free_unref_page_list(&l_active);
@ -2454,7 +2457,7 @@ static void get_scan_count(struct lruvec *lruvec, struct mem_cgroup *memcg,
file = lruvec_lru_size(lruvec, LRU_ACTIVE_FILE, MAX_NR_ZONES) +
lruvec_lru_size(lruvec, LRU_INACTIVE_FILE, MAX_NR_ZONES);
spin_lock_irq(&lruvec->lru_lock);
spin_lock_irq(&pgdat->lru_lock);
if (unlikely(reclaim_stat->recent_scanned[0] > anon / 4)) {
reclaim_stat->recent_scanned[0] /= 2;
reclaim_stat->recent_rotated[0] /= 2;
@ -2475,7 +2478,7 @@ static void get_scan_count(struct lruvec *lruvec, struct mem_cgroup *memcg,
fp = file_prio * (reclaim_stat->recent_scanned[1] + 1);
fp /= reclaim_stat->recent_rotated[1] + 1;
spin_unlock_irq(&lruvec->lru_lock);
spin_unlock_irq(&pgdat->lru_lock);
fraction[0] = ap;
fraction[1] = fp;
@ -4752,14 +4755,15 @@ int page_evictable(struct page *page)
*/
void check_move_unevictable_pages(struct pagevec *pvec)
{
struct lruvec *lruvec = NULL;
struct lruvec *lruvec;
struct pglist_data *pgdat = NULL;
int pgscanned = 0;
int pgrescued = 0;
int i;
for (i = 0; i < pvec->nr; i++) {
struct page *page = pvec->pages[i];
struct lruvec *new_lruvec;
struct pglist_data *pagepgdat = page_pgdat(page);
pgscanned++;
@ -4767,12 +4771,13 @@ void check_move_unevictable_pages(struct pagevec *pvec)
if (!TestClearPageLRU(page))
continue;
new_lruvec = mem_cgroup_page_lruvec(page, page_pgdat(page));
if (lruvec != new_lruvec) {
if (lruvec)
unlock_page_lruvec_irq(lruvec);
lruvec = lock_page_lruvec_irq(page);
if (pagepgdat != pgdat) {
if (pgdat)
spin_unlock_irq(&pgdat->lru_lock);
pgdat = pagepgdat;
spin_lock_irq(&pgdat->lru_lock);
}
lruvec = mem_cgroup_page_lruvec(page, pgdat);
if (page_evictable(page) && PageUnevictable(page)) {
enum lru_list lru = page_lru_base_type(page);
@ -4786,10 +4791,10 @@ void check_move_unevictable_pages(struct pagevec *pvec)
}
}
if (lruvec) {
if (pgdat) {
__count_vm_events(UNEVICTABLE_PGRESCUED, pgrescued);
__count_vm_events(UNEVICTABLE_PGSCANNED, pgscanned);
unlock_page_lruvec_irq(lruvec);
spin_unlock_irq(&pgdat->lru_lock);
} else if (pgscanned) {
count_vm_events(UNEVICTABLE_PGSCANNED, pgscanned);
}