diff --git a/mm/memcontrol.c b/mm/memcontrol.c index cc2e73f46..d2ba82559 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -63,7 +63,7 @@ #include #include #include "slab.h" - +#include #include #include @@ -2561,10 +2561,12 @@ void mem_cgroup_handle_over_high(void) unsigned long pflags; unsigned int nr_pages = current->memcg_nr_pages_over_high; struct mem_cgroup *memcg; + u64 start; if (likely(!nr_pages)) return; + sli_memlat_stat_start(&start); memcg = get_mem_cgroup_from_mm(current->mm); reclaim_high(memcg, nr_pages, GFP_KERNEL); current->memcg_nr_pages_over_high = 0; @@ -2594,6 +2596,7 @@ void mem_cgroup_handle_over_high(void) psi_memstall_leave(&pflags); out: + sli_memlat_stat_end(MEM_LAT_MEMCG_DIRECT_RECLAIM, start); css_put(&memcg->css); } @@ -2608,6 +2611,7 @@ static int try_charge(struct mem_cgroup *memcg, gfp_t gfp_mask, bool may_swap = true; bool drained = false; enum oom_status oom_status; + u64 start; if (mem_cgroup_is_root(memcg)) return 0; @@ -2667,9 +2671,11 @@ retry: memcg_memory_event(mem_over_limit, MEMCG_MAX); + sli_memlat_stat_start(&start); nr_reclaimed = try_to_free_mem_cgroup_pages(mem_over_limit, nr_pages, gfp_mask, may_swap); + sli_memlat_stat_end(MEM_LAT_MEMCG_DIRECT_RECLAIM, start); if (mem_cgroup_margin(mem_over_limit) >= nr_pages) goto retry; diff --git a/mm/memory.c b/mm/memory.c index 13a575ce2..465c6b6bb 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -79,6 +79,7 @@ #include #include #include +#include #include "internal.h" @@ -3958,8 +3959,15 @@ static vm_fault_t handle_pte_fault(struct vm_fault *vmf) return do_fault(vmf); } - if (!pte_present(vmf->orig_pte)) - return do_swap_page(vmf); + if (!pte_present(vmf->orig_pte)) { + vm_fault_t retval; + u64 start; + + sli_memlat_stat_start(&start); + retval = do_swap_page(vmf); + sli_memlat_stat_end(MEM_LAT_DIRECT_SWAPIN, start); + return retval; + } if (pte_protnone(vmf->orig_pte) && vma_is_accessible(vmf->vma)) return do_numa_page(vmf); diff --git a/mm/page_alloc.c b/mm/page_alloc.c index f94b9c0b1..e48e8c69c 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -69,6 +69,7 @@ #include #include #include +#include #include #include @@ -3908,17 +3909,19 @@ __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order, struct page *page = NULL; unsigned long pflags; unsigned int noreclaim_flag; + u64 start; if (!order) return NULL; psi_memstall_enter(&pflags); + sli_memlat_stat_start(&start); noreclaim_flag = memalloc_noreclaim_save(); - *compact_result = try_to_compact_pages(gfp_mask, order, alloc_flags, ac, prio, &page); memalloc_noreclaim_restore(noreclaim_flag); + sli_memlat_stat_end(MEM_LAT_DIRECT_COMPACT, start); psi_memstall_leave(&pflags); /* @@ -4129,12 +4132,14 @@ __perform_reclaim(gfp_t gfp_mask, unsigned int order, int progress; unsigned int noreclaim_flag; unsigned long pflags; + u64 start; cond_resched(); /* We now go into synchronous reclaim */ cpuset_memory_pressure_bump(); psi_memstall_enter(&pflags); + sli_memlat_stat_start(&start); fs_reclaim_acquire(gfp_mask); noreclaim_flag = memalloc_noreclaim_save(); @@ -4143,6 +4148,7 @@ __perform_reclaim(gfp_t gfp_mask, unsigned int order, memalloc_noreclaim_restore(noreclaim_flag); fs_reclaim_release(gfp_mask); + sli_memlat_stat_end(MEM_LAT_GLOBAL_DIRECT_RECLAIM, start); psi_memstall_leave(&pflags); cond_resched(); diff --git a/mm/shmem.c b/mm/shmem.c index 0fcd969ac..9699ecd1a 100644 --- a/mm/shmem.c +++ b/mm/shmem.c @@ -38,6 +38,7 @@ #include #include #include +#include #include /* for arch/microblaze update_mmu_cache() */ @@ -1649,6 +1650,7 @@ static int shmem_swapin_page(struct inode *inode, pgoff_t index, struct page *page; swp_entry_t swap; int error; + u64 start; VM_BUG_ON(!*pagep || !xa_is_value(*pagep)); swap = radix_to_swp_entry(*pagep); @@ -1664,7 +1666,9 @@ static int shmem_swapin_page(struct inode *inode, pgoff_t index, count_memcg_event_mm(charge_mm, PGMAJFAULT); } /* Here we actually start the io */ + sli_memlat_stat_start(&start); page = shmem_swapin(swap, gfp, info, index); + sli_memlat_stat_end(MEM_LAT_DIRECT_SWAPIN, start); if (!page) { error = -ENOMEM; goto failed; diff --git a/mm/vmscan.c b/mm/vmscan.c index 68a2b1caa..24e02c0a7 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -57,6 +57,7 @@ #include #include +#include #include "internal.h" @@ -839,6 +840,7 @@ typedef enum { static pageout_t pageout(struct page *page, struct address_space *mapping, struct scan_control *sc) { + u64 start; /* * If the page is dirty, only perform writeback if that write * will be non-blocking. To prevent this allocation from being @@ -887,7 +889,14 @@ static pageout_t pageout(struct page *page, struct address_space *mapping, }; SetPageReclaim(page); + if (!current_is_kswapd()) + sli_memlat_stat_start(&start); res = mapping->a_ops->writepage(page, &wbc); + if (!current_is_kswapd()) + sli_memlat_stat_end(global_reclaim(sc) ? + MEM_LAT_GLOBAL_DIRECT_SWAPOUT : + MEM_LAT_MEMCG_DIRECT_SWAPOUT, + start); if (res < 0) handle_write_error(mapping, page, res); if (res == AOP_WRITEPAGE_ACTIVATE) {