From 3ad771dafc2cc7d097319d8be12521e6bfebbdef Mon Sep 17 00:00:00 2001 From: johnnyaiai Date: Mon, 10 Oct 2022 15:12:54 +0800 Subject: [PATCH 1/7] config: Disable CONFIG_LATENCYTOP by default Performance degradation due to multi-core contending for global spinlock. Signed-off-by: johnnyaiai --- package/default/config.default | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package/default/config.default b/package/default/config.default index 943c7700e..be250d03a 100644 --- a/package/default/config.default +++ b/package/default/config.default @@ -4995,7 +4995,7 @@ CONFIG_RCU_CPU_STALL_TIMEOUT=60 # CONFIG_NOTIFIER_ERROR_INJECTION is not set CONFIG_FUNCTION_ERROR_INJECTION=y # CONFIG_FAULT_INJECTION is not set -CONFIG_LATENCYTOP=y +# CONFIG_LATENCYTOP is not set CONFIG_USER_STACKTRACE_SUPPORT=y CONFIG_NOP_TRACER=y CONFIG_HAVE_FUNCTION_TRACER=y From e16b1d92b5bc5e0f2df49d922bdbbe9eb7b2376d Mon Sep 17 00:00:00 2001 From: Jianping Liu Date: Wed, 18 May 2022 20:09:51 +0800 Subject: [PATCH 2/7] open CONFIG_JUMP_LABEL to optimize very unlikely/likely branches This option enables a transparent branch optimization that makes certain almost-always-true or almost-always-false branch conditions even cheaper to execute within the kernel. Signed-off-by: johnnyaiai --- package/default/config.default | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/package/default/config.default b/package/default/config.default index be250d03a..220c013fb 100644 --- a/package/default/config.default +++ b/package/default/config.default @@ -700,7 +700,8 @@ CONFIG_OPROFILE_EVENT_MULTIPLEX=y CONFIG_HAVE_OPROFILE=y CONFIG_OPROFILE_NMI_TIMER=y CONFIG_KPROBES=y -# CONFIG_JUMP_LABEL is not set +CONFIG_JUMP_LABEL=y +# CONFIG_STATIC_KEYS_SELFTEST is not set CONFIG_OPTPROBES=y CONFIG_KPROBES_ON_FTRACE=y CONFIG_UPROBES=y From a0507babe277dc0ec537eeee0a1c992aac5fbd7f Mon Sep 17 00:00:00 2001 From: Hyeonggon Yoo <42.hyeyoo@gmail.com> Date: Fri, 5 Nov 2021 13:35:24 -0700 Subject: [PATCH 3/7] mm, slub: use prefetchw instead of prefetch commit 04b4b006139b58ffbd90df3befecc13711b1faf8 upstream. Commit 0ad9500e16fe ("slub: prefetch next freelist pointer in slab_alloc()") introduced prefetch_freepointer() because when other cpu(s) freed objects into a page that current cpu owns, the freelist link is hot on cpu(s) which freed objects and possibly very cold on current cpu. But if freelist link chain is hot on cpu(s) which freed objects, it's better to invalidate that chain because they're not going to access again within a short time. So use prefetchw instead of prefetch. On supported architectures like x86 and arm, it invalidates other copied instances of a cache line when prefetching it. Before: Time: 91.677 Performance counter stats for 'hackbench -g 100 -l 10000': 1462938.07 msec cpu-clock # 15.908 CPUs utilized 18072550 context-switches # 12.354 K/sec 1018814 cpu-migrations # 696.416 /sec 104558 page-faults # 71.471 /sec 1580035699271 cycles # 1.080 GHz (54.51%) 2003670016013 instructions # 1.27 insn per cycle (54.31%) 5702204863 branch-misses (54.28%) 643368500985 cache-references # 439.778 M/sec (54.26%) 18475582235 cache-misses # 2.872 % of all cache refs (54.28%) 642206796636 L1-dcache-loads # 438.984 M/sec (46.87%) 18215813147 L1-dcache-load-misses # 2.84% of all L1-dcache accesses (46.83%) 653842996501 dTLB-loads # 446.938 M/sec (46.63%) 3227179675 dTLB-load-misses # 0.49% of all dTLB cache accesses (46.85%) 537531951350 iTLB-loads # 367.433 M/sec (54.33%) 114750630 iTLB-load-misses # 0.02% of all iTLB cache accesses (54.37%) 630135543177 L1-icache-loads # 430.733 M/sec (46.80%) 22923237620 L1-icache-load-misses # 3.64% of all L1-icache accesses (46.76%) 91.964452802 seconds time elapsed 43.416742000 seconds user 1422.441123000 seconds sys After: Time: 90.220 Performance counter stats for 'hackbench -g 100 -l 10000': 1437418.48 msec cpu-clock # 15.880 CPUs utilized 17694068 context-switches # 12.310 K/sec 958257 cpu-migrations # 666.651 /sec 100604 page-faults # 69.989 /sec 1583259429428 cycles # 1.101 GHz (54.57%) 2004002484935 instructions # 1.27 insn per cycle (54.37%) 5594202389 branch-misses (54.36%) 643113574524 cache-references # 447.409 M/sec (54.39%) 18233791870 cache-misses # 2.835 % of all cache refs (54.37%) 640205852062 L1-dcache-loads # 445.386 M/sec (46.75%) 17968160377 L1-dcache-load-misses # 2.81% of all L1-dcache accesses (46.79%) 651747432274 dTLB-loads # 453.415 M/sec (46.59%) 3127124271 dTLB-load-misses # 0.48% of all dTLB cache accesses (46.75%) 535395273064 iTLB-loads # 372.470 M/sec (54.38%) 113500056 iTLB-load-misses # 0.02% of all iTLB cache accesses (54.35%) 628871845924 L1-icache-loads # 437.501 M/sec (46.80%) 22585641203 L1-icache-load-misses # 3.59% of all L1-icache accesses (46.79%) 90.514819303 seconds time elapsed 43.877656000 seconds user 1397.176001000 seconds sys [xuyu]: The latency of 'hackbench -g 100 -l 10000' in a x86 guest with 8 CPU and 16G memory is 92.81 (w/o this path) and 91.2018 (w/ this patch), respectively. Link: https://lkml.org/lkml/2021/10/8/598=20 Link: https://lkml.kernel.org/r/20211011144331.70084-1-42.hyeyoo@gmail.com Signed-off-by: Hyeonggon Yoo <42.hyeyoo@gmail.com> Acked-by: Vlastimil Babka Cc: Christoph Lameter Cc: Pekka Enberg Cc: David Rientjes Cc: Joonsoo Kim Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds Signed-off-by: Xu Yu Reviewed-by: Gang Deng Signed-off-by: johnnyaiai --- mm/slub.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/slub.c b/mm/slub.c index 52ded855b..06726e181 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -282,7 +282,7 @@ static inline void *get_freepointer(struct kmem_cache *s, void *object) static void prefetch_freepointer(const struct kmem_cache *s, void *object) { - prefetch(object + s->offset); + prefetchw(object + s->offset); } static inline void *get_freepointer_safe(struct kmem_cache *s, void *object) From 38f49fe4f478fd9813e287c875c0bc9aaac2a439 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Wed, 11 Dec 2019 14:10:35 -0700 Subject: [PATCH 4/7] fs: move filp_close() outside of __close_fd_get_file() upstream commit: 6e802a4ba056a6f2f51ac9d54eead3ed6f9829a2 Just one caller of this, and just use filp_close() there manually. This is important to allow async close/removal of the fd. Signed-off-by: Jens Axboe Signed-off-by: johnnyaiai --- drivers/android/binder.c | 6 ++++-- fs/file.c | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/drivers/android/binder.c b/drivers/android/binder.c index 89b590c95..08f6b7b01 100644 --- a/drivers/android/binder.c +++ b/drivers/android/binder.c @@ -2232,10 +2232,12 @@ static void binder_deferred_fd_close(int fd) return; init_task_work(&twcb->twork, binder_do_fd_close); __close_fd_get_file(fd, &twcb->file); - if (twcb->file) + if (twcb->file) { + filp_close(twcb->file, current->files); task_work_add(current, &twcb->twork, true); - else + } else { kfree(twcb); + } } static void binder_transaction_buffer_release(struct binder_proc *proc, diff --git a/fs/file.c b/fs/file.c index e5d328335..b597b07a9 100644 --- a/fs/file.c +++ b/fs/file.c @@ -642,7 +642,9 @@ out_unlock: EXPORT_SYMBOL(__close_fd); /* for ksys_close() */ /* - * variant of __close_fd that gets a ref on the file for later fput + * variant of __close_fd that gets a ref on the file for later fput. + * The caller must ensure that filp_close() called on the file, and then + * an fput(). */ int __close_fd_get_file(unsigned int fd, struct file **res) { @@ -662,7 +664,7 @@ int __close_fd_get_file(unsigned int fd, struct file **res) spin_unlock(&files->file_lock); get_file(file); *res = file; - return filp_close(file, files); + return 0; out_unlock: spin_unlock(&files->file_lock); From e877f6f4935739637b69793056176af67f25cb6d Mon Sep 17 00:00:00 2001 From: Linus Torvalds Date: Fri, 14 Jan 2022 19:29:15 +0800 Subject: [PATCH 5/7] fget: check that the fd still exists after getting a ref to it commit 054aa8d439b9185d4f5eb9a90282d1ce74772969 upstream. Jann Horn points out that there is another possible race wrt Unix domain socket garbage collection, somewhat reminiscent of the one fixed in commit cbcf01128d0a ("af_unix: fix garbage collect vs MSG_PEEK"). See the extended comment about the garbage collection requirements added to unix_peek_fds() by that commit for details. The race comes from how we can locklessly look up a file descriptor just as it is in the process of being closed, and with the right artificial timing (Jann added a few strategic 'mdelay(500)' calls to do that), the Unix domain socket garbage collector could see the reference count decrement of the close() happen before fget() took its reference to the file and the file was attached onto a new file descriptor. This is all (intentionally) correct on the 'struct file *' side, with RCU lookups and lockless reference counting very much part of the design. Getting that reference count out of order isn't a problem per se. But the garbage collector can get confused by seeing this situation of having seen a file not having any remaining external references and then seeing it being attached to an fd. In commit cbcf01128d0a ("af_unix: fix garbage collect vs MSG_PEEK") the fix was to serialize the file descriptor install with the garbage collector by taking and releasing the unix_gc_lock. That's not really an option here, but since this all happens when we are in the process of looking up a file descriptor, we can instead simply just re-check that the file hasn't been closed in the meantime, and just re-do the lookup if we raced with a concurrent close() of the same file descriptor. Reported-and-tested-by: Jann Horn Acked-by: Miklos Szeredi Signed-off-by: Linus Torvalds Signed-off-by: Greg Kroah-Hartman Signed-off-by: Baokun Li Signed-off-by: Zheng Zengkai Signed-off-by: johnnyaiai --- fs/file.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/fs/file.c b/fs/file.c index b597b07a9..b232c5ea2 100644 --- a/fs/file.c +++ b/fs/file.c @@ -725,6 +725,10 @@ loop: file = NULL; else if (!get_file_rcu_many(file, refs)) goto loop; + else if (__fcheck_files(files, fd) != file) { + fput_many(file, refs); + goto loop; + } } rcu_read_unlock(); From 4b6b0e64cadd69b55c57c9f7ee9f9badb710d44e Mon Sep 17 00:00:00 2001 From: Sargun Dhillon Date: Tue, 7 Jan 2020 09:59:24 -0800 Subject: [PATCH 6/7] vfs, fdtable: Add fget_task helper upstream commit: 5e876fb43dbf24c941a323139752bcb2f0a80da0 This introduces a function which can be used to fetch a file, given an arbitrary task. As long as the user holds a reference (refcnt) to the task_struct it is safe to call, and will either return NULL on failure, or a pointer to the file, with a refcnt. This patch is based on Oleg Nesterov's (cf. [1]) patch from September 2018. [1]: Link: https://lore.kernel.org/r/20180915160423.GA31461@redhat.com Signed-off-by: Sargun Dhillon Suggested-by: Oleg Nesterov Acked-by: Christian Brauner Reviewed-by: Arnd Bergmann Link: https://lore.kernel.org/r/20200107175927.4558-2-sargun@sargun.me Signed-off-by: Christian Brauner Signed-off-by: johnnyaiai --- fs/file.c | 22 ++++++++++++++++++++-- include/linux/file.h | 2 ++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/fs/file.c b/fs/file.c index b232c5ea2..9b0e8adad 100644 --- a/fs/file.c +++ b/fs/file.c @@ -708,9 +708,9 @@ void do_close_on_exec(struct files_struct *files) spin_unlock(&files->file_lock); } -static struct file *__fget(unsigned int fd, fmode_t mask, unsigned int refs) +static struct file *__fget_files(struct files_struct *files, unsigned int fd, + fmode_t mask, unsigned int refs) { - struct files_struct *files = current->files; struct file *file; rcu_read_lock(); @@ -735,6 +735,12 @@ loop: return file; } +static inline struct file *__fget(unsigned int fd, fmode_t mask, + unsigned int refs) +{ + return __fget_files(current->files, fd, mask, refs); +} + struct file *fget_many(unsigned int fd, unsigned int refs) { return __fget(fd, FMODE_PATH, refs); @@ -752,6 +758,18 @@ struct file *fget_raw(unsigned int fd) } EXPORT_SYMBOL(fget_raw); +struct file *fget_task(struct task_struct *task, unsigned int fd) +{ + struct file *file = NULL; + + task_lock(task); + if (task->files) + file = __fget_files(task->files, fd, 0, 1); + task_unlock(task); + + return file; +} + /* * Lightweight file lookup - no refcnt increment if fd table isn't shared. * diff --git a/include/linux/file.h b/include/linux/file.h index 3fcddff56..c6c7b24ea 100644 --- a/include/linux/file.h +++ b/include/linux/file.h @@ -16,6 +16,7 @@ extern void fput(struct file *); extern void fput_many(struct file *, unsigned int); struct file_operations; +struct task_struct; struct vfsmount; struct dentry; struct inode; @@ -47,6 +48,7 @@ static inline void fdput(struct fd fd) extern struct file *fget(unsigned int fd); extern struct file *fget_many(unsigned int fd, unsigned int refs); extern struct file *fget_raw(unsigned int fd); +extern struct file *fget_task(struct task_struct *task, unsigned int fd); extern unsigned long __fdget(unsigned int fd); extern unsigned long __fdget_raw(unsigned int fd); extern unsigned long __fdget_pos(unsigned int fd); From 5788a3f8bf6133dcf07451d4cd05c15cfeb20114 Mon Sep 17 00:00:00 2001 From: Linus Torvalds Date: Sat, 29 Jan 2022 15:10:41 +0800 Subject: [PATCH 7/7] fget: clarify and improve __fget_files() implementation upstream commit: e386dfc56f837da66d00a078e5314bc8382fab83 Commit 054aa8d439b9 ("fget: check that the fd still exists after getting a ref to it") fixed a race with getting a reference to a file just as it was being closed. It was a fairly minimal patch, and I didn't think re-checking the file pointer lookup would be a measurable overhead, since it was all right there and cached. But I was wrong, as pointed out by the kernel test robot. The 'poll2' case of the will-it-scale.per_thread_ops benchmark regressed quite noticeably. Admittedly it seems to be a very artificial test: doing "poll()" system calls on regular files in a very tight loop in multiple threads. That means that basically all the time is spent just looking up file descriptors without ever doing anything useful with them (not that doing 'poll()' on a regular file is useful to begin with). And as a result it shows the extra "re-check fd" cost as a sore thumb. Happily, the regression is fixable by just writing the code to loook up the fd to be better and clearer. There's still a cost to verify the file pointer, but now it's basically in the noise even for that benchmark that does nothing else - and the code is more understandable and has better comments too. [ Side note: this patch is also a classic case of one that looks very messy with the default greedy Myers diff - it's much more legible with either the patience of histogram diff algorithm ] Link: https://lore.kernel.org/lkml/20211210053743.GA36420@xsang-OptiPlex-9020/ Link: https://lore.kernel.org/lkml/20211213083154.GA20853@linux.intel.com/ Reported-by: kernel test robot Tested-by: Carel Si Cc: Jann Horn Cc: Miklos Szeredi Signed-off-by: Linus Torvalds Signed-off-by: johnnyaiai --- fs/file.c | 72 ++++++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 56 insertions(+), 16 deletions(-) diff --git a/fs/file.c b/fs/file.c index 9b0e8adad..759f4a534 100644 --- a/fs/file.c +++ b/fs/file.c @@ -708,28 +708,68 @@ void do_close_on_exec(struct files_struct *files) spin_unlock(&files->file_lock); } +static inline struct file *__fget_files_rcu(struct files_struct *files, + unsigned int fd, fmode_t mask, unsigned int refs) +{ + for (;;) { + struct file *file; + struct fdtable *fdt = rcu_dereference_raw(files->fdt); + struct file __rcu **fdentry; + + if (unlikely(fd >= fdt->max_fds)) + return NULL; + + fdentry = fdt->fd + array_index_nospec(fd, fdt->max_fds); + file = rcu_dereference_raw(*fdentry); + if (unlikely(!file)) + return NULL; + + if (unlikely(file->f_mode & mask)) + return NULL; + + /* + * Ok, we have a file pointer. However, because we do + * this all locklessly under RCU, we may be racing with + * that file being closed. + * + * Such a race can take two forms: + * + * (a) the file ref already went down to zero, + * and get_file_rcu_many() fails. Just try + * again: + */ + if (unlikely(!get_file_rcu_many(file, refs))) + continue; + + /* + * (b) the file table entry has changed under us. + * Note that we don't need to re-check the 'fdt->fd' + * pointer having changed, because it always goes + * hand-in-hand with 'fdt'. + * + * If so, we need to put our refs and try again. + */ + if (unlikely(rcu_dereference_raw(files->fdt) != fdt) || + unlikely(rcu_dereference_raw(*fdentry) != file)) { + fput_many(file, refs); + continue; + } + + /* + * Ok, we have a ref to the file, and checked that it + * still exists. + */ + return file; + } +} + static struct file *__fget_files(struct files_struct *files, unsigned int fd, fmode_t mask, unsigned int refs) { struct file *file; rcu_read_lock(); -loop: - file = fcheck_files(files, fd); - if (file) { - /* File object ref couldn't be taken. - * dup2() atomicity guarantee is the reason - * we loop to catch the new file (or NULL pointer) - */ - if (file->f_mode & mask) - file = NULL; - else if (!get_file_rcu_many(file, refs)) - goto loop; - else if (__fcheck_files(files, fd) != file) { - fput_many(file, refs); - goto loop; - } - } + file = __fget_files_rcu(files, fd, mask, refs); rcu_read_unlock(); return file;