From febc9e4157f7871fcec00cc7dfb91b4847db06fa Mon Sep 17 00:00:00 2001 From: Mungerjiang Date: Thu, 25 Nov 2021 14:33:15 +0800 Subject: [PATCH] locking/qspinlock: Introduce llc aware spinlock Cross-LLC is the same as cross-numa, which will cause longer cache synchronization. This patch changes numa aware to LLC aware and disables llc spinlock by default. Signed-off-by: Mungerjiang --- arch/x86/include/asm/qspinlock.h | 1 + arch/x86/kernel/cpu/common.c | 8 ++++ kernel/locking/qspinlock_cna.h | 77 +++++++++++++++++++++----------- 3 files changed, 60 insertions(+), 26 deletions(-) diff --git a/arch/x86/include/asm/qspinlock.h b/arch/x86/include/asm/qspinlock.h index 21d09e8db..35e5a252c 100644 --- a/arch/x86/include/asm/qspinlock.h +++ b/arch/x86/include/asm/qspinlock.h @@ -29,6 +29,7 @@ static __always_inline u32 queued_fetch_set_pending_acquire(struct qspinlock *lo #ifdef CONFIG_NUMA_AWARE_SPINLOCKS extern void cna_configure_spin_lock_slowpath(void); +extern void cna_set_llc_id_per_cpu(unsigned int cpu); #endif #ifdef CONFIG_PARAVIRT_SPINLOCKS diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c index 74843e762..8ec156923 100644 --- a/arch/x86/kernel/cpu/common.c +++ b/arch/x86/kernel/cpu/common.c @@ -58,6 +58,10 @@ #include #endif +#ifdef CONFIG_NUMA_AWARE_SPINLOCKS +#include +#endif + #include "cpu.h" u32 elf_hwcap2 __read_mostly; @@ -1532,6 +1536,10 @@ static void identify_cpu(struct cpuinfo_x86 *c) if (this_cpu->c_init) this_cpu->c_init(c); +#ifdef CONFIG_NUMA_AWARE_SPINLOCKS + cna_set_llc_id_per_cpu(smp_processor_id()); +#endif + /* Disable the PN if appropriate */ squash_the_stupid_serial_number(c); diff --git a/kernel/locking/qspinlock_cna.h b/kernel/locking/qspinlock_cna.h index 672dee993..2c252660b 100644 --- a/kernel/locking/qspinlock_cna.h +++ b/kernel/locking/qspinlock_cna.h @@ -51,14 +51,17 @@ #define CNA_PRIORITY_NODE 0xffff +#define DEFAULT_LLC_ID 0xfffe + struct cna_node { struct mcs_spinlock mcs; - u16 numa_node; - u16 real_numa_node; + u16 llc_id; + u16 real_llc_id; u32 encoded_tail; /* self */ u64 start_time; }; +bool use_llc_spinlock = false; static ulong numa_spinlock_threshold_ns = 1000000; /* 1ms, by default */ module_param(numa_spinlock_threshold_ns, ulong, 0644); @@ -102,13 +105,16 @@ static bool probably(unsigned int num_bits) static void __init cna_init_nodes_per_cpu(unsigned int cpu) { struct mcs_spinlock *base = per_cpu_ptr(&qnodes[0].mcs, cpu); - int numa_node = cpu_to_node(cpu); int i; for (i = 0; i < MAX_NODES; i++) { struct cna_node *cn = (struct cna_node *)grab_mcs_node(base, i); - cn->real_numa_node = numa_node; + /* + *cpu_llc_id is not initialized when when this function is called + *so just set a fake llc id. + */ + cn->real_llc_id = DEFAULT_LLC_ID; cn->encoded_tail = encode_tail(cpu, i); /* * make sure @encoded_tail is not confused with other valid @@ -118,6 +124,24 @@ static void __init cna_init_nodes_per_cpu(unsigned int cpu) } } +/* +* must be called after cpu_llc_id is initialized. +*/ +void cna_set_llc_id_per_cpu(unsigned int cpu) +{ + struct mcs_spinlock *base = per_cpu_ptr(&qnodes[0].mcs, cpu); + int i; + + if (!use_llc_spinlock) + return; + + for (i = 0; i < MAX_NODES; i++) { + struct cna_node *cn = (struct cna_node *)grab_mcs_node(base, i); + + cn->real_llc_id = per_cpu(cpu_llc_id, cpu); + } +} + static int __init cna_init_nodes(void) { unsigned int cpu; @@ -141,7 +165,7 @@ static __always_inline void cna_init_node(struct mcs_spinlock *node) bool priority = !in_task() || irqs_disabled() || rt_task(current); struct cna_node *cn = (struct cna_node *)node; - cn->numa_node = priority ? CNA_PRIORITY_NODE : cn->real_numa_node; + cn->llc_id = priority ? CNA_PRIORITY_NODE : cn->real_llc_id; cn->start_time = 0; } @@ -272,15 +296,15 @@ static int cna_order_queue(struct mcs_spinlock *node) { struct mcs_spinlock *next = READ_ONCE(node->next); struct cna_node *cn = (struct cna_node *)node; - int numa_node, next_numa_node; + int llc_id, next_llc_id; if (!next) return 0; - numa_node = cn->numa_node; - next_numa_node = ((struct cna_node *)next)->numa_node; + llc_id = cn->llc_id; + next_llc_id = ((struct cna_node *)next)->llc_id; - if (next_numa_node != numa_node && next_numa_node != CNA_PRIORITY_NODE) { + if (next_llc_id != llc_id && next_llc_id != CNA_PRIORITY_NODE) { struct mcs_spinlock *nnext = READ_ONCE(next->next); if (nnext) @@ -315,8 +339,8 @@ static __always_inline u32 cna_wait_head_or_lock(struct qspinlock *lock, * We are at the head of the wait queue, no need to use * the fake NUMA node ID. */ - if (cn->numa_node == CNA_PRIORITY_NODE) - cn->numa_node = cn->real_numa_node; + if (cn->llc_id == CNA_PRIORITY_NODE) + cn->llc_id = cn->real_llc_id; /* * Try and put the time otherwise spent spin waiting on @@ -352,7 +376,7 @@ static inline void cna_lock_handoff(struct mcs_spinlock *node, * Pass over NUMA node id of primary queue, to maintain the * preference even if the next waiter is on a different node. */ - ((struct cna_node *)next)->numa_node = cn->numa_node; + ((struct cna_node *)next)->llc_id = cn->llc_id; ((struct cna_node *)next)->start_time = cn->start_time; } @@ -375,48 +399,49 @@ static inline void cna_lock_handoff(struct mcs_spinlock *node, } /* - * Constant (boot-param configurable) flag selecting the NUMA-aware variant + * Constant (boot-param configurable) flag selecting the LLC-aware variant * of spinlock. Possible values: -1 (off) / 0 (auto, default) / 1 (on). */ -static int numa_spinlock_flag; +static int llc_spinlock_flag = -1; -static int __init numa_spinlock_setup(char *str) +static int __init llc_spinlock_setup(char *str) { if (!strcmp(str, "auto")) { - numa_spinlock_flag = 0; + llc_spinlock_flag = 0; return 1; } else if (!strcmp(str, "on")) { - numa_spinlock_flag = 1; + llc_spinlock_flag = 1; return 1; } else if (!strcmp(str, "off")) { - numa_spinlock_flag = -1; + llc_spinlock_flag = -1; return 1; } return 0; } -__setup("numa_spinlock=", numa_spinlock_setup); +__setup("llc_spinlock=", llc_spinlock_setup); void __cna_queued_spin_lock_slowpath(struct qspinlock *lock, u32 val); /* - * Switch to the NUMA-friendly slow path for spinlocks when we have - * multiple NUMA nodes in native environment, unless the user has - * overridden this default behavior by setting the numa_spinlock flag. + * Switch to the llc-friendly slow path for spinlocks when we have + * multiple llc nodes in native environment, unless the user has + * overridden this default behavior by setting the llc_spinlock flag. */ void __init cna_configure_spin_lock_slowpath(void) { - - if (numa_spinlock_flag < 0) + if (llc_spinlock_flag < 0) return; - if (numa_spinlock_flag == 0 && (nr_node_ids < 2 || + if (llc_spinlock_flag == 0 && pv_ops.lock.queued_spin_lock_slowpath != - native_queued_spin_lock_slowpath)) + native_queued_spin_lock_slowpath) return; cna_init_nodes(); + use_llc_spinlock = true; + pv_ops.lock.queued_spin_lock_slowpath = __cna_queued_spin_lock_slowpath; pr_info("Enabling CNA spinlock\n");