Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

riscv: Use IPIs for remote cache/TLB flushes by default

An IPI backend is always required in an SMP configuration, but an SBI
implementation is not. For example, SBI will be unavailable when the
kernel runs in M mode. For this reason, consider IPI delivery of cache
and TLB flushes to be the base case, and any other implementation (such
as the SBI remote fence extension) to be an optimization.

Generally, if IPIs can be delivered without firmware assistance, they
are assumed to be faster than SBI calls due to the SBI context switch
overhead. However, when SBI is used as the IPI backend, then the context
switch cost must be paid anyway, and performing the cache/TLB flush
directly in the SBI implementation is more efficient than injecting an
interrupt to S-mode. This is the only existing scenario where
riscv_ipi_set_virq_range() is called with use_for_rfence set to false.

sbi_ipi_init() already checks riscv_ipi_have_virq_range(), so it only
calls riscv_ipi_set_virq_range() when no other IPI device is available.
This allows moving the static key and dropping the use_for_rfence
parameter. This decouples the static key from the irqchip driver probe
order.

Furthermore, the static branch only makes sense when CONFIG_RISCV_SBI is
enabled. Optherwise, IPIs must be used. Add a fallback definition of
riscv_use_sbi_for_rfence() which handles this case and removes the need
to check CONFIG_RISCV_SBI elsewhere, such as in cacheflush.c.

Reviewed-by: Anup Patel <anup@brainfault.org>
Signed-off-by: Samuel Holland <samuel.holland@sifive.com>
Reviewed-by: Alexandre Ghiti <alexghiti@rivosinc.com>
Link: https://lore.kernel.org/r/20240327045035.368512-4-samuel.holland@sifive.com
Signed-off-by: Palmer Dabbelt <palmer@rivosinc.com>

authored by

Samuel Holland and committed by
Palmer Dabbelt
dc892fb4 aaa56c8f

+38 -48
+4 -3
arch/riscv/include/asm/pgalloc.h
··· 8 8 #define _ASM_RISCV_PGALLOC_H 9 9 10 10 #include <linux/mm.h> 11 + #include <asm/sbi.h> 11 12 #include <asm/tlb.h> 12 13 13 14 #ifdef CONFIG_MMU ··· 18 17 19 18 static inline void riscv_tlb_remove_ptdesc(struct mmu_gather *tlb, void *pt) 20 19 { 21 - if (riscv_use_ipi_for_rfence()) 22 - tlb_remove_page_ptdesc(tlb, pt); 23 - else 20 + if (riscv_use_sbi_for_rfence()) 24 21 tlb_remove_ptdesc(tlb, pt); 22 + else 23 + tlb_remove_page_ptdesc(tlb, pt); 25 24 } 26 25 27 26 static inline void pmd_populate_kernel(struct mm_struct *mm,
+4
arch/riscv/include/asm/sbi.h
··· 375 375 unsigned long riscv_cached_mimpid(unsigned int cpu_id); 376 376 377 377 #if IS_ENABLED(CONFIG_SMP) && IS_ENABLED(CONFIG_RISCV_SBI) 378 + DECLARE_STATIC_KEY_FALSE(riscv_sbi_for_rfence); 379 + #define riscv_use_sbi_for_rfence() \ 380 + static_branch_unlikely(&riscv_sbi_for_rfence) 378 381 void sbi_ipi_init(void); 379 382 #else 383 + static inline bool riscv_use_sbi_for_rfence(void) { return false; } 380 384 static inline void sbi_ipi_init(void) { } 381 385 #endif 382 386
+2 -13
arch/riscv/include/asm/smp.h
··· 49 49 bool riscv_ipi_have_virq_range(void); 50 50 51 51 /* Set the IPI interrupt numbers for arch (called by irqchip drivers) */ 52 - void riscv_ipi_set_virq_range(int virq, int nr, bool use_for_rfence); 53 - 54 - /* Check if we can use IPIs for remote FENCEs */ 55 - DECLARE_STATIC_KEY_FALSE(riscv_ipi_for_rfence); 56 - #define riscv_use_ipi_for_rfence() \ 57 - static_branch_unlikely(&riscv_ipi_for_rfence) 52 + void riscv_ipi_set_virq_range(int virq, int nr); 58 53 59 54 /* Check other CPUs stop or not */ 60 55 bool smp_crash_stop_failed(void); ··· 99 104 return false; 100 105 } 101 106 102 - static inline void riscv_ipi_set_virq_range(int virq, int nr, 103 - bool use_for_rfence) 107 + static inline void riscv_ipi_set_virq_range(int virq, int nr) 104 108 { 105 - } 106 - 107 - static inline bool riscv_use_ipi_for_rfence(void) 108 - { 109 - return false; 110 109 } 111 110 112 111 #endif /* CONFIG_SMP */
+10 -1
arch/riscv/kernel/sbi-ipi.c
··· 13 13 #include <linux/irqdomain.h> 14 14 #include <asm/sbi.h> 15 15 16 + DEFINE_STATIC_KEY_FALSE(riscv_sbi_for_rfence); 17 + EXPORT_SYMBOL_GPL(riscv_sbi_for_rfence); 18 + 16 19 static int sbi_ipi_virq; 17 20 18 21 static void sbi_ipi_handle(struct irq_desc *desc) ··· 75 72 "irqchip/sbi-ipi:starting", 76 73 sbi_ipi_starting_cpu, NULL); 77 74 78 - riscv_ipi_set_virq_range(virq, BITS_PER_BYTE, false); 75 + riscv_ipi_set_virq_range(virq, BITS_PER_BYTE); 79 76 pr_info("providing IPIs using SBI IPI extension\n"); 77 + 78 + /* 79 + * Use the SBI remote fence extension to avoid 80 + * the extra context switch needed to handle IPIs. 81 + */ 82 + static_branch_enable(&riscv_sbi_for_rfence); 80 83 }
+1 -10
arch/riscv/kernel/smp.c
··· 171 171 return (ipi_virq_base) ? true : false; 172 172 } 173 173 174 - DEFINE_STATIC_KEY_FALSE(riscv_ipi_for_rfence); 175 - EXPORT_SYMBOL_GPL(riscv_ipi_for_rfence); 176 - 177 - void riscv_ipi_set_virq_range(int virq, int nr, bool use_for_rfence) 174 + void riscv_ipi_set_virq_range(int virq, int nr) 178 175 { 179 176 int i, err; 180 177 ··· 194 197 195 198 /* Enabled IPIs for boot CPU immediately */ 196 199 riscv_ipi_enable(); 197 - 198 - /* Update RFENCE static key */ 199 - if (use_for_rfence) 200 - static_branch_enable(&riscv_ipi_for_rfence); 201 - else 202 - static_branch_disable(&riscv_ipi_for_rfence); 203 200 } 204 201 205 202 static const char * const ipi_names[] = {
+2 -3
arch/riscv/mm/cacheflush.c
··· 21 21 { 22 22 local_flush_icache_all(); 23 23 24 - if (IS_ENABLED(CONFIG_RISCV_SBI) && !riscv_use_ipi_for_rfence()) 24 + if (riscv_use_sbi_for_rfence()) 25 25 sbi_remote_fence_i(NULL); 26 26 else 27 27 on_each_cpu(ipi_remote_fence_i, NULL, 1); ··· 69 69 * with flush_icache_deferred(). 70 70 */ 71 71 smp_mb(); 72 - } else if (IS_ENABLED(CONFIG_RISCV_SBI) && 73 - !riscv_use_ipi_for_rfence()) { 72 + } else if (riscv_use_sbi_for_rfence()) { 74 73 sbi_remote_fence_i(&others); 75 74 } else { 76 75 on_each_cpu_mask(&others, ipi_remote_fence_i, NULL, 1);
+14 -17
arch/riscv/mm/tlbflush.c
··· 79 79 80 80 void flush_tlb_all(void) 81 81 { 82 - if (riscv_use_ipi_for_rfence()) 83 - on_each_cpu(__ipi_flush_tlb_all, NULL, 1); 84 - else 82 + if (riscv_use_sbi_for_rfence()) 85 83 sbi_remote_sfence_vma_asid(NULL, 0, FLUSH_TLB_MAX_SIZE, FLUSH_TLB_NO_ASID); 84 + else 85 + on_each_cpu(__ipi_flush_tlb_all, NULL, 1); 86 86 } 87 87 88 88 struct flush_tlb_range_data { ··· 103 103 unsigned long start, unsigned long size, 104 104 unsigned long stride) 105 105 { 106 - struct flush_tlb_range_data ftd; 107 106 bool broadcast; 108 107 109 108 if (cpumask_empty(cmask)) ··· 118 119 broadcast = true; 119 120 } 120 121 121 - if (broadcast) { 122 - if (riscv_use_ipi_for_rfence()) { 123 - ftd.asid = asid; 124 - ftd.start = start; 125 - ftd.size = size; 126 - ftd.stride = stride; 127 - on_each_cpu_mask(cmask, 128 - __ipi_flush_tlb_range_asid, 129 - &ftd, 1); 130 - } else 131 - sbi_remote_sfence_vma_asid(cmask, 132 - start, size, asid); 133 - } else { 122 + if (!broadcast) { 134 123 local_flush_tlb_range_asid(start, size, stride, asid); 124 + } else if (riscv_use_sbi_for_rfence()) { 125 + sbi_remote_sfence_vma_asid(cmask, start, size, asid); 126 + } else { 127 + struct flush_tlb_range_data ftd; 128 + 129 + ftd.asid = asid; 130 + ftd.start = start; 131 + ftd.size = size; 132 + ftd.stride = stride; 133 + on_each_cpu_mask(cmask, __ipi_flush_tlb_range_asid, &ftd, 1); 135 134 } 136 135 137 136 if (cmask != cpu_online_mask)
+1 -1
drivers/clocksource/timer-clint.c
··· 251 251 } 252 252 253 253 irq_set_chained_handler(clint_ipi_irq, clint_ipi_interrupt); 254 - riscv_ipi_set_virq_range(rc, BITS_PER_BYTE, true); 254 + riscv_ipi_set_virq_range(rc, BITS_PER_BYTE); 255 255 clint_clear_ipi(); 256 256 #endif 257 257