x86/mm/tlb: Make enter_lazy_tlb() always inline on x86

enter_lazy_tlb() on x86 is short enough, and is called in context
switching, which is the hot code path.

Make enter_lazy_tlb() always inline on x86 to optimize performance.

Suggested-by: Dave Hansen <dave.hansen@intel.com>
Signed-off-by: Xie Yuanbin <qq570070308@gmail.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Link: https://patch.msgid.link/20260216164950.147617-2-qq570070308@gmail.com

authored by

Xie Yuanbin and committed by

Ingo Molnar 3 months ago 4b9ef32c 12f80691

+26 -24

3 changed files

expand all

arch

x86

include

asm

mmu_context.h

tlbflush.h

tlb.c

-3

arch/x86/include/asm/mmu_context.h

··· 136 136 } 137 137 #endif 138 138 139 - #define enter_lazy_tlb enter_lazy_tlb 140 - extern void enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk); 141 - 142 139 extern void mm_init_global_asid(struct mm_struct *mm); 143 140 extern void mm_free_global_asid(struct mm_struct *mm); 144 141

+26

arch/x86/include/asm/tlbflush.h

··· 172 172 }; 173 173 DECLARE_PER_CPU_SHARED_ALIGNED(struct tlb_state_shared, cpu_tlbstate_shared); 174 174 175 + /* 176 + * Please ignore the name of this function. It should be called 177 + * switch_to_kernel_thread(). 178 + * 179 + * enter_lazy_tlb() is a hint from the scheduler that we are entering a 180 + * kernel thread or other context without an mm. Acceptable implementations 181 + * include doing nothing whatsoever, switching to init_mm, or various clever 182 + * lazy tricks to try to minimize TLB flushes. 183 + * 184 + * The scheduler reserves the right to call enter_lazy_tlb() several times 185 + * in a row. It will notify us that we're going back to a real mm by 186 + * calling switch_mm_irqs_off(). 187 + */ 188 + #define enter_lazy_tlb enter_lazy_tlb 189 + static __always_inline void enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk) 190 + { 191 + if (this_cpu_read(cpu_tlbstate.loaded_mm) == &init_mm) 192 + return; 193 + 194 + this_cpu_write(cpu_tlbstate_shared.is_lazy, true); 195 + } 196 + 175 197 bool nmi_uaccess_okay(void); 176 198 #define nmi_uaccess_okay nmi_uaccess_okay 177 199 ··· 502 480 { 503 481 } 504 482 #endif 483 + #else /* !MODULE */ 484 + #define enter_lazy_tlb enter_lazy_tlb 485 + extern void enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk) 486 + __compiletime_error("enter_lazy_tlb() should not be used in modules"); 505 487 #endif /* !MODULE */ 506 488 507 489 static inline void __native_tlb_flush_global(unsigned long cr4)

-21

arch/x86/mm/tlb.c

··· 972 972 } 973 973 974 974 /* 975 - * Please ignore the name of this function. It should be called 976 - * switch_to_kernel_thread(). 977 - * 978 - * enter_lazy_tlb() is a hint from the scheduler that we are entering a 979 - * kernel thread or other context without an mm. Acceptable implementations 980 - * include doing nothing whatsoever, switching to init_mm, or various clever 981 - * lazy tricks to try to minimize TLB flushes. 982 - * 983 - * The scheduler reserves the right to call enter_lazy_tlb() several times 984 - * in a row. It will notify us that we're going back to a real mm by 985 - * calling switch_mm_irqs_off(). 986 - */ 987 - void enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk) 988 - { 989 - if (this_cpu_read(cpu_tlbstate.loaded_mm) == &init_mm) 990 - return; 991 - 992 - this_cpu_write(cpu_tlbstate_shared.is_lazy, true); 993 - } 994 - 995 - /* 996 975 * Using a temporary mm allows to set temporary mappings that are not accessible 997 976 * by other CPUs. Such mappings are needed to perform sensitive memory writes 998 977 * that override the kernel memory protections (e.g., W^X), without exposing the

Configure Feed

Configure Feed