tjh.dev / kernel
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

powerpc/book3s64/pkeys: Optimize KUAP and KUEP feature disabled case

If FTR_BOOK3S_KUAP is disabled, kernel will continue to run with the same AMR
value with which it was entered. Hence there is a high chance that
we can return without restoring the AMR value. This also helps the case
when applications are not using the pkey feature. In this case, different
applications will have the same AMR values and hence we can avoid restoring
AMR in this case too.

Also avoid isync() if not really needed.

Do the same for IAMR.

null-syscall benchmark results:

With smap/smep disabled:
Without patch:
957.95 ns 2778.17 cycles
With patch:
858.38 ns 2489.30 cycles

With smap/smep enabled:
Without patch:
1017.26 ns 2950.36 cycles
With patch:
1021.51 ns 2962.44 cycles

Signed-off-by: Aneesh Kumar K.V <aneesh.kumar@linux.ibm.com>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
Link: https://lore.kernel.org/r/20201127044424.40686-23-aneesh.kumar@linux.ibm.com

authored by

Aneesh Kumar K.V and committed by
Michael Ellerman ec0f9b98 61130e20

+67 -10
+57 -6
arch/powerpc/include/asm/book3s/64/kup.h
··· 12 12 13 13 #ifdef __ASSEMBLY__ 14 14 15 - .macro kuap_user_restore gpr1 15 + .macro kuap_user_restore gpr1, gpr2 16 16 #if defined(CONFIG_PPC_PKEY) 17 17 BEGIN_MMU_FTR_SECTION_NESTED(67) 18 + b 100f // skip_restore_amr 19 + END_MMU_FTR_SECTION_NESTED_IFCLR(MMU_FTR_PKEY, 67) 18 20 /* 19 21 * AMR and IAMR are going to be different when 20 22 * returning to userspace. 21 23 */ 22 24 ld \gpr1, STACK_REGS_AMR(r1) 25 + 26 + /* 27 + * If kuap feature is not enabled, do the mtspr 28 + * only if AMR value is different. 29 + */ 30 + BEGIN_MMU_FTR_SECTION_NESTED(68) 31 + mfspr \gpr2, SPRN_AMR 32 + cmpd \gpr1, \gpr2 33 + beq 99f 34 + END_MMU_FTR_SECTION_NESTED_IFCLR(MMU_FTR_BOOK3S_KUAP, 68) 35 + 23 36 isync 24 37 mtspr SPRN_AMR, \gpr1 38 + 99: 25 39 /* 26 40 * Restore IAMR only when returning to userspace 27 41 */ 28 42 ld \gpr1, STACK_REGS_IAMR(r1) 43 + 44 + /* 45 + * If kuep feature is not enabled, do the mtspr 46 + * only if IAMR value is different. 47 + */ 48 + BEGIN_MMU_FTR_SECTION_NESTED(69) 49 + mfspr \gpr2, SPRN_IAMR 50 + cmpd \gpr1, \gpr2 51 + beq 100f 52 + END_MMU_FTR_SECTION_NESTED_IFCLR(MMU_FTR_BOOK3S_KUEP, 69) 53 + 54 + isync 29 55 mtspr SPRN_IAMR, \gpr1 30 56 57 + 100: //skip_restore_amr 31 58 /* No isync required, see kuap_user_restore() */ 32 - END_MMU_FTR_SECTION_NESTED_IFSET(MMU_FTR_PKEY, 67) 33 59 #endif 34 60 .endm 35 61 36 - .macro kuap_kernel_restore gpr1, gpr2 62 + .macro kuap_kernel_restore gpr1, gpr2 37 63 #if defined(CONFIG_PPC_PKEY) 38 64 39 65 BEGIN_MMU_FTR_SECTION_NESTED(67) ··· 225 199 226 200 static inline void kuap_user_restore(struct pt_regs *regs) 227 201 { 202 + bool restore_amr = false, restore_iamr = false; 203 + unsigned long amr, iamr; 204 + 228 205 if (!mmu_has_feature(MMU_FTR_PKEY)) 229 206 return; 230 207 231 - isync(); 232 - mtspr(SPRN_AMR, regs->amr); 233 - mtspr(SPRN_IAMR, regs->iamr); 208 + if (!mmu_has_feature(MMU_FTR_BOOK3S_KUAP)) { 209 + amr = mfspr(SPRN_AMR); 210 + if (amr != regs->amr) 211 + restore_amr = true; 212 + } else { 213 + restore_amr = true; 214 + } 215 + 216 + if (!mmu_has_feature(MMU_FTR_BOOK3S_KUEP)) { 217 + iamr = mfspr(SPRN_IAMR); 218 + if (iamr != regs->iamr) 219 + restore_iamr = true; 220 + } else { 221 + restore_iamr = true; 222 + } 223 + 224 + 225 + if (restore_amr || restore_iamr) { 226 + isync(); 227 + if (restore_amr) 228 + mtspr(SPRN_AMR, regs->amr); 229 + if (restore_iamr) 230 + mtspr(SPRN_IAMR, regs->iamr); 231 + } 234 232 /* 235 233 * No isync required here because we are about to rfi 236 234 * back to previous context before any user accesses 237 235 * would be made, which is a CSI. 238 236 */ 239 237 } 238 + 240 239 static inline void kuap_kernel_restore(struct pt_regs *regs, 241 240 unsigned long amr) 242 241 {
+1 -1
arch/powerpc/kernel/entry_64.S
··· 675 675 bne- .Lrestore_nvgprs 676 676 677 677 .Lfast_user_interrupt_return_amr: 678 - kuap_user_restore r3 678 + kuap_user_restore r3, r4 679 679 .Lfast_user_interrupt_return: 680 680 ld r11,_NIP(r1) 681 681 ld r12,_MSR(r1)
+9 -3
arch/powerpc/kernel/syscall_64.c
··· 38 38 #ifdef CONFIG_PPC_PKEY 39 39 if (mmu_has_feature(MMU_FTR_PKEY)) { 40 40 unsigned long amr, iamr; 41 + bool flush_needed = false; 41 42 /* 42 43 * When entering from userspace we mostly have the AMR/IAMR 43 44 * different from kernel default values. Hence don't compare. ··· 47 46 iamr = mfspr(SPRN_IAMR); 48 47 regs->amr = amr; 49 48 regs->iamr = iamr; 50 - if (mmu_has_feature(MMU_FTR_BOOK3S_KUAP)) 49 + if (mmu_has_feature(MMU_FTR_BOOK3S_KUAP)) { 51 50 mtspr(SPRN_AMR, AMR_KUAP_BLOCKED); 52 - if (mmu_has_feature(MMU_FTR_BOOK3S_KUEP)) 51 + flush_needed = true; 52 + } 53 + if (mmu_has_feature(MMU_FTR_BOOK3S_KUEP)) { 53 54 mtspr(SPRN_IAMR, AMR_KUEP_BLOCKED); 54 - isync(); 55 + flush_needed = true; 56 + } 57 + if (flush_needed) 58 + isync(); 55 59 } else 56 60 #endif 57 61 kuap_check_amr();