Merge tag 'x86_mm_for_6.4' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

+6

arch/arm64/include/asm/mmu_context.h

··· 288 288 unsigned long arm64_mm_context_get(struct mm_struct *mm); 289 289 void arm64_mm_context_put(struct mm_struct *mm); 290 290 291 + #define mm_untag_mask mm_untag_mask 292 + static inline unsigned long mm_untag_mask(struct mm_struct *mm) 293 + { 294 + return -1UL >> 8; 295 + } 296 + 291 297 #include <asm-generic/mmu_context.h> 292 298 293 299 #endif /* !__ASSEMBLY__ */

+6

arch/sparc/include/asm/mmu_context_64.h

··· 185 185 } 186 186 } 187 187 188 + #define mm_untag_mask mm_untag_mask 189 + static inline unsigned long mm_untag_mask(struct mm_struct *mm) 190 + { 191 + return -1UL >> adi_nbits(); 192 + } 193 + 188 194 #include <asm-generic/mmu_context.h> 189 195 190 196 #endif /* !(__ASSEMBLY__) */

+2

arch/sparc/include/asm/uaccess_64.h

··· 8 8 9 9 #include <linux/compiler.h> 10 10 #include <linux/string.h> 11 + #include <linux/mm_types.h> 11 12 #include <asm/asi.h> 12 13 #include <asm/spitfire.h> 14 + #include <asm/pgtable.h> 13 15 14 16 #include <asm/processor.h> 15 17 #include <asm-generic/access_ok.h>

+11

arch/x86/Kconfig

··· 2290 2290 2291 2291 If unsure, leave at the default value. 2292 2292 2293 + config ADDRESS_MASKING 2294 + bool "Linear Address Masking support" 2295 + depends on X86_64 2296 + help 2297 + Linear Address Masking (LAM) modifies the checking that is applied 2298 + to 64-bit linear addresses, allowing software to use of the 2299 + untranslated address bits for metadata. 2300 + 2301 + The capability can be used for efficient address sanitizers (ASAN) 2302 + implementation and for optimizations in JITs. 2303 + 2293 2304 config HOTPLUG_CPU 2294 2305 def_bool y 2295 2306 depends on SMP

+1 -1

arch/x86/entry/vsyscall/vsyscall_64.c

··· 317 317 struct vm_area_struct *get_gate_vma(struct mm_struct *mm) 318 318 { 319 319 #ifdef CONFIG_COMPAT 320 - if (!mm || !(mm->context.flags & MM_CONTEXT_HAS_VSYSCALL)) 320 + if (!mm || !test_bit(MM_CONTEXT_HAS_VSYSCALL, &mm->context.flags)) 321 321 return NULL; 322 322 #endif 323 323 if (vsyscall_mode == NONE)

+1

arch/x86/include/asm/cpufeatures.h

··· 321 321 #define X86_FEATURE_LKGS (12*32+18) /* "" Load "kernel" (userspace) GS */ 322 322 #define X86_FEATURE_AMX_FP16 (12*32+21) /* "" AMX fp16 Support */ 323 323 #define X86_FEATURE_AVX_IFMA (12*32+23) /* "" Support for VPMADD52[H,L]UQ */ 324 + #define X86_FEATURE_LAM (12*32+26) /* Linear Address Masking */ 324 325 325 326 /* AMD-defined CPU features, CPUID level 0x80000008 (EBX), word 13 */ 326 327 #define X86_FEATURE_CLZERO (13*32+ 0) /* CLZERO instruction */

+7 -1

arch/x86/include/asm/disabled-features.h

··· 75 75 # define DISABLE_CALL_DEPTH_TRACKING (1 << (X86_FEATURE_CALL_DEPTH & 31)) 76 76 #endif 77 77 78 + #ifdef CONFIG_ADDRESS_MASKING 79 + # define DISABLE_LAM 0 80 + #else 81 + # define DISABLE_LAM (1 << (X86_FEATURE_LAM & 31)) 82 + #endif 83 + 78 84 #ifdef CONFIG_INTEL_IOMMU_SVM 79 85 # define DISABLE_ENQCMD 0 80 86 #else ··· 121 115 #define DISABLED_MASK10 0 122 116 #define DISABLED_MASK11 (DISABLE_RETPOLINE|DISABLE_RETHUNK|DISABLE_UNRET| \ 123 117 DISABLE_CALL_DEPTH_TRACKING) 124 - #define DISABLED_MASK12 0 118 + #define DISABLED_MASK12 (DISABLE_LAM) 125 119 #define DISABLED_MASK13 0 126 120 #define DISABLED_MASK14 0 127 121 #define DISABLED_MASK15 0

+15 -3

arch/x86/include/asm/mmu.h

··· 9 9 #include <linux/bits.h> 10 10 11 11 /* Uprobes on this MM assume 32-bit code */ 12 - #define MM_CONTEXT_UPROBE_IA32 BIT(0) 12 + #define MM_CONTEXT_UPROBE_IA32 0 13 13 /* vsyscall page is accessible on this MM */ 14 - #define MM_CONTEXT_HAS_VSYSCALL BIT(1) 14 + #define MM_CONTEXT_HAS_VSYSCALL 1 15 + /* Do not allow changing LAM mode */ 16 + #define MM_CONTEXT_LOCK_LAM 2 17 + /* Allow LAM and SVA coexisting */ 18 + #define MM_CONTEXT_FORCE_TAGGED_SVA 3 15 19 16 20 /* 17 21 * x86 has arch-specific MMU state beyond what lives in mm_struct. ··· 43 39 #endif 44 40 45 41 #ifdef CONFIG_X86_64 46 - unsigned short flags; 42 + unsigned long flags; 43 + #endif 44 + 45 + #ifdef CONFIG_ADDRESS_MASKING 46 + /* Active LAM mode: X86_CR3_LAM_U48 or X86_CR3_LAM_U57 or 0 (disabled) */ 47 + unsigned long lam_cr3_mask; 48 + 49 + /* Significant bits of the virtual address. Excludes tag bits. */ 50 + u64 untag_mask; 47 51 #endif 48 52 49 53 struct mutex lock;

+48 -1

arch/x86/include/asm/mmu_context.h

··· 85 85 } 86 86 #endif 87 87 88 + #ifdef CONFIG_ADDRESS_MASKING 89 + static inline unsigned long mm_lam_cr3_mask(struct mm_struct *mm) 90 + { 91 + return mm->context.lam_cr3_mask; 92 + } 93 + 94 + static inline void dup_lam(struct mm_struct *oldmm, struct mm_struct *mm) 95 + { 96 + mm->context.lam_cr3_mask = oldmm->context.lam_cr3_mask; 97 + mm->context.untag_mask = oldmm->context.untag_mask; 98 + } 99 + 100 + #define mm_untag_mask mm_untag_mask 101 + static inline unsigned long mm_untag_mask(struct mm_struct *mm) 102 + { 103 + return mm->context.untag_mask; 104 + } 105 + 106 + static inline void mm_reset_untag_mask(struct mm_struct *mm) 107 + { 108 + mm->context.untag_mask = -1UL; 109 + } 110 + 111 + #define arch_pgtable_dma_compat arch_pgtable_dma_compat 112 + static inline bool arch_pgtable_dma_compat(struct mm_struct *mm) 113 + { 114 + return !mm_lam_cr3_mask(mm) || 115 + test_bit(MM_CONTEXT_FORCE_TAGGED_SVA, &mm->context.flags); 116 + } 117 + #else 118 + 119 + static inline unsigned long mm_lam_cr3_mask(struct mm_struct *mm) 120 + { 121 + return 0; 122 + } 123 + 124 + static inline void dup_lam(struct mm_struct *oldmm, struct mm_struct *mm) 125 + { 126 + } 127 + 128 + static inline void mm_reset_untag_mask(struct mm_struct *mm) 129 + { 130 + } 131 + #endif 132 + 88 133 #define enter_lazy_tlb enter_lazy_tlb 89 134 extern void enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk); 90 135 ··· 154 109 mm->context.execute_only_pkey = -1; 155 110 } 156 111 #endif 112 + mm_reset_untag_mask(mm); 157 113 init_new_context_ldt(mm); 158 114 return 0; 159 115 } ··· 208 162 { 209 163 arch_dup_pkeys(oldmm, mm); 210 164 paravirt_enter_mmap(mm); 165 + dup_lam(oldmm, mm); 211 166 return ldt_dup_context(oldmm, mm); 212 167 } 213 168 ··· 222 175 static inline bool is_64bit_mm(struct mm_struct *mm) 223 176 { 224 177 return !IS_ENABLED(CONFIG_IA32_EMULATION) || 225 - !(mm->context.flags & MM_CONTEXT_UPROBE_IA32); 178 + !test_bit(MM_CONTEXT_UPROBE_IA32, &mm->context.flags); 226 179 } 227 180 #else 228 181 static inline bool is_64bit_mm(struct mm_struct *mm)

+2

arch/x86/include/asm/processor-flags.h

··· 28 28 * On systems with SME, one bit (in a variable position!) is stolen to indicate 29 29 * that the top-level paging structure is encrypted. 30 30 * 31 + * On systemms with LAM, bits 61 and 62 are used to indicate LAM mode. 32 + * 31 33 * All of the remaining bits indicate the physical address of the top-level 32 34 * paging structure. 33 35 *

+47 -1

arch/x86/include/asm/tlbflush.h

··· 2 2 #ifndef _ASM_X86_TLBFLUSH_H 3 3 #define _ASM_X86_TLBFLUSH_H 4 4 5 - #include <linux/mm.h> 5 + #include <linux/mm_types.h> 6 6 #include <linux/sched.h> 7 7 8 8 #include <asm/processor.h> ··· 12 12 #include <asm/invpcid.h> 13 13 #include <asm/pti.h> 14 14 #include <asm/processor-flags.h> 15 + #include <asm/pgtable.h> 15 16 16 17 void __flush_tlb_all(void); 17 18 ··· 53 52 cr4_clear_bits_irqsoff(mask); 54 53 local_irq_restore(flags); 55 54 } 55 + 56 + #ifdef CONFIG_ADDRESS_MASKING 57 + DECLARE_PER_CPU(u64, tlbstate_untag_mask); 58 + 59 + static inline u64 current_untag_mask(void) 60 + { 61 + return this_cpu_read(tlbstate_untag_mask); 62 + } 63 + #endif 56 64 57 65 #ifndef MODULE 58 66 /* ··· 110 100 * need to be invalidated. 111 101 */ 112 102 bool invalidate_other; 103 + 104 + #ifdef CONFIG_ADDRESS_MASKING 105 + /* 106 + * Active LAM mode. 107 + * 108 + * X86_CR3_LAM_U57/U48 shifted right by X86_CR3_LAM_U57_BIT or 0 if LAM 109 + * disabled. 110 + */ 111 + u8 lam; 112 + #endif 113 113 114 114 /* 115 115 * Mask that contains TLB_NR_DYN_ASIDS+1 bits to indicate ··· 377 357 } 378 358 #define huge_pmd_needs_flush huge_pmd_needs_flush 379 359 360 + #ifdef CONFIG_ADDRESS_MASKING 361 + static inline u64 tlbstate_lam_cr3_mask(void) 362 + { 363 + u64 lam = this_cpu_read(cpu_tlbstate.lam); 364 + 365 + return lam << X86_CR3_LAM_U57_BIT; 366 + } 367 + 368 + static inline void set_tlbstate_lam_mode(struct mm_struct *mm) 369 + { 370 + this_cpu_write(cpu_tlbstate.lam, 371 + mm->context.lam_cr3_mask >> X86_CR3_LAM_U57_BIT); 372 + this_cpu_write(tlbstate_untag_mask, mm->context.untag_mask); 373 + } 374 + 375 + #else 376 + 377 + static inline u64 tlbstate_lam_cr3_mask(void) 378 + { 379 + return 0; 380 + } 381 + 382 + static inline void set_tlbstate_lam_mode(struct mm_struct *mm) 383 + { 384 + } 385 + #endif 380 386 #endif /* !MODULE */ 381 387 382 388 static inline void __native_tlb_flush_global(unsigned long cr4)

+56 -2

arch/x86/include/asm/uaccess.h

··· 7 7 #include <linux/compiler.h> 8 8 #include <linux/instrumented.h> 9 9 #include <linux/kasan-checks.h> 10 + #include <linux/mm_types.h> 10 11 #include <linux/string.h> 12 + #include <linux/mmap_lock.h> 11 13 #include <asm/asm.h> 12 14 #include <asm/page.h> 13 15 #include <asm/smap.h> 14 16 #include <asm/extable.h> 17 + #include <asm/tlbflush.h> 15 18 16 19 #ifdef CONFIG_DEBUG_ATOMIC_SLEEP 17 20 static inline bool pagefault_disabled(void); ··· 22 19 WARN_ON_ONCE(!in_task() && !pagefault_disabled()) 23 20 #else 24 21 # define WARN_ON_IN_IRQ() 22 + #endif 23 + 24 + #ifdef CONFIG_ADDRESS_MASKING 25 + /* 26 + * Mask out tag bits from the address. 27 + * 28 + * Magic with the 'sign' allows to untag userspace pointer without any branches 29 + * while leaving kernel addresses intact. 30 + */ 31 + static inline unsigned long __untagged_addr(unsigned long addr) 32 + { 33 + long sign; 34 + 35 + /* 36 + * Refer tlbstate_untag_mask directly to avoid RIP-relative relocation 37 + * in alternative instructions. The relocation gets wrong when gets 38 + * copied to the target place. 39 + */ 40 + asm (ALTERNATIVE("", 41 + "sar $63, %[sign]\n\t" /* user_ptr ? 0 : -1UL */ 42 + "or %%gs:tlbstate_untag_mask, %[sign]\n\t" 43 + "and %[sign], %[addr]\n\t", X86_FEATURE_LAM) 44 + : [addr] "+r" (addr), [sign] "=r" (sign) 45 + : "m" (tlbstate_untag_mask), "[sign]" (addr)); 46 + 47 + return addr; 48 + } 49 + 50 + #define untagged_addr(addr) ({ \ 51 + unsigned long __addr = (__force unsigned long)(addr); \ 52 + (__force __typeof__(addr))__untagged_addr(__addr); \ 53 + }) 54 + 55 + static inline unsigned long __untagged_addr_remote(struct mm_struct *mm, 56 + unsigned long addr) 57 + { 58 + long sign = addr >> 63; 59 + 60 + mmap_assert_locked(mm); 61 + addr &= (mm)->context.untag_mask | sign; 62 + 63 + return addr; 64 + } 65 + 66 + #define untagged_addr_remote(mm, addr) ({ \ 67 + unsigned long __addr = (__force unsigned long)(addr); \ 68 + (__force __typeof__(addr))__untagged_addr_remote(mm, __addr); \ 69 + }) 70 + 71 + #else 72 + #define untagged_addr(addr) (addr) 25 73 #endif 26 74 27 75 /** ··· 92 38 * Return: true (nonzero) if the memory block may be valid, false (zero) 93 39 * if it is definitely invalid. 94 40 */ 95 - #define access_ok(addr, size) \ 41 + #define access_ok(addr, size) \ 96 42 ({ \ 97 43 WARN_ON_IN_IRQ(); \ 98 - likely(__access_ok(addr, size)); \ 44 + likely(__access_ok(untagged_addr(addr), size)); \ 99 45 }) 100 46 101 47 #include <asm-generic/access_ok.h>

+5

arch/x86/include/uapi/asm/prctl.h

··· 23 23 #define ARCH_MAP_VDSO_32 0x2002 24 24 #define ARCH_MAP_VDSO_64 0x2003 25 25 26 + #define ARCH_GET_UNTAG_MASK 0x4001 27 + #define ARCH_ENABLE_TAGGED_ADDR 0x4002 28 + #define ARCH_GET_MAX_TAG_BITS 0x4003 29 + #define ARCH_FORCE_TAGGED_SVA 0x4004 30 + 26 31 #endif /* _ASM_X86_PRCTL_H */

+6

arch/x86/include/uapi/asm/processor-flags.h

··· 82 82 #define X86_CR3_PCID_BITS 12 83 83 #define X86_CR3_PCID_MASK (_AC((1UL << X86_CR3_PCID_BITS) - 1, UL)) 84 84 85 + #define X86_CR3_LAM_U57_BIT 61 /* Activate LAM for userspace, 62:57 bits masked */ 86 + #define X86_CR3_LAM_U57 _BITULL(X86_CR3_LAM_U57_BIT) 87 + #define X86_CR3_LAM_U48_BIT 62 /* Activate LAM for userspace, 62:48 bits masked */ 88 + #define X86_CR3_LAM_U48 _BITULL(X86_CR3_LAM_U48_BIT) 85 89 #define X86_CR3_PCID_NOFLUSH_BIT 63 /* Preserve old PCID */ 86 90 #define X86_CR3_PCID_NOFLUSH _BITULL(X86_CR3_PCID_NOFLUSH_BIT) 87 91 ··· 136 132 #define X86_CR4_PKE _BITUL(X86_CR4_PKE_BIT) 137 133 #define X86_CR4_CET_BIT 23 /* enable Control-flow Enforcement Technology */ 138 134 #define X86_CR4_CET _BITUL(X86_CR4_CET_BIT) 135 + #define X86_CR4_LAM_SUP_BIT 28 /* LAM for supervisor pointers */ 136 + #define X86_CR4_LAM_SUP _BITUL(X86_CR4_LAM_SUP_BIT) 139 137 140 138 /* 141 139 * x86-64 Task Priority Register, CR8

+6

arch/x86/kernel/process.c

··· 48 48 #include <asm/frame.h> 49 49 #include <asm/unwind.h> 50 50 #include <asm/tdx.h> 51 + #include <asm/mmu_context.h> 51 52 52 53 #include "process.h" 53 54 ··· 163 162 164 163 savesegment(es, p->thread.es); 165 164 savesegment(ds, p->thread.ds); 165 + 166 + if (p->mm && (clone_flags & (CLONE_VM | CLONE_VFORK)) == CLONE_VM) 167 + set_bit(MM_CONTEXT_LOCK_LAM, &p->mm->context.flags); 166 168 #else 167 169 p->thread.sp0 = (unsigned long) (childregs + 1); 168 170 savesegment(gs, p->thread.gs); ··· 372 368 task_clear_spec_ssb_noexec(current); 373 369 speculation_ctrl_update(read_thread_flags()); 374 370 } 371 + 372 + mm_reset_untag_mask(current->mm); 375 373 } 376 374 377 375 #ifdef CONFIG_X86_IOPL_IOPERM

+65 -3

arch/x86/kernel/process_64.c

··· 671 671 task_pt_regs(current)->orig_ax = __NR_execve; 672 672 current_thread_info()->status &= ~TS_COMPAT; 673 673 if (current->mm) 674 - current->mm->context.flags = MM_CONTEXT_HAS_VSYSCALL; 674 + __set_bit(MM_CONTEXT_HAS_VSYSCALL, &current->mm->context.flags); 675 675 676 676 /* TBD: overwrites user setup. Should have two bits. 677 677 But 64bit processes have always behaved this way, ··· 708 708 * uprobes applied to this MM need to know this and 709 709 * cannot use user_64bit_mode() at that time. 710 710 */ 711 - current->mm->context.flags = MM_CONTEXT_UPROBE_IA32; 711 + __set_bit(MM_CONTEXT_UPROBE_IA32, &current->mm->context.flags); 712 712 } 713 713 714 714 current->personality |= force_personality32; ··· 740 740 return ret; 741 741 742 742 return (long)image->size; 743 + } 744 + #endif 745 + 746 + #ifdef CONFIG_ADDRESS_MASKING 747 + 748 + #define LAM_U57_BITS 6 749 + 750 + static int prctl_enable_tagged_addr(struct mm_struct *mm, unsigned long nr_bits) 751 + { 752 + if (!cpu_feature_enabled(X86_FEATURE_LAM)) 753 + return -ENODEV; 754 + 755 + /* PTRACE_ARCH_PRCTL */ 756 + if (current->mm != mm) 757 + return -EINVAL; 758 + 759 + if (mm_valid_pasid(mm) && 760 + !test_bit(MM_CONTEXT_FORCE_TAGGED_SVA, &mm->context.flags)) 761 + return -EINVAL; 762 + 763 + if (mmap_write_lock_killable(mm)) 764 + return -EINTR; 765 + 766 + if (test_bit(MM_CONTEXT_LOCK_LAM, &mm->context.flags)) { 767 + mmap_write_unlock(mm); 768 + return -EBUSY; 769 + } 770 + 771 + if (!nr_bits) { 772 + mmap_write_unlock(mm); 773 + return -EINVAL; 774 + } else if (nr_bits <= LAM_U57_BITS) { 775 + mm->context.lam_cr3_mask = X86_CR3_LAM_U57; 776 + mm->context.untag_mask = ~GENMASK(62, 57); 777 + } else { 778 + mmap_write_unlock(mm); 779 + return -EINVAL; 780 + } 781 + 782 + write_cr3(__read_cr3() | mm->context.lam_cr3_mask); 783 + set_tlbstate_lam_mode(mm); 784 + set_bit(MM_CONTEXT_LOCK_LAM, &mm->context.flags); 785 + 786 + mmap_write_unlock(mm); 787 + 788 + return 0; 743 789 } 744 790 #endif 745 791 ··· 876 830 case ARCH_MAP_VDSO_64: 877 831 return prctl_map_vdso(&vdso_image_64, arg2); 878 832 #endif 879 - 833 + #ifdef CONFIG_ADDRESS_MASKING 834 + case ARCH_GET_UNTAG_MASK: 835 + return put_user(task->mm->context.untag_mask, 836 + (unsigned long __user *)arg2); 837 + case ARCH_ENABLE_TAGGED_ADDR: 838 + return prctl_enable_tagged_addr(task->mm, arg2); 839 + case ARCH_FORCE_TAGGED_SVA: 840 + if (current != task) 841 + return -EINVAL; 842 + set_bit(MM_CONTEXT_FORCE_TAGGED_SVA, &task->mm->context.flags); 843 + return 0; 844 + case ARCH_GET_MAX_TAG_BITS: 845 + if (!cpu_feature_enabled(X86_FEATURE_LAM)) 846 + return put_user(0, (unsigned long __user *)arg2); 847 + else 848 + return put_user(LAM_U57_BITS, (unsigned long __user *)arg2); 849 + #endif 880 850 default: 881 851 ret = -EINVAL; 882 852 break;

+3 -3

arch/x86/kernel/traps.c

··· 671 671 if (!cpu_feature_enabled(X86_FEATURE_ENQCMD)) 672 672 return false; 673 673 674 - pasid = current->mm->pasid; 675 - 676 674 /* 677 675 * If the mm has not been allocated a 678 676 * PASID, the #GP can not be fixed up. 679 677 */ 680 - if (!pasid_valid(pasid)) 678 + if (!mm_valid_pasid(current->mm)) 681 679 return false; 680 + 681 + pasid = current->mm->pasid; 682 682 683 683 /* 684 684 * Did this thread already have its PASID activated?

+31 -52

arch/x86/lib/getuser.S

··· 37 37 38 38 #define ASM_BARRIER_NOSPEC ALTERNATIVE "", "lfence", X86_FEATURE_LFENCE_RDTSC 39 39 40 - #ifdef CONFIG_X86_5LEVEL 41 - #define LOAD_TASK_SIZE_MINUS_N(n) \ 42 - ALTERNATIVE __stringify(mov $((1 << 47) - 4096 - (n)),%rdx), \ 43 - __stringify(mov $((1 << 56) - 4096 - (n)),%rdx), X86_FEATURE_LA57 44 - #else 45 - #define LOAD_TASK_SIZE_MINUS_N(n) \ 46 - mov $(TASK_SIZE_MAX - (n)),%_ASM_DX 47 - #endif 40 + .macro check_range size:req 41 + .if IS_ENABLED(CONFIG_X86_64) 42 + mov %rax, %rdx 43 + sar $63, %rdx 44 + or %rdx, %rax 45 + .else 46 + cmp $TASK_SIZE_MAX-\size+1, %eax 47 + jae .Lbad_get_user 48 + sbb %edx, %edx /* array_index_mask_nospec() */ 49 + and %edx, %eax 50 + .endif 51 + .endm 48 52 49 53 .text 50 54 SYM_FUNC_START(__get_user_1) 51 - LOAD_TASK_SIZE_MINUS_N(0) 52 - cmp %_ASM_DX,%_ASM_AX 53 - jae bad_get_user 54 - sbb %_ASM_DX, %_ASM_DX /* array_index_mask_nospec() */ 55 - and %_ASM_DX, %_ASM_AX 55 + check_range size=1 56 56 ASM_STAC 57 57 1: movzbl (%_ASM_AX),%edx 58 58 xor %eax,%eax ··· 62 62 EXPORT_SYMBOL(__get_user_1) 63 63 64 64 SYM_FUNC_START(__get_user_2) 65 - LOAD_TASK_SIZE_MINUS_N(1) 66 - cmp %_ASM_DX,%_ASM_AX 67 - jae bad_get_user 68 - sbb %_ASM_DX, %_ASM_DX /* array_index_mask_nospec() */ 69 - and %_ASM_DX, %_ASM_AX 65 + check_range size=2 70 66 ASM_STAC 71 67 2: movzwl (%_ASM_AX),%edx 72 68 xor %eax,%eax ··· 72 76 EXPORT_SYMBOL(__get_user_2) 73 77 74 78 SYM_FUNC_START(__get_user_4) 75 - LOAD_TASK_SIZE_MINUS_N(3) 76 - cmp %_ASM_DX,%_ASM_AX 77 - jae bad_get_user 78 - sbb %_ASM_DX, %_ASM_DX /* array_index_mask_nospec() */ 79 - and %_ASM_DX, %_ASM_AX 79 + check_range size=4 80 80 ASM_STAC 81 81 3: movl (%_ASM_AX),%edx 82 82 xor %eax,%eax ··· 82 90 EXPORT_SYMBOL(__get_user_4) 83 91 84 92 SYM_FUNC_START(__get_user_8) 93 + check_range size=8 94 + ASM_STAC 85 95 #ifdef CONFIG_X86_64 86 - LOAD_TASK_SIZE_MINUS_N(7) 87 - cmp %_ASM_DX,%_ASM_AX 88 - jae bad_get_user 89 - sbb %_ASM_DX, %_ASM_DX /* array_index_mask_nospec() */ 90 - and %_ASM_DX, %_ASM_AX 91 - ASM_STAC 92 96 4: movq (%_ASM_AX),%rdx 93 - xor %eax,%eax 94 - ASM_CLAC 95 - RET 96 97 #else 97 - LOAD_TASK_SIZE_MINUS_N(7) 98 - cmp %_ASM_DX,%_ASM_AX 99 - jae bad_get_user_8 100 - sbb %_ASM_DX, %_ASM_DX /* array_index_mask_nospec() */ 101 - and %_ASM_DX, %_ASM_AX 102 - ASM_STAC 103 98 4: movl (%_ASM_AX),%edx 104 99 5: movl 4(%_ASM_AX),%ecx 100 + #endif 105 101 xor %eax,%eax 106 102 ASM_CLAC 107 103 RET 108 - #endif 109 104 SYM_FUNC_END(__get_user_8) 110 105 EXPORT_SYMBOL(__get_user_8) 111 106 ··· 145 166 146 167 SYM_CODE_START_LOCAL(.Lbad_get_user_clac) 147 168 ASM_CLAC 148 - bad_get_user: 169 + .Lbad_get_user: 149 170 xor %edx,%edx 150 171 mov $(-EFAULT),%_ASM_AX 151 172 RET ··· 163 184 #endif 164 185 165 186 /* get_user */ 166 - _ASM_EXTABLE_UA(1b, .Lbad_get_user_clac) 167 - _ASM_EXTABLE_UA(2b, .Lbad_get_user_clac) 168 - _ASM_EXTABLE_UA(3b, .Lbad_get_user_clac) 187 + _ASM_EXTABLE(1b, .Lbad_get_user_clac) 188 + _ASM_EXTABLE(2b, .Lbad_get_user_clac) 189 + _ASM_EXTABLE(3b, .Lbad_get_user_clac) 169 190 #ifdef CONFIG_X86_64 170 - _ASM_EXTABLE_UA(4b, .Lbad_get_user_clac) 191 + _ASM_EXTABLE(4b, .Lbad_get_user_clac) 171 192 #else 172 - _ASM_EXTABLE_UA(4b, .Lbad_get_user_8_clac) 173 - _ASM_EXTABLE_UA(5b, .Lbad_get_user_8_clac) 193 + _ASM_EXTABLE(4b, .Lbad_get_user_8_clac) 194 + _ASM_EXTABLE(5b, .Lbad_get_user_8_clac) 174 195 #endif 175 196 176 197 /* __get_user */ 177 - _ASM_EXTABLE_UA(6b, .Lbad_get_user_clac) 178 - _ASM_EXTABLE_UA(7b, .Lbad_get_user_clac) 179 - _ASM_EXTABLE_UA(8b, .Lbad_get_user_clac) 198 + _ASM_EXTABLE(6b, .Lbad_get_user_clac) 199 + _ASM_EXTABLE(7b, .Lbad_get_user_clac) 200 + _ASM_EXTABLE(8b, .Lbad_get_user_clac) 180 201 #ifdef CONFIG_X86_64 181 - _ASM_EXTABLE_UA(9b, .Lbad_get_user_clac) 202 + _ASM_EXTABLE(9b, .Lbad_get_user_clac) 182 203 #else 183 - _ASM_EXTABLE_UA(9b, .Lbad_get_user_8_clac) 184 - _ASM_EXTABLE_UA(10b, .Lbad_get_user_8_clac) 204 + _ASM_EXTABLE(9b, .Lbad_get_user_8_clac) 205 + _ASM_EXTABLE(10b, .Lbad_get_user_8_clac) 185 206 #endif

+24 -30

arch/x86/lib/putuser.S

··· 33 33 * as they get called from within inline assembly. 34 34 */ 35 35 36 - #ifdef CONFIG_X86_5LEVEL 37 - #define LOAD_TASK_SIZE_MINUS_N(n) \ 38 - ALTERNATIVE __stringify(mov $((1 << 47) - 4096 - (n)),%rbx), \ 39 - __stringify(mov $((1 << 56) - 4096 - (n)),%rbx), X86_FEATURE_LA57 40 - #else 41 - #define LOAD_TASK_SIZE_MINUS_N(n) \ 42 - mov $(TASK_SIZE_MAX - (n)),%_ASM_BX 43 - #endif 36 + .macro check_range size:req 37 + .if IS_ENABLED(CONFIG_X86_64) 38 + mov %rcx, %rbx 39 + sar $63, %rbx 40 + or %rbx, %rcx 41 + .else 42 + cmp $TASK_SIZE_MAX-\size+1, %ecx 43 + jae .Lbad_put_user 44 + .endif 45 + .endm 44 46 45 47 .text 46 48 SYM_FUNC_START(__put_user_1) 47 - LOAD_TASK_SIZE_MINUS_N(0) 48 - cmp %_ASM_BX,%_ASM_CX 49 - jae .Lbad_put_user 49 + check_range size=1 50 50 ASM_STAC 51 51 1: movb %al,(%_ASM_CX) 52 52 xor %ecx,%ecx ··· 66 66 EXPORT_SYMBOL(__put_user_nocheck_1) 67 67 68 68 SYM_FUNC_START(__put_user_2) 69 - LOAD_TASK_SIZE_MINUS_N(1) 70 - cmp %_ASM_BX,%_ASM_CX 71 - jae .Lbad_put_user 69 + check_range size=2 72 70 ASM_STAC 73 71 3: movw %ax,(%_ASM_CX) 74 72 xor %ecx,%ecx ··· 86 88 EXPORT_SYMBOL(__put_user_nocheck_2) 87 89 88 90 SYM_FUNC_START(__put_user_4) 89 - LOAD_TASK_SIZE_MINUS_N(3) 90 - cmp %_ASM_BX,%_ASM_CX 91 - jae .Lbad_put_user 91 + check_range size=4 92 92 ASM_STAC 93 93 5: movl %eax,(%_ASM_CX) 94 94 xor %ecx,%ecx ··· 106 110 EXPORT_SYMBOL(__put_user_nocheck_4) 107 111 108 112 SYM_FUNC_START(__put_user_8) 109 - LOAD_TASK_SIZE_MINUS_N(7) 110 - cmp %_ASM_BX,%_ASM_CX 111 - jae .Lbad_put_user 113 + check_range size=8 112 114 ASM_STAC 113 115 7: mov %_ASM_AX,(%_ASM_CX) 114 116 #ifdef CONFIG_X86_32 ··· 138 144 RET 139 145 SYM_CODE_END(.Lbad_put_user_clac) 140 146 141 - _ASM_EXTABLE_UA(1b, .Lbad_put_user_clac) 142 - _ASM_EXTABLE_UA(2b, .Lbad_put_user_clac) 143 - _ASM_EXTABLE_UA(3b, .Lbad_put_user_clac) 144 - _ASM_EXTABLE_UA(4b, .Lbad_put_user_clac) 145 - _ASM_EXTABLE_UA(5b, .Lbad_put_user_clac) 146 - _ASM_EXTABLE_UA(6b, .Lbad_put_user_clac) 147 - _ASM_EXTABLE_UA(7b, .Lbad_put_user_clac) 148 - _ASM_EXTABLE_UA(9b, .Lbad_put_user_clac) 147 + _ASM_EXTABLE(1b, .Lbad_put_user_clac) 148 + _ASM_EXTABLE(2b, .Lbad_put_user_clac) 149 + _ASM_EXTABLE(3b, .Lbad_put_user_clac) 150 + _ASM_EXTABLE(4b, .Lbad_put_user_clac) 151 + _ASM_EXTABLE(5b, .Lbad_put_user_clac) 152 + _ASM_EXTABLE(6b, .Lbad_put_user_clac) 153 + _ASM_EXTABLE(7b, .Lbad_put_user_clac) 154 + _ASM_EXTABLE(9b, .Lbad_put_user_clac) 149 155 #ifdef CONFIG_X86_32 150 - _ASM_EXTABLE_UA(8b, .Lbad_put_user_clac) 151 - _ASM_EXTABLE_UA(10b, .Lbad_put_user_clac) 156 + _ASM_EXTABLE(8b, .Lbad_put_user_clac) 157 + _ASM_EXTABLE(10b, .Lbad_put_user_clac) 152 158 #endif

+5

arch/x86/mm/init.c

··· 1048 1048 .cr4 = ~0UL, /* fail hard if we screw up cr4 shadow initialization */ 1049 1049 }; 1050 1050 1051 + #ifdef CONFIG_ADDRESS_MASKING 1052 + DEFINE_PER_CPU(u64, tlbstate_untag_mask); 1053 + EXPORT_PER_CPU_SYMBOL(tlbstate_untag_mask); 1054 + #endif 1055 + 1051 1056 void update_cache_mode_entry(unsigned entry, enum page_cache_mode cache) 1052 1057 { 1053 1058 /* entry 0 MUST be WB (hardwired to speed up translations) */

+37 -16

arch/x86/mm/tlb.c

··· 154 154 return ret; 155 155 } 156 156 157 - static inline unsigned long build_cr3(pgd_t *pgd, u16 asid) 157 + static inline unsigned long build_cr3(pgd_t *pgd, u16 asid, unsigned long lam) 158 158 { 159 + unsigned long cr3 = __sme_pa(pgd) | lam; 160 + 159 161 if (static_cpu_has(X86_FEATURE_PCID)) { 160 - return __sme_pa(pgd) | kern_pcid(asid); 162 + VM_WARN_ON_ONCE(asid > MAX_ASID_AVAILABLE); 163 + cr3 |= kern_pcid(asid); 161 164 } else { 162 165 VM_WARN_ON_ONCE(asid != 0); 163 - return __sme_pa(pgd); 164 166 } 167 + 168 + return cr3; 165 169 } 166 170 167 - static inline unsigned long build_cr3_noflush(pgd_t *pgd, u16 asid) 171 + static inline unsigned long build_cr3_noflush(pgd_t *pgd, u16 asid, 172 + unsigned long lam) 168 173 { 169 - VM_WARN_ON_ONCE(asid > MAX_ASID_AVAILABLE); 170 174 /* 171 175 * Use boot_cpu_has() instead of this_cpu_has() as this function 172 176 * might be called during early boot. This should work even after 173 177 * boot because all CPU's the have same capabilities: 174 178 */ 175 179 VM_WARN_ON_ONCE(!boot_cpu_has(X86_FEATURE_PCID)); 176 - return __sme_pa(pgd) | kern_pcid(asid) | CR3_NOFLUSH; 180 + return build_cr3(pgd, asid, lam) | CR3_NOFLUSH; 177 181 } 178 182 179 183 /* ··· 278 274 (unsigned long *)this_cpu_ptr(&cpu_tlbstate.user_pcid_flush_mask)); 279 275 } 280 276 281 - static void load_new_mm_cr3(pgd_t *pgdir, u16 new_asid, bool need_flush) 277 + static void load_new_mm_cr3(pgd_t *pgdir, u16 new_asid, unsigned long lam, 278 + bool need_flush) 282 279 { 283 280 unsigned long new_mm_cr3; 284 281 285 282 if (need_flush) { 286 283 invalidate_user_asid(new_asid); 287 - new_mm_cr3 = build_cr3(pgdir, new_asid); 284 + new_mm_cr3 = build_cr3(pgdir, new_asid, lam); 288 285 } else { 289 - new_mm_cr3 = build_cr3_noflush(pgdir, new_asid); 286 + new_mm_cr3 = build_cr3_noflush(pgdir, new_asid, lam); 290 287 } 291 288 292 289 /* ··· 496 491 { 497 492 struct mm_struct *real_prev = this_cpu_read(cpu_tlbstate.loaded_mm); 498 493 u16 prev_asid = this_cpu_read(cpu_tlbstate.loaded_mm_asid); 494 + unsigned long new_lam = mm_lam_cr3_mask(next); 499 495 bool was_lazy = this_cpu_read(cpu_tlbstate_shared.is_lazy); 500 496 unsigned cpu = smp_processor_id(); 501 497 u64 next_tlb_gen; ··· 526 520 * isn't free. 527 521 */ 528 522 #ifdef CONFIG_DEBUG_VM 529 - if (WARN_ON_ONCE(__read_cr3() != build_cr3(real_prev->pgd, prev_asid))) { 523 + if (WARN_ON_ONCE(__read_cr3() != build_cr3(real_prev->pgd, prev_asid, 524 + tlbstate_lam_cr3_mask()))) { 530 525 /* 531 526 * If we were to BUG here, we'd be very likely to kill 532 527 * the system so hard that we don't see the call trace. ··· 559 552 * instruction. 560 553 */ 561 554 if (real_prev == next) { 555 + /* Not actually switching mm's */ 562 556 VM_WARN_ON(this_cpu_read(cpu_tlbstate.ctxs[prev_asid].ctx_id) != 563 557 next->context.ctx_id); 558 + 559 + /* 560 + * If this races with another thread that enables lam, 'new_lam' 561 + * might not match tlbstate_lam_cr3_mask(). 562 + */ 564 563 565 564 /* 566 565 * Even in lazy TLB mode, the CPU should stay set in the ··· 635 622 barrier(); 636 623 } 637 624 625 + set_tlbstate_lam_mode(next); 638 626 if (need_flush) { 639 627 this_cpu_write(cpu_tlbstate.ctxs[new_asid].ctx_id, next->context.ctx_id); 640 628 this_cpu_write(cpu_tlbstate.ctxs[new_asid].tlb_gen, next_tlb_gen); 641 - load_new_mm_cr3(next->pgd, new_asid, true); 629 + load_new_mm_cr3(next->pgd, new_asid, new_lam, true); 642 630 643 631 trace_tlb_flush(TLB_FLUSH_ON_TASK_SWITCH, TLB_FLUSH_ALL); 644 632 } else { 645 633 /* The new ASID is already up to date. */ 646 - load_new_mm_cr3(next->pgd, new_asid, false); 634 + load_new_mm_cr3(next->pgd, new_asid, new_lam, false); 647 635 648 636 trace_tlb_flush(TLB_FLUSH_ON_TASK_SWITCH, 0); 649 637 } ··· 705 691 /* Assert that CR3 already references the right mm. */ 706 692 WARN_ON((cr3 & CR3_ADDR_MASK) != __pa(mm->pgd)); 707 693 694 + /* LAM expected to be disabled */ 695 + WARN_ON(cr3 & (X86_CR3_LAM_U48 | X86_CR3_LAM_U57)); 696 + WARN_ON(mm_lam_cr3_mask(mm)); 697 + 708 698 /* 709 699 * Assert that CR4.PCIDE is set if needed. (CR4.PCIDE initialization 710 700 * doesn't work like other CR4 bits because it can only be set from ··· 717 699 WARN_ON(boot_cpu_has(X86_FEATURE_PCID) && 718 700 !(cr4_read_shadow() & X86_CR4_PCIDE)); 719 701 720 - /* Force ASID 0 and force a TLB flush. */ 721 - write_cr3(build_cr3(mm->pgd, 0)); 702 + /* Disable LAM, force ASID 0 and force a TLB flush. */ 703 + write_cr3(build_cr3(mm->pgd, 0, 0)); 722 704 723 705 /* Reinitialize tlbstate. */ 724 706 this_cpu_write(cpu_tlbstate.last_user_mm_spec, LAST_USER_MM_INIT); ··· 726 708 this_cpu_write(cpu_tlbstate.next_asid, 1); 727 709 this_cpu_write(cpu_tlbstate.ctxs[0].ctx_id, mm->context.ctx_id); 728 710 this_cpu_write(cpu_tlbstate.ctxs[0].tlb_gen, tlb_gen); 711 + set_tlbstate_lam_mode(mm); 729 712 730 713 for (i = 1; i < TLB_NR_DYN_ASIDS; i++) 731 714 this_cpu_write(cpu_tlbstate.ctxs[i].ctx_id, 0); ··· 1090 1071 */ 1091 1072 unsigned long __get_current_cr3_fast(void) 1092 1073 { 1093 - unsigned long cr3 = build_cr3(this_cpu_read(cpu_tlbstate.loaded_mm)->pgd, 1094 - this_cpu_read(cpu_tlbstate.loaded_mm_asid)); 1074 + unsigned long cr3 = 1075 + build_cr3(this_cpu_read(cpu_tlbstate.loaded_mm)->pgd, 1076 + this_cpu_read(cpu_tlbstate.loaded_mm_asid), 1077 + tlbstate_lam_cr3_mask()); 1095 1078 1096 1079 /* For now, be very restrictive about when this can be called. */ 1097 1080 VM_WARN_ON(in_nmi() || preemptible());

+6 -2

drivers/iommu/iommu-sva.c

··· 2 2 /* 3 3 * Helpers for IOMMU drivers implementing SVA 4 4 */ 5 + #include <linux/mmu_context.h> 5 6 #include <linux/mutex.h> 6 7 #include <linux/sched/mm.h> 7 8 #include <linux/iommu.h> ··· 33 32 min == 0 || max < min) 34 33 return -EINVAL; 35 34 35 + if (!arch_pgtable_dma_compat(mm)) 36 + return -EBUSY; 37 + 36 38 mutex_lock(&iommu_sva_lock); 37 39 /* Is a PASID already associated with this mm? */ 38 - if (pasid_valid(mm->pasid)) { 40 + if (mm_valid_pasid(mm)) { 39 41 if (mm->pasid < min || mm->pasid >= max) 40 42 ret = -EOVERFLOW; 41 43 goto out; 42 44 } 43 45 44 46 pasid = ioasid_alloc(&iommu_sva_pasid, min, max, mm); 45 - if (!pasid_valid(pasid)) 47 + if (pasid == INVALID_IOASID) 46 48 ret = -ENOMEM; 47 49 else 48 50 mm_pasid_set(mm, pasid);

+1 -1

drivers/vfio/vfio_iommu_type1.c

··· 580 580 goto done; 581 581 } 582 582 583 - vaddr = untagged_addr(vaddr); 583 + vaddr = untagged_addr_remote(mm, vaddr); 584 584 585 585 retry: 586 586 vma = vma_lookup(mm, vaddr);

+7

fs/proc/array.c

··· 91 91 #include <linux/user_namespace.h> 92 92 #include <linux/fs_struct.h> 93 93 #include <linux/kthread.h> 94 + #include <linux/mmu_context.h> 94 95 95 96 #include <asm/processor.h> 96 97 #include "internal.h" ··· 426 425 seq_printf(m, "THP_enabled:\t%d\n", thp_enabled); 427 426 } 428 427 428 + static inline void task_untag_mask(struct seq_file *m, struct mm_struct *mm) 429 + { 430 + seq_printf(m, "untag_mask:\t%#lx\n", mm_untag_mask(mm)); 431 + } 432 + 429 433 int proc_pid_status(struct seq_file *m, struct pid_namespace *ns, 430 434 struct pid *pid, struct task_struct *task) 431 435 { ··· 446 440 task_mem(m, mm); 447 441 task_core_dumping(m, task); 448 442 task_thp_status(m, mm); 443 + task_untag_mask(m, mm); 449 444 mmput(mm); 450 445 } 451 446 task_sig(m, task);

+7 -2

fs/proc/task_mmu.c

··· 1688 1688 1689 1689 /* watch out for wraparound */ 1690 1690 start_vaddr = end_vaddr; 1691 - if (svpfn <= (ULONG_MAX >> PAGE_SHIFT)) 1692 - start_vaddr = untagged_addr(svpfn << PAGE_SHIFT); 1691 + if (svpfn <= (ULONG_MAX >> PAGE_SHIFT)) { 1692 + ret = mmap_read_lock_killable(mm); 1693 + if (ret) 1694 + goto out_free; 1695 + start_vaddr = untagged_addr_remote(mm, svpfn << PAGE_SHIFT); 1696 + mmap_read_unlock(mm); 1697 + } 1693 1698 1694 1699 /* Ensure the address is inside the task */ 1695 1700 if (start_vaddr > mm->task_size)

-9

include/linux/ioasid.h

··· 40 40 int ioasid_register_allocator(struct ioasid_allocator_ops *allocator); 41 41 void ioasid_unregister_allocator(struct ioasid_allocator_ops *allocator); 42 42 int ioasid_set_data(ioasid_t ioasid, void *data); 43 - static inline bool pasid_valid(ioasid_t ioasid) 44 - { 45 - return ioasid != INVALID_IOASID; 46 - } 47 43 48 44 #else /* !CONFIG_IOASID */ 49 45 static inline ioasid_t ioasid_alloc(struct ioasid_set *set, ioasid_t min, ··· 68 72 static inline int ioasid_set_data(ioasid_t ioasid, void *data) 69 73 { 70 74 return -ENOTSUPP; 71 - } 72 - 73 - static inline bool pasid_valid(ioasid_t ioasid) 74 - { 75 - return false; 76 75 } 77 76 78 77 #endif /* CONFIG_IOASID */

-11

include/linux/mm.h

··· 98 98 #include <asm/page.h> 99 99 #include <asm/processor.h> 100 100 101 - /* 102 - * Architectures that support memory tagging (assigning tags to memory regions, 103 - * embedding these tags into addresses that point to these memory regions, and 104 - * checking that the memory and the pointer tags match on memory accesses) 105 - * redefine this macro to strip tags from pointers. 106 - * It's defined as noop for architectures that don't support memory tagging. 107 - */ 108 - #ifndef untagged_addr 109 - #define untagged_addr(addr) (addr) 110 - #endif 111 - 112 101 #ifndef __pa_symbol 113 102 #define __pa_symbol(x) __pa(RELOC_HIDE((unsigned long)(x), 0)) 114 103 #endif

+14

include/linux/mmu_context.h

··· 28 28 # define task_cpu_possible(cpu, p) cpumask_test_cpu((cpu), task_cpu_possible_mask(p)) 29 29 #endif 30 30 31 + #ifndef mm_untag_mask 32 + static inline unsigned long mm_untag_mask(struct mm_struct *mm) 33 + { 34 + return -1UL; 35 + } 36 + #endif 37 + 38 + #ifndef arch_pgtable_dma_compat 39 + static inline bool arch_pgtable_dma_compat(struct mm_struct *mm) 40 + { 41 + return true; 42 + } 43 + #endif 44 + 31 45 #endif

+7 -1

include/linux/sched/mm.h

··· 485 485 mm->pasid = INVALID_IOASID; 486 486 } 487 487 488 + static inline bool mm_valid_pasid(struct mm_struct *mm) 489 + { 490 + return mm->pasid != INVALID_IOASID; 491 + } 492 + 488 493 /* Associate a PASID with an mm_struct: */ 489 494 static inline void mm_pasid_set(struct mm_struct *mm, u32 pasid) 490 495 { ··· 498 493 499 494 static inline void mm_pasid_drop(struct mm_struct *mm) 500 495 { 501 - if (pasid_valid(mm->pasid)) { 496 + if (mm_valid_pasid(mm)) { 502 497 ioasid_free(mm->pasid); 503 498 mm->pasid = INVALID_IOASID; 504 499 } 505 500 } 506 501 #else 507 502 static inline void mm_pasid_init(struct mm_struct *mm) {} 503 + static inline bool mm_valid_pasid(struct mm_struct *mm) { return false; } 508 504 static inline void mm_pasid_set(struct mm_struct *mm, u32 pasid) {} 509 505 static inline void mm_pasid_drop(struct mm_struct *mm) {} 510 506 #endif

+22

include/linux/uaccess.h

··· 11 11 #include <asm/uaccess.h> 12 12 13 13 /* 14 + * Architectures that support memory tagging (assigning tags to memory regions, 15 + * embedding these tags into addresses that point to these memory regions, and 16 + * checking that the memory and the pointer tags match on memory accesses) 17 + * redefine this macro to strip tags from pointers. 18 + * 19 + * Passing down mm_struct allows to define untagging rules on per-process 20 + * basis. 21 + * 22 + * It's defined as noop for architectures that don't support memory tagging. 23 + */ 24 + #ifndef untagged_addr 25 + #define untagged_addr(addr) (addr) 26 + #endif 27 + 28 + #ifndef untagged_addr_remote 29 + #define untagged_addr_remote(mm, addr) ({ \ 30 + mmap_assert_locked(mm); \ 31 + untagged_addr(addr); \ 32 + }) 33 + #endif 34 + 35 + /* 14 36 * Architectures should provide two primitives (raw_copy_{to,from}_user()) 15 37 * and get rid of their private instances of copy_{to,from}_user() and 16 38 * __copy_{to,from}_user{,_inatomic}().

+2 -2

mm/gup.c

··· 1085 1085 if (!nr_pages) 1086 1086 return 0; 1087 1087 1088 - start = untagged_addr(start); 1088 + start = untagged_addr_remote(mm, start); 1089 1089 1090 1090 VM_BUG_ON(!!pages != !!(gup_flags & (FOLL_GET | FOLL_PIN))); 1091 1091 ··· 1259 1259 struct vm_area_struct *vma; 1260 1260 vm_fault_t ret; 1261 1261 1262 - address = untagged_addr(address); 1262 + address = untagged_addr_remote(mm, address); 1263 1263 1264 1264 if (unlocked) 1265 1265 fault_flags |= FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE;

+3 -2

mm/madvise.c

··· 1390 1390 size_t len; 1391 1391 struct blk_plug plug; 1392 1392 1393 - start = untagged_addr(start); 1394 - 1395 1393 if (!madvise_behavior_valid(behavior)) 1396 1394 return -EINVAL; 1397 1395 ··· 1420 1422 } else { 1421 1423 mmap_read_lock(mm); 1422 1424 } 1425 + 1426 + start = untagged_addr_remote(mm, start); 1427 + end = start + len; 1423 1428 1424 1429 blk_start_plug(&plug); 1425 1430 error = madvise_walk_vmas(mm, start, end, behavior,

+6 -5

mm/migrate.c

··· 2099 2099 * target node 2100 2100 * 1 - when it has been queued 2101 2101 */ 2102 - static int add_page_for_migration(struct mm_struct *mm, unsigned long addr, 2102 + static int add_page_for_migration(struct mm_struct *mm, const void __user *p, 2103 2103 int node, struct list_head *pagelist, bool migrate_all) 2104 2104 { 2105 2105 struct vm_area_struct *vma; 2106 + unsigned long addr; 2106 2107 struct page *page; 2107 2108 int err; 2108 2109 bool isolated; 2109 2110 2110 2111 mmap_read_lock(mm); 2112 + addr = (unsigned long)untagged_addr_remote(mm, p); 2113 + 2111 2114 err = -EFAULT; 2112 2115 vma = vma_lookup(mm, addr); 2113 2116 if (!vma || !vma_migratable(vma)) ··· 2216 2213 2217 2214 for (i = start = 0; i < nr_pages; i++) { 2218 2215 const void __user *p; 2219 - unsigned long addr; 2220 2216 int node; 2221 2217 2222 2218 err = -EFAULT; ··· 2223 2221 goto out_flush; 2224 2222 if (get_user(node, nodes + i)) 2225 2223 goto out_flush; 2226 - addr = (unsigned long)untagged_addr(p); 2227 2224 2228 2225 err = -ENODEV; 2229 2226 if (node < 0 || node >= MAX_NUMNODES) ··· 2250 2249 * Errors in the page lookup or isolation are not fatal and we simply 2251 2250 * report them via status 2252 2251 */ 2253 - err = add_page_for_migration(mm, addr, current_node, 2254 - &pagelist, flags & MPOL_MF_MOVE_ALL); 2252 + err = add_page_for_migration(mm, p, current_node, &pagelist, 2253 + flags & MPOL_MF_MOVE_ALL); 2255 2254 2256 2255 if (err > 0) { 2257 2256 /* The page is successfully queued for migration */

+1 -1

tools/testing/selftests/x86/Makefile

··· 18 18 test_FCMOV test_FCOMI test_FISTTP \ 19 19 vdso_restorer 20 20 TARGETS_C_64BIT_ONLY := fsgsbase sysret_rip syscall_numbering \ 21 - corrupt_xstate_header amx 21 + corrupt_xstate_header amx lam 22 22 # Some selftests require 32bit support enabled also on 64bit systems 23 23 TARGETS_C_32BIT_NEEDED := ldt_gdt ptrace_syscall 24 24

+1241

tools/testing/selftests/x86/lam.c

··· 1 + // SPDX-License-Identifier: GPL-2.0 2 + #define _GNU_SOURCE 3 + #include <stdio.h> 4 + #include <stdlib.h> 5 + #include <string.h> 6 + #include <sys/syscall.h> 7 + #include <time.h> 8 + #include <signal.h> 9 + #include <setjmp.h> 10 + #include <sys/mman.h> 11 + #include <sys/utsname.h> 12 + #include <sys/wait.h> 13 + #include <sys/stat.h> 14 + #include <fcntl.h> 15 + #include <inttypes.h> 16 + #include <sched.h> 17 + 18 + #include <sys/uio.h> 19 + #include <linux/io_uring.h> 20 + #include "../kselftest.h" 21 + 22 + #ifndef __x86_64__ 23 + # error This test is 64-bit only 24 + #endif 25 + 26 + /* LAM modes, these definitions were copied from kernel code */ 27 + #define LAM_NONE 0 28 + #define LAM_U57_BITS 6 29 + 30 + #define LAM_U57_MASK (0x3fULL << 57) 31 + /* arch prctl for LAM */ 32 + #define ARCH_GET_UNTAG_MASK 0x4001 33 + #define ARCH_ENABLE_TAGGED_ADDR 0x4002 34 + #define ARCH_GET_MAX_TAG_BITS 0x4003 35 + #define ARCH_FORCE_TAGGED_SVA 0x4004 36 + 37 + /* Specified test function bits */ 38 + #define FUNC_MALLOC 0x1 39 + #define FUNC_BITS 0x2 40 + #define FUNC_MMAP 0x4 41 + #define FUNC_SYSCALL 0x8 42 + #define FUNC_URING 0x10 43 + #define FUNC_INHERITE 0x20 44 + #define FUNC_PASID 0x40 45 + 46 + #define TEST_MASK 0x7f 47 + 48 + #define LOW_ADDR (0x1UL << 30) 49 + #define HIGH_ADDR (0x3UL << 48) 50 + 51 + #define MALLOC_LEN 32 52 + 53 + #define PAGE_SIZE (4 << 10) 54 + 55 + #define STACK_SIZE 65536 56 + 57 + #define barrier() ({ \ 58 + __asm__ __volatile__("" : : : "memory"); \ 59 + }) 60 + 61 + #define URING_QUEUE_SZ 1 62 + #define URING_BLOCK_SZ 2048 63 + 64 + /* Pasid test define */ 65 + #define LAM_CMD_BIT 0x1 66 + #define PAS_CMD_BIT 0x2 67 + #define SVA_CMD_BIT 0x4 68 + 69 + #define PAS_CMD(cmd1, cmd2, cmd3) (((cmd3) << 8) | ((cmd2) << 4) | ((cmd1) << 0)) 70 + 71 + struct testcases { 72 + unsigned int later; 73 + int expected; /* 2: SIGSEGV Error; 1: other errors */ 74 + unsigned long lam; 75 + uint64_t addr; 76 + uint64_t cmd; 77 + int (*test_func)(struct testcases *test); 78 + const char *msg; 79 + }; 80 + 81 + /* Used by CQ of uring, source file handler and file's size */ 82 + struct file_io { 83 + int file_fd; 84 + off_t file_sz; 85 + struct iovec iovecs[]; 86 + }; 87 + 88 + struct io_uring_queue { 89 + unsigned int *head; 90 + unsigned int *tail; 91 + unsigned int *ring_mask; 92 + unsigned int *ring_entries; 93 + unsigned int *flags; 94 + unsigned int *array; 95 + union { 96 + struct io_uring_cqe *cqes; 97 + struct io_uring_sqe *sqes; 98 + } queue; 99 + size_t ring_sz; 100 + }; 101 + 102 + struct io_ring { 103 + int ring_fd; 104 + struct io_uring_queue sq_ring; 105 + struct io_uring_queue cq_ring; 106 + }; 107 + 108 + int tests_cnt; 109 + jmp_buf segv_env; 110 + 111 + static void segv_handler(int sig) 112 + { 113 + ksft_print_msg("Get segmentation fault(%d).", sig); 114 + 115 + siglongjmp(segv_env, 1); 116 + } 117 + 118 + static inline int cpu_has_lam(void) 119 + { 120 + unsigned int cpuinfo[4]; 121 + 122 + __cpuid_count(0x7, 1, cpuinfo[0], cpuinfo[1], cpuinfo[2], cpuinfo[3]); 123 + 124 + return (cpuinfo[0] & (1 << 26)); 125 + } 126 + 127 + /* Check 5-level page table feature in CPUID.(EAX=07H, ECX=00H):ECX.[bit 16] */ 128 + static inline int cpu_has_la57(void) 129 + { 130 + unsigned int cpuinfo[4]; 131 + 132 + __cpuid_count(0x7, 0, cpuinfo[0], cpuinfo[1], cpuinfo[2], cpuinfo[3]); 133 + 134 + return (cpuinfo[2] & (1 << 16)); 135 + } 136 + 137 + /* 138 + * Set tagged address and read back untag mask. 139 + * check if the untagged mask is expected. 140 + * 141 + * @return: 142 + * 0: Set LAM mode successfully 143 + * others: failed to set LAM 144 + */ 145 + static int set_lam(unsigned long lam) 146 + { 147 + int ret = 0; 148 + uint64_t ptr = 0; 149 + 150 + if (lam != LAM_U57_BITS && lam != LAM_NONE) 151 + return -1; 152 + 153 + /* Skip check return */ 154 + syscall(SYS_arch_prctl, ARCH_ENABLE_TAGGED_ADDR, lam); 155 + 156 + /* Get untagged mask */ 157 + syscall(SYS_arch_prctl, ARCH_GET_UNTAG_MASK, &ptr); 158 + 159 + /* Check mask returned is expected */ 160 + if (lam == LAM_U57_BITS) 161 + ret = (ptr != ~(LAM_U57_MASK)); 162 + else if (lam == LAM_NONE) 163 + ret = (ptr != -1ULL); 164 + 165 + return ret; 166 + } 167 + 168 + static unsigned long get_default_tag_bits(void) 169 + { 170 + pid_t pid; 171 + int lam = LAM_NONE; 172 + int ret = 0; 173 + 174 + pid = fork(); 175 + if (pid < 0) { 176 + perror("Fork failed."); 177 + } else if (pid == 0) { 178 + /* Set LAM mode in child process */ 179 + if (set_lam(LAM_U57_BITS) == 0) 180 + lam = LAM_U57_BITS; 181 + else 182 + lam = LAM_NONE; 183 + exit(lam); 184 + } else { 185 + wait(&ret); 186 + lam = WEXITSTATUS(ret); 187 + } 188 + 189 + return lam; 190 + } 191 + 192 + /* 193 + * Set tagged address and read back untag mask. 194 + * check if the untag mask is expected. 195 + */ 196 + static int get_lam(void) 197 + { 198 + uint64_t ptr = 0; 199 + int ret = -1; 200 + /* Get untagged mask */ 201 + if (syscall(SYS_arch_prctl, ARCH_GET_UNTAG_MASK, &ptr) == -1) 202 + return -1; 203 + 204 + /* Check mask returned is expected */ 205 + if (ptr == ~(LAM_U57_MASK)) 206 + ret = LAM_U57_BITS; 207 + else if (ptr == -1ULL) 208 + ret = LAM_NONE; 209 + 210 + 211 + return ret; 212 + } 213 + 214 + /* According to LAM mode, set metadata in high bits */ 215 + static uint64_t set_metadata(uint64_t src, unsigned long lam) 216 + { 217 + uint64_t metadata; 218 + 219 + srand(time(NULL)); 220 + 221 + switch (lam) { 222 + case LAM_U57_BITS: /* Set metadata in bits 62:57 */ 223 + /* Get a random non-zero value as metadata */ 224 + metadata = (rand() % ((1UL << LAM_U57_BITS) - 1) + 1) << 57; 225 + metadata |= (src & ~(LAM_U57_MASK)); 226 + break; 227 + default: 228 + metadata = src; 229 + break; 230 + } 231 + 232 + return metadata; 233 + } 234 + 235 + /* 236 + * Set metadata in user pointer, compare new pointer with original pointer. 237 + * both pointers should point to the same address. 238 + * 239 + * @return: 240 + * 0: value on the pointer with metadate and value on original are same 241 + * 1: not same. 242 + */ 243 + static int handle_lam_test(void *src, unsigned int lam) 244 + { 245 + char *ptr; 246 + 247 + strcpy((char *)src, "USER POINTER"); 248 + 249 + ptr = (char *)set_metadata((uint64_t)src, lam); 250 + if (src == ptr) 251 + return 0; 252 + 253 + /* Copy a string into the pointer with metadata */ 254 + strcpy((char *)ptr, "METADATA POINTER"); 255 + 256 + return (!!strcmp((char *)src, (char *)ptr)); 257 + } 258 + 259 + 260 + int handle_max_bits(struct testcases *test) 261 + { 262 + unsigned long exp_bits = get_default_tag_bits(); 263 + unsigned long bits = 0; 264 + 265 + if (exp_bits != LAM_NONE) 266 + exp_bits = LAM_U57_BITS; 267 + 268 + /* Get LAM max tag bits */ 269 + if (syscall(SYS_arch_prctl, ARCH_GET_MAX_TAG_BITS, &bits) == -1) 270 + return 1; 271 + 272 + return (exp_bits != bits); 273 + } 274 + 275 + /* 276 + * Test lam feature through dereference pointer get from malloc. 277 + * @return 0: Pass test. 1: Get failure during test 2: Get SIGSEGV 278 + */ 279 + static int handle_malloc(struct testcases *test) 280 + { 281 + char *ptr = NULL; 282 + int ret = 0; 283 + 284 + if (test->later == 0 && test->lam != 0) 285 + if (set_lam(test->lam) == -1) 286 + return 1; 287 + 288 + ptr = (char *)malloc(MALLOC_LEN); 289 + if (ptr == NULL) { 290 + perror("malloc() failure\n"); 291 + return 1; 292 + } 293 + 294 + /* Set signal handler */ 295 + if (sigsetjmp(segv_env, 1) == 0) { 296 + signal(SIGSEGV, segv_handler); 297 + ret = handle_lam_test(ptr, test->lam); 298 + } else { 299 + ret = 2; 300 + } 301 + 302 + if (test->later != 0 && test->lam != 0) 303 + if (set_lam(test->lam) == -1 && ret == 0) 304 + ret = 1; 305 + 306 + free(ptr); 307 + 308 + return ret; 309 + } 310 + 311 + static int handle_mmap(struct testcases *test) 312 + { 313 + void *ptr; 314 + unsigned int flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED; 315 + int ret = 0; 316 + 317 + if (test->later == 0 && test->lam != 0) 318 + if (set_lam(test->lam) != 0) 319 + return 1; 320 + 321 + ptr = mmap((void *)test->addr, PAGE_SIZE, PROT_READ | PROT_WRITE, 322 + flags, -1, 0); 323 + if (ptr == MAP_FAILED) { 324 + if (test->addr == HIGH_ADDR) 325 + if (!cpu_has_la57()) 326 + return 3; /* unsupport LA57 */ 327 + return 1; 328 + } 329 + 330 + if (test->later != 0 && test->lam != 0) 331 + if (set_lam(test->lam) != 0) 332 + ret = 1; 333 + 334 + if (ret == 0) { 335 + if (sigsetjmp(segv_env, 1) == 0) { 336 + signal(SIGSEGV, segv_handler); 337 + ret = handle_lam_test(ptr, test->lam); 338 + } else { 339 + ret = 2; 340 + } 341 + } 342 + 343 + munmap(ptr, PAGE_SIZE); 344 + return ret; 345 + } 346 + 347 + static int handle_syscall(struct testcases *test) 348 + { 349 + struct utsname unme, *pu; 350 + int ret = 0; 351 + 352 + if (test->later == 0 && test->lam != 0) 353 + if (set_lam(test->lam) != 0) 354 + return 1; 355 + 356 + if (sigsetjmp(segv_env, 1) == 0) { 357 + signal(SIGSEGV, segv_handler); 358 + pu = (struct utsname *)set_metadata((uint64_t)&unme, test->lam); 359 + ret = uname(pu); 360 + if (ret < 0) 361 + ret = 1; 362 + } else { 363 + ret = 2; 364 + } 365 + 366 + if (test->later != 0 && test->lam != 0) 367 + if (set_lam(test->lam) != -1 && ret == 0) 368 + ret = 1; 369 + 370 + return ret; 371 + } 372 + 373 + int sys_uring_setup(unsigned int entries, struct io_uring_params *p) 374 + { 375 + return (int)syscall(__NR_io_uring_setup, entries, p); 376 + } 377 + 378 + int sys_uring_enter(int fd, unsigned int to, unsigned int min, unsigned int flags) 379 + { 380 + return (int)syscall(__NR_io_uring_enter, fd, to, min, flags, NULL, 0); 381 + } 382 + 383 + /* Init submission queue and completion queue */ 384 + int mmap_io_uring(struct io_uring_params p, struct io_ring *s) 385 + { 386 + struct io_uring_queue *sring = &s->sq_ring; 387 + struct io_uring_queue *cring = &s->cq_ring; 388 + 389 + sring->ring_sz = p.sq_off.array + p.sq_entries * sizeof(unsigned int); 390 + cring->ring_sz = p.cq_off.cqes + p.cq_entries * sizeof(struct io_uring_cqe); 391 + 392 + if (p.features & IORING_FEAT_SINGLE_MMAP) { 393 + if (cring->ring_sz > sring->ring_sz) 394 + sring->ring_sz = cring->ring_sz; 395 + 396 + cring->ring_sz = sring->ring_sz; 397 + } 398 + 399 + void *sq_ptr = mmap(0, sring->ring_sz, PROT_READ | PROT_WRITE, 400 + MAP_SHARED | MAP_POPULATE, s->ring_fd, 401 + IORING_OFF_SQ_RING); 402 + 403 + if (sq_ptr == MAP_FAILED) { 404 + perror("sub-queue!"); 405 + return 1; 406 + } 407 + 408 + void *cq_ptr = sq_ptr; 409 + 410 + if (!(p.features & IORING_FEAT_SINGLE_MMAP)) { 411 + cq_ptr = mmap(0, cring->ring_sz, PROT_READ | PROT_WRITE, 412 + MAP_SHARED | MAP_POPULATE, s->ring_fd, 413 + IORING_OFF_CQ_RING); 414 + if (cq_ptr == MAP_FAILED) { 415 + perror("cpl-queue!"); 416 + munmap(sq_ptr, sring->ring_sz); 417 + return 1; 418 + } 419 + } 420 + 421 + sring->head = sq_ptr + p.sq_off.head; 422 + sring->tail = sq_ptr + p.sq_off.tail; 423 + sring->ring_mask = sq_ptr + p.sq_off.ring_mask; 424 + sring->ring_entries = sq_ptr + p.sq_off.ring_entries; 425 + sring->flags = sq_ptr + p.sq_off.flags; 426 + sring->array = sq_ptr + p.sq_off.array; 427 + 428 + /* Map a queue as mem map */ 429 + s->sq_ring.queue.sqes = mmap(0, p.sq_entries * sizeof(struct io_uring_sqe), 430 + PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE, 431 + s->ring_fd, IORING_OFF_SQES); 432 + if (s->sq_ring.queue.sqes == MAP_FAILED) { 433 + munmap(sq_ptr, sring->ring_sz); 434 + if (sq_ptr != cq_ptr) { 435 + ksft_print_msg("failed to mmap uring queue!"); 436 + munmap(cq_ptr, cring->ring_sz); 437 + return 1; 438 + } 439 + } 440 + 441 + cring->head = cq_ptr + p.cq_off.head; 442 + cring->tail = cq_ptr + p.cq_off.tail; 443 + cring->ring_mask = cq_ptr + p.cq_off.ring_mask; 444 + cring->ring_entries = cq_ptr + p.cq_off.ring_entries; 445 + cring->queue.cqes = cq_ptr + p.cq_off.cqes; 446 + 447 + return 0; 448 + } 449 + 450 + /* Init io_uring queues */ 451 + int setup_io_uring(struct io_ring *s) 452 + { 453 + struct io_uring_params para; 454 + 455 + memset(&para, 0, sizeof(para)); 456 + s->ring_fd = sys_uring_setup(URING_QUEUE_SZ, &para); 457 + if (s->ring_fd < 0) 458 + return 1; 459 + 460 + return mmap_io_uring(para, s); 461 + } 462 + 463 + /* 464 + * Get data from completion queue. the data buffer saved the file data 465 + * return 0: success; others: error; 466 + */ 467 + int handle_uring_cq(struct io_ring *s) 468 + { 469 + struct file_io *fi = NULL; 470 + struct io_uring_queue *cring = &s->cq_ring; 471 + struct io_uring_cqe *cqe; 472 + unsigned int head; 473 + off_t len = 0; 474 + 475 + head = *cring->head; 476 + 477 + do { 478 + barrier(); 479 + if (head == *cring->tail) 480 + break; 481 + /* Get the entry */ 482 + cqe = &cring->queue.cqes[head & *s->cq_ring.ring_mask]; 483 + fi = (struct file_io *)cqe->user_data; 484 + if (cqe->res < 0) 485 + break; 486 + 487 + int blocks = (int)(fi->file_sz + URING_BLOCK_SZ - 1) / URING_BLOCK_SZ; 488 + 489 + for (int i = 0; i < blocks; i++) 490 + len += fi->iovecs[i].iov_len; 491 + 492 + head++; 493 + } while (1); 494 + 495 + *cring->head = head; 496 + barrier(); 497 + 498 + return (len != fi->file_sz); 499 + } 500 + 501 + /* 502 + * Submit squeue. specify via IORING_OP_READV. 503 + * the buffer need to be set metadata according to LAM mode 504 + */ 505 + int handle_uring_sq(struct io_ring *ring, struct file_io *fi, unsigned long lam) 506 + { 507 + int file_fd = fi->file_fd; 508 + struct io_uring_queue *sring = &ring->sq_ring; 509 + unsigned int index = 0, cur_block = 0, tail = 0, next_tail = 0; 510 + struct io_uring_sqe *sqe; 511 + 512 + off_t remain = fi->file_sz; 513 + int blocks = (int)(remain + URING_BLOCK_SZ - 1) / URING_BLOCK_SZ; 514 + 515 + while (remain) { 516 + off_t bytes = remain; 517 + void *buf; 518 + 519 + if (bytes > URING_BLOCK_SZ) 520 + bytes = URING_BLOCK_SZ; 521 + 522 + fi->iovecs[cur_block].iov_len = bytes; 523 + 524 + if (posix_memalign(&buf, URING_BLOCK_SZ, URING_BLOCK_SZ)) 525 + return 1; 526 + 527 + fi->iovecs[cur_block].iov_base = (void *)set_metadata((uint64_t)buf, lam); 528 + remain -= bytes; 529 + cur_block++; 530 + } 531 + 532 + next_tail = *sring->tail; 533 + tail = next_tail; 534 + next_tail++; 535 + 536 + barrier(); 537 + 538 + index = tail & *ring->sq_ring.ring_mask; 539 + 540 + sqe = &ring->sq_ring.queue.sqes[index]; 541 + sqe->fd = file_fd; 542 + sqe->flags = 0; 543 + sqe->opcode = IORING_OP_READV; 544 + sqe->addr = (unsigned long)fi->iovecs; 545 + sqe->len = blocks; 546 + sqe->off = 0; 547 + sqe->user_data = (uint64_t)fi; 548 + 549 + sring->array[index] = index; 550 + tail = next_tail; 551 + 552 + if (*sring->tail != tail) { 553 + *sring->tail = tail; 554 + barrier(); 555 + } 556 + 557 + if (sys_uring_enter(ring->ring_fd, 1, 1, IORING_ENTER_GETEVENTS) < 0) 558 + return 1; 559 + 560 + return 0; 561 + } 562 + 563 + /* 564 + * Test LAM in async I/O and io_uring, read current binery through io_uring 565 + * Set metadata in pointers to iovecs buffer. 566 + */ 567 + int do_uring(unsigned long lam) 568 + { 569 + struct io_ring *ring; 570 + struct file_io *fi; 571 + struct stat st; 572 + int ret = 1; 573 + char path[PATH_MAX] = {0}; 574 + 575 + /* get current process path */ 576 + if (readlink("/proc/self/exe", path, PATH_MAX) <= 0) 577 + return 1; 578 + 579 + int file_fd = open(path, O_RDONLY); 580 + 581 + if (file_fd < 0) 582 + return 1; 583 + 584 + if (fstat(file_fd, &st) < 0) 585 + return 1; 586 + 587 + off_t file_sz = st.st_size; 588 + 589 + int blocks = (int)(file_sz + URING_BLOCK_SZ - 1) / URING_BLOCK_SZ; 590 + 591 + fi = malloc(sizeof(*fi) + sizeof(struct iovec) * blocks); 592 + if (!fi) 593 + return 1; 594 + 595 + fi->file_sz = file_sz; 596 + fi->file_fd = file_fd; 597 + 598 + ring = malloc(sizeof(*ring)); 599 + if (!ring) 600 + return 1; 601 + 602 + memset(ring, 0, sizeof(struct io_ring)); 603 + 604 + if (setup_io_uring(ring)) 605 + goto out; 606 + 607 + if (handle_uring_sq(ring, fi, lam)) 608 + goto out; 609 + 610 + ret = handle_uring_cq(ring); 611 + 612 + out: 613 + free(ring); 614 + 615 + for (int i = 0; i < blocks; i++) { 616 + if (fi->iovecs[i].iov_base) { 617 + uint64_t addr = ((uint64_t)fi->iovecs[i].iov_base); 618 + 619 + switch (lam) { 620 + case LAM_U57_BITS: /* Clear bits 62:57 */ 621 + addr = (addr & ~(LAM_U57_MASK)); 622 + break; 623 + } 624 + free((void *)addr); 625 + fi->iovecs[i].iov_base = NULL; 626 + } 627 + } 628 + 629 + free(fi); 630 + 631 + return ret; 632 + } 633 + 634 + int handle_uring(struct testcases *test) 635 + { 636 + int ret = 0; 637 + 638 + if (test->later == 0 && test->lam != 0) 639 + if (set_lam(test->lam) != 0) 640 + return 1; 641 + 642 + if (sigsetjmp(segv_env, 1) == 0) { 643 + signal(SIGSEGV, segv_handler); 644 + ret = do_uring(test->lam); 645 + } else { 646 + ret = 2; 647 + } 648 + 649 + return ret; 650 + } 651 + 652 + static int fork_test(struct testcases *test) 653 + { 654 + int ret, child_ret; 655 + pid_t pid; 656 + 657 + pid = fork(); 658 + if (pid < 0) { 659 + perror("Fork failed."); 660 + ret = 1; 661 + } else if (pid == 0) { 662 + ret = test->test_func(test); 663 + exit(ret); 664 + } else { 665 + wait(&child_ret); 666 + ret = WEXITSTATUS(child_ret); 667 + } 668 + 669 + return ret; 670 + } 671 + 672 + static int handle_execve(struct testcases *test) 673 + { 674 + int ret, child_ret; 675 + int lam = test->lam; 676 + pid_t pid; 677 + 678 + pid = fork(); 679 + if (pid < 0) { 680 + perror("Fork failed."); 681 + ret = 1; 682 + } else if (pid == 0) { 683 + char path[PATH_MAX]; 684 + 685 + /* Set LAM mode in parent process */ 686 + if (set_lam(lam) != 0) 687 + return 1; 688 + 689 + /* Get current binary's path and the binary was run by execve */ 690 + if (readlink("/proc/self/exe", path, PATH_MAX) <= 0) 691 + exit(-1); 692 + 693 + /* run binary to get LAM mode and return to parent process */ 694 + if (execlp(path, path, "-t 0x0", NULL) < 0) { 695 + perror("error on exec"); 696 + exit(-1); 697 + } 698 + } else { 699 + wait(&child_ret); 700 + ret = WEXITSTATUS(child_ret); 701 + if (ret != LAM_NONE) 702 + return 1; 703 + } 704 + 705 + return 0; 706 + } 707 + 708 + static int handle_inheritance(struct testcases *test) 709 + { 710 + int ret, child_ret; 711 + int lam = test->lam; 712 + pid_t pid; 713 + 714 + /* Set LAM mode in parent process */ 715 + if (set_lam(lam) != 0) 716 + return 1; 717 + 718 + pid = fork(); 719 + if (pid < 0) { 720 + perror("Fork failed."); 721 + return 1; 722 + } else if (pid == 0) { 723 + /* Set LAM mode in parent process */ 724 + int child_lam = get_lam(); 725 + 726 + exit(child_lam); 727 + } else { 728 + wait(&child_ret); 729 + ret = WEXITSTATUS(child_ret); 730 + 731 + if (lam != ret) 732 + return 1; 733 + } 734 + 735 + return 0; 736 + } 737 + 738 + static int thread_fn_get_lam(void *arg) 739 + { 740 + return get_lam(); 741 + } 742 + 743 + static int thread_fn_set_lam(void *arg) 744 + { 745 + struct testcases *test = arg; 746 + 747 + return set_lam(test->lam); 748 + } 749 + 750 + static int handle_thread(struct testcases *test) 751 + { 752 + char stack[STACK_SIZE]; 753 + int ret, child_ret; 754 + int lam = 0; 755 + pid_t pid; 756 + 757 + /* Set LAM mode in parent process */ 758 + if (!test->later) { 759 + lam = test->lam; 760 + if (set_lam(lam) != 0) 761 + return 1; 762 + } 763 + 764 + pid = clone(thread_fn_get_lam, stack + STACK_SIZE, 765 + SIGCHLD | CLONE_FILES | CLONE_FS | CLONE_VM, NULL); 766 + if (pid < 0) { 767 + perror("Clone failed."); 768 + return 1; 769 + } 770 + 771 + waitpid(pid, &child_ret, 0); 772 + ret = WEXITSTATUS(child_ret); 773 + 774 + if (lam != ret) 775 + return 1; 776 + 777 + if (test->later) { 778 + if (set_lam(test->lam) != 0) 779 + return 1; 780 + } 781 + 782 + return 0; 783 + } 784 + 785 + static int handle_thread_enable(struct testcases *test) 786 + { 787 + char stack[STACK_SIZE]; 788 + int ret, child_ret; 789 + int lam = test->lam; 790 + pid_t pid; 791 + 792 + pid = clone(thread_fn_set_lam, stack + STACK_SIZE, 793 + SIGCHLD | CLONE_FILES | CLONE_FS | CLONE_VM, test); 794 + if (pid < 0) { 795 + perror("Clone failed."); 796 + return 1; 797 + } 798 + 799 + waitpid(pid, &child_ret, 0); 800 + ret = WEXITSTATUS(child_ret); 801 + 802 + if (lam != ret) 803 + return 1; 804 + 805 + return 0; 806 + } 807 + static void run_test(struct testcases *test, int count) 808 + { 809 + int i, ret = 0; 810 + 811 + for (i = 0; i < count; i++) { 812 + struct testcases *t = test + i; 813 + 814 + /* fork a process to run test case */ 815 + tests_cnt++; 816 + ret = fork_test(t); 817 + 818 + /* return 3 is not support LA57, the case should be skipped */ 819 + if (ret == 3) { 820 + ksft_test_result_skip(t->msg); 821 + continue; 822 + } 823 + 824 + if (ret != 0) 825 + ret = (t->expected == ret); 826 + else 827 + ret = !(t->expected); 828 + 829 + ksft_test_result(ret, t->msg); 830 + } 831 + } 832 + 833 + static struct testcases uring_cases[] = { 834 + { 835 + .later = 0, 836 + .lam = LAM_U57_BITS, 837 + .test_func = handle_uring, 838 + .msg = "URING: LAM_U57. Dereferencing pointer with metadata\n", 839 + }, 840 + { 841 + .later = 1, 842 + .expected = 1, 843 + .lam = LAM_U57_BITS, 844 + .test_func = handle_uring, 845 + .msg = "URING:[Negative] Disable LAM. Dereferencing pointer with metadata.\n", 846 + }, 847 + }; 848 + 849 + static struct testcases malloc_cases[] = { 850 + { 851 + .later = 0, 852 + .lam = LAM_U57_BITS, 853 + .test_func = handle_malloc, 854 + .msg = "MALLOC: LAM_U57. Dereferencing pointer with metadata\n", 855 + }, 856 + { 857 + .later = 1, 858 + .expected = 2, 859 + .lam = LAM_U57_BITS, 860 + .test_func = handle_malloc, 861 + .msg = "MALLOC:[Negative] Disable LAM. Dereferencing pointer with metadata.\n", 862 + }, 863 + }; 864 + 865 + static struct testcases bits_cases[] = { 866 + { 867 + .test_func = handle_max_bits, 868 + .msg = "BITS: Check default tag bits\n", 869 + }, 870 + }; 871 + 872 + static struct testcases syscall_cases[] = { 873 + { 874 + .later = 0, 875 + .lam = LAM_U57_BITS, 876 + .test_func = handle_syscall, 877 + .msg = "SYSCALL: LAM_U57. syscall with metadata\n", 878 + }, 879 + { 880 + .later = 1, 881 + .expected = 1, 882 + .lam = LAM_U57_BITS, 883 + .test_func = handle_syscall, 884 + .msg = "SYSCALL:[Negative] Disable LAM. Dereferencing pointer with metadata.\n", 885 + }, 886 + }; 887 + 888 + static struct testcases mmap_cases[] = { 889 + { 890 + .later = 1, 891 + .expected = 0, 892 + .lam = LAM_U57_BITS, 893 + .addr = HIGH_ADDR, 894 + .test_func = handle_mmap, 895 + .msg = "MMAP: First mmap high address, then set LAM_U57.\n", 896 + }, 897 + { 898 + .later = 0, 899 + .expected = 0, 900 + .lam = LAM_U57_BITS, 901 + .addr = HIGH_ADDR, 902 + .test_func = handle_mmap, 903 + .msg = "MMAP: First LAM_U57, then High address.\n", 904 + }, 905 + { 906 + .later = 0, 907 + .expected = 0, 908 + .lam = LAM_U57_BITS, 909 + .addr = LOW_ADDR, 910 + .test_func = handle_mmap, 911 + .msg = "MMAP: First LAM_U57, then Low address.\n", 912 + }, 913 + }; 914 + 915 + static struct testcases inheritance_cases[] = { 916 + { 917 + .expected = 0, 918 + .lam = LAM_U57_BITS, 919 + .test_func = handle_inheritance, 920 + .msg = "FORK: LAM_U57, child process should get LAM mode same as parent\n", 921 + }, 922 + { 923 + .expected = 0, 924 + .lam = LAM_U57_BITS, 925 + .test_func = handle_thread, 926 + .msg = "THREAD: LAM_U57, child thread should get LAM mode same as parent\n", 927 + }, 928 + { 929 + .expected = 1, 930 + .lam = LAM_U57_BITS, 931 + .test_func = handle_thread_enable, 932 + .msg = "THREAD: [NEGATIVE] Enable LAM in child.\n", 933 + }, 934 + { 935 + .expected = 1, 936 + .later = 1, 937 + .lam = LAM_U57_BITS, 938 + .test_func = handle_thread, 939 + .msg = "THREAD: [NEGATIVE] Enable LAM in parent after thread created.\n", 940 + }, 941 + { 942 + .expected = 0, 943 + .lam = LAM_U57_BITS, 944 + .test_func = handle_execve, 945 + .msg = "EXECVE: LAM_U57, child process should get disabled LAM mode\n", 946 + }, 947 + }; 948 + 949 + static void cmd_help(void) 950 + { 951 + printf("usage: lam [-h] [-t test list]\n"); 952 + printf("\t-t test list: run tests specified in the test list, default:0x%x\n", TEST_MASK); 953 + printf("\t\t0x1:malloc; 0x2:max_bits; 0x4:mmap; 0x8:syscall; 0x10:io_uring; 0x20:inherit;\n"); 954 + printf("\t-h: help\n"); 955 + } 956 + 957 + /* Check for file existence */ 958 + uint8_t file_Exists(const char *fileName) 959 + { 960 + struct stat buffer; 961 + 962 + uint8_t ret = (stat(fileName, &buffer) == 0); 963 + 964 + return ret; 965 + } 966 + 967 + /* Sysfs idxd files */ 968 + const char *dsa_configs[] = { 969 + "echo 1 > /sys/bus/dsa/devices/dsa0/wq0.1/group_id", 970 + "echo shared > /sys/bus/dsa/devices/dsa0/wq0.1/mode", 971 + "echo 10 > /sys/bus/dsa/devices/dsa0/wq0.1/priority", 972 + "echo 16 > /sys/bus/dsa/devices/dsa0/wq0.1/size", 973 + "echo 15 > /sys/bus/dsa/devices/dsa0/wq0.1/threshold", 974 + "echo user > /sys/bus/dsa/devices/dsa0/wq0.1/type", 975 + "echo MyApp1 > /sys/bus/dsa/devices/dsa0/wq0.1/name", 976 + "echo 1 > /sys/bus/dsa/devices/dsa0/engine0.1/group_id", 977 + "echo dsa0 > /sys/bus/dsa/drivers/idxd/bind", 978 + /* bind files and devices, generated a device file in /dev */ 979 + "echo wq0.1 > /sys/bus/dsa/drivers/user/bind", 980 + }; 981 + 982 + /* DSA device file */ 983 + const char *dsaDeviceFile = "/dev/dsa/wq0.1"; 984 + /* file for io*/ 985 + const char *dsaPasidEnable = "/sys/bus/dsa/devices/dsa0/pasid_enabled"; 986 + 987 + /* 988 + * DSA depends on kernel cmdline "intel_iommu=on,sm_on" 989 + * return pasid_enabled (0: disable 1:enable) 990 + */ 991 + int Check_DSA_Kernel_Setting(void) 992 + { 993 + char command[256] = ""; 994 + char buf[256] = ""; 995 + char *ptr; 996 + int rv = -1; 997 + 998 + snprintf(command, sizeof(command) - 1, "cat %s", dsaPasidEnable); 999 + 1000 + FILE *cmd = popen(command, "r"); 1001 + 1002 + if (cmd) { 1003 + while (fgets(buf, sizeof(buf) - 1, cmd) != NULL); 1004 + 1005 + pclose(cmd); 1006 + rv = strtol(buf, &ptr, 16); 1007 + } 1008 + 1009 + return rv; 1010 + } 1011 + 1012 + /* 1013 + * Config DSA's sysfs files as shared DSA's WQ. 1014 + * Generated a device file /dev/dsa/wq0.1 1015 + * Return: 0 OK; 1 Failed; 3 Skip(SVA disabled). 1016 + */ 1017 + int Dsa_Init_Sysfs(void) 1018 + { 1019 + uint len = ARRAY_SIZE(dsa_configs); 1020 + const char **p = dsa_configs; 1021 + 1022 + if (file_Exists(dsaDeviceFile) == 1) 1023 + return 0; 1024 + 1025 + /* check the idxd driver */ 1026 + if (file_Exists(dsaPasidEnable) != 1) { 1027 + printf("Please make sure idxd driver was loaded\n"); 1028 + return 3; 1029 + } 1030 + 1031 + /* Check SVA feature */ 1032 + if (Check_DSA_Kernel_Setting() != 1) { 1033 + printf("Please enable SVA.(Add intel_iommu=on,sm_on in kernel cmdline)\n"); 1034 + return 3; 1035 + } 1036 + 1037 + /* Check the idxd device file on /dev/dsa/ */ 1038 + for (int i = 0; i < len; i++) { 1039 + if (system(p[i])) 1040 + return 1; 1041 + } 1042 + 1043 + /* After config, /dev/dsa/wq0.1 should be generated */ 1044 + return (file_Exists(dsaDeviceFile) != 1); 1045 + } 1046 + 1047 + /* 1048 + * Open DSA device file, triger API: iommu_sva_alloc_pasid 1049 + */ 1050 + void *allocate_dsa_pasid(void) 1051 + { 1052 + int fd; 1053 + void *wq; 1054 + 1055 + fd = open(dsaDeviceFile, O_RDWR); 1056 + if (fd < 0) { 1057 + perror("open"); 1058 + return MAP_FAILED; 1059 + } 1060 + 1061 + wq = mmap(NULL, 0x1000, PROT_WRITE, 1062 + MAP_SHARED | MAP_POPULATE, fd, 0); 1063 + if (wq == MAP_FAILED) 1064 + perror("mmap"); 1065 + 1066 + return wq; 1067 + } 1068 + 1069 + int set_force_svm(void) 1070 + { 1071 + int ret = 0; 1072 + 1073 + ret = syscall(SYS_arch_prctl, ARCH_FORCE_TAGGED_SVA); 1074 + 1075 + return ret; 1076 + } 1077 + 1078 + int handle_pasid(struct testcases *test) 1079 + { 1080 + uint tmp = test->cmd; 1081 + uint runed = 0x0; 1082 + int ret = 0; 1083 + void *wq = NULL; 1084 + 1085 + ret = Dsa_Init_Sysfs(); 1086 + if (ret != 0) 1087 + return ret; 1088 + 1089 + for (int i = 0; i < 3; i++) { 1090 + int err = 0; 1091 + 1092 + if (tmp & 0x1) { 1093 + /* run set lam mode*/ 1094 + if ((runed & 0x1) == 0) { 1095 + err = set_lam(LAM_U57_BITS); 1096 + runed = runed | 0x1; 1097 + } else 1098 + err = 1; 1099 + } else if (tmp & 0x4) { 1100 + /* run force svm */ 1101 + if ((runed & 0x4) == 0) { 1102 + err = set_force_svm(); 1103 + runed = runed | 0x4; 1104 + } else 1105 + err = 1; 1106 + } else if (tmp & 0x2) { 1107 + /* run allocate pasid */ 1108 + if ((runed & 0x2) == 0) { 1109 + runed = runed | 0x2; 1110 + wq = allocate_dsa_pasid(); 1111 + if (wq == MAP_FAILED) 1112 + err = 1; 1113 + } else 1114 + err = 1; 1115 + } 1116 + 1117 + ret = ret + err; 1118 + if (ret > 0) 1119 + break; 1120 + 1121 + tmp = tmp >> 4; 1122 + } 1123 + 1124 + if (wq != MAP_FAILED && wq != NULL) 1125 + if (munmap(wq, 0x1000)) 1126 + printf("munmap failed %d\n", errno); 1127 + 1128 + if (runed != 0x7) 1129 + ret = 1; 1130 + 1131 + return (ret != 0); 1132 + } 1133 + 1134 + /* 1135 + * Pasid test depends on idxd and SVA, kernel should enable iommu and sm. 1136 + * command line(intel_iommu=on,sm_on) 1137 + */ 1138 + static struct testcases pasid_cases[] = { 1139 + { 1140 + .expected = 1, 1141 + .cmd = PAS_CMD(LAM_CMD_BIT, PAS_CMD_BIT, SVA_CMD_BIT), 1142 + .test_func = handle_pasid, 1143 + .msg = "PASID: [Negative] Execute LAM, PASID, SVA in sequence\n", 1144 + }, 1145 + { 1146 + .expected = 0, 1147 + .cmd = PAS_CMD(LAM_CMD_BIT, SVA_CMD_BIT, PAS_CMD_BIT), 1148 + .test_func = handle_pasid, 1149 + .msg = "PASID: Execute LAM, SVA, PASID in sequence\n", 1150 + }, 1151 + { 1152 + .expected = 1, 1153 + .cmd = PAS_CMD(PAS_CMD_BIT, LAM_CMD_BIT, SVA_CMD_BIT), 1154 + .test_func = handle_pasid, 1155 + .msg = "PASID: [Negative] Execute PASID, LAM, SVA in sequence\n", 1156 + }, 1157 + { 1158 + .expected = 0, 1159 + .cmd = PAS_CMD(PAS_CMD_BIT, SVA_CMD_BIT, LAM_CMD_BIT), 1160 + .test_func = handle_pasid, 1161 + .msg = "PASID: Execute PASID, SVA, LAM in sequence\n", 1162 + }, 1163 + { 1164 + .expected = 0, 1165 + .cmd = PAS_CMD(SVA_CMD_BIT, LAM_CMD_BIT, PAS_CMD_BIT), 1166 + .test_func = handle_pasid, 1167 + .msg = "PASID: Execute SVA, LAM, PASID in sequence\n", 1168 + }, 1169 + { 1170 + .expected = 0, 1171 + .cmd = PAS_CMD(SVA_CMD_BIT, PAS_CMD_BIT, LAM_CMD_BIT), 1172 + .test_func = handle_pasid, 1173 + .msg = "PASID: Execute SVA, PASID, LAM in sequence\n", 1174 + }, 1175 + }; 1176 + 1177 + int main(int argc, char **argv) 1178 + { 1179 + int c = 0; 1180 + unsigned int tests = TEST_MASK; 1181 + 1182 + tests_cnt = 0; 1183 + 1184 + if (!cpu_has_lam()) { 1185 + ksft_print_msg("Unsupported LAM feature!\n"); 1186 + return -1; 1187 + } 1188 + 1189 + while ((c = getopt(argc, argv, "ht:")) != -1) { 1190 + switch (c) { 1191 + case 't': 1192 + tests = strtoul(optarg, NULL, 16); 1193 + if (tests && !(tests & TEST_MASK)) { 1194 + ksft_print_msg("Invalid argument!\n"); 1195 + return -1; 1196 + } 1197 + break; 1198 + case 'h': 1199 + cmd_help(); 1200 + return 0; 1201 + default: 1202 + ksft_print_msg("Invalid argument\n"); 1203 + return -1; 1204 + } 1205 + } 1206 + 1207 + /* 1208 + * When tests is 0, it is not a real test case; 1209 + * the option used by test case(execve) to check the lam mode in 1210 + * process generated by execve, the process read back lam mode and 1211 + * check with lam mode in parent process. 1212 + */ 1213 + if (!tests) 1214 + return (get_lam()); 1215 + 1216 + /* Run test cases */ 1217 + if (tests & FUNC_MALLOC) 1218 + run_test(malloc_cases, ARRAY_SIZE(malloc_cases)); 1219 + 1220 + if (tests & FUNC_BITS) 1221 + run_test(bits_cases, ARRAY_SIZE(bits_cases)); 1222 + 1223 + if (tests & FUNC_MMAP) 1224 + run_test(mmap_cases, ARRAY_SIZE(mmap_cases)); 1225 + 1226 + if (tests & FUNC_SYSCALL) 1227 + run_test(syscall_cases, ARRAY_SIZE(syscall_cases)); 1228 + 1229 + if (tests & FUNC_URING) 1230 + run_test(uring_cases, ARRAY_SIZE(uring_cases)); 1231 + 1232 + if (tests & FUNC_INHERITE) 1233 + run_test(inheritance_cases, ARRAY_SIZE(inheritance_cases)); 1234 + 1235 + if (tests & FUNC_PASID) 1236 + run_test(pasid_cases, ARRAY_SIZE(pasid_cases)); 1237 + 1238 + ksft_set_plan(tests_cnt); 1239 + 1240 + return ksft_exit_pass(); 1241 + }

Configure Feed

Configure Feed