Merge tag 'arm64-upstream' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux

+21

Documentation/arch/arm64/booting.rst

··· 388 388 389 389 - SMCR_EL2.EZT0 (bit 30) must be initialised to 0b1. 390 390 391 + For CPUs with the Branch Record Buffer Extension (FEAT_BRBE): 392 + 393 + - If EL3 is present: 394 + 395 + - MDCR_EL3.SBRBE (bits 33:32) must be initialised to 0b01 or 0b11. 396 + 397 + - If the kernel is entered at EL1 and EL2 is present: 398 + 399 + - BRBCR_EL2.CC (bit 3) must be initialised to 0b1. 400 + - BRBCR_EL2.MPRED (bit 4) must be initialised to 0b1. 401 + 402 + - HDFGRTR_EL2.nBRBDATA (bit 61) must be initialised to 0b1. 403 + - HDFGRTR_EL2.nBRBCTL (bit 60) must be initialised to 0b1. 404 + - HDFGRTR_EL2.nBRBIDR (bit 59) must be initialised to 0b1. 405 + 406 + - HDFGWTR_EL2.nBRBDATA (bit 61) must be initialised to 0b1. 407 + - HDFGWTR_EL2.nBRBCTL (bit 60) must be initialised to 0b1. 408 + 409 + - HFGITR_EL2.nBRBIALL (bit 56) must be initialised to 0b1. 410 + - HFGITR_EL2.nBRBINJ (bit 55) must be initialised to 0b1. 411 + 391 412 For CPUs with the Performance Monitors Extension (FEAT_PMUv3p9): 392 413 393 414 - If EL3 is present:

+6

Documentation/arch/arm64/elf_hwcaps.rst

··· 435 435 HWCAP2_POE 436 436 Functionality implied by ID_AA64MMFR3_EL1.S1POE == 0b0001. 437 437 438 + HWCAP3_MTE_FAR 439 + Functionality implied by ID_AA64PFR2_EL1.MTEFAR == 0b0001. 440 + 441 + HWCAP3_MTE_STORE_ONLY 442 + Functionality implied by ID_AA64PFR2_EL1.MTESTOREONLY == 0b0001. 443 + 438 444 4. Unused AT_HWCAP bits 439 445 ----------------------- 440 446

+6 -5

Documentation/arch/arm64/tagged-pointers.rst

··· 60 60 on the tag information for user virtual addresses being maintained 61 61 in these fields unless the flag was set. 62 62 63 - Due to architecture limitations, bits 63:60 of the fault address 64 - are not preserved in response to synchronous tag check faults 65 - (SEGV_MTESERR) even if SA_EXPOSE_TAGBITS was set. Applications should 66 - treat the values of these bits as undefined in order to accommodate 67 - future architecture revisions which may preserve the bits. 63 + If FEAT_MTE_TAGGED_FAR (Armv8.9) is supported, bits 63:60 of the fault address 64 + are preserved in response to synchronous tag check faults (SEGV_MTESERR) 65 + otherwise not preserved even if SA_EXPOSE_TAGBITS was set. 66 + Applications should interpret the values of these bits based on 67 + the support for the HWCAP3_MTE_FAR. If the support is not present, 68 + the values of these bits should be considered as undefined otherwise valid. 68 69 69 70 For signals raised in response to watchpoint debug exceptions, the 70 71 tag information will be preserved regardless of the SA_EXPOSE_TAGBITS

+4

arch/arm64/Kconfig

··· 232 232 select HAVE_HW_BREAKPOINT if PERF_EVENTS 233 233 select HAVE_IOREMAP_PROT 234 234 select HAVE_IRQ_TIME_ACCOUNTING 235 + select HAVE_LIVEPATCH 235 236 select HAVE_MOD_ARCH_SPECIFIC 236 237 select HAVE_NMI 237 238 select HAVE_PERF_EVENTS ··· 241 240 select HAVE_PERF_USER_STACK_DUMP 242 241 select HAVE_PREEMPT_DYNAMIC_KEY 243 242 select HAVE_REGS_AND_STACK_ACCESS_API 243 + select HAVE_RELIABLE_STACKTRACE 244 244 select HAVE_POSIX_CPU_TIMERS_TASK_WORK 245 245 select HAVE_FUNCTION_ARG_ACCESS_API 246 246 select MMU_GATHER_RCU_TABLE_FREE ··· 280 278 select HAVE_SOFTIRQ_ON_OWN_STACK 281 279 select USER_STACKTRACE_SUPPORT 282 280 select VDSO_GETRANDOM 281 + select VMAP_STACK 283 282 help 284 283 ARM 64-bit (AArch64) Linux support. 285 284 ··· 2501 2498 2502 2499 source "arch/arm64/kvm/Kconfig" 2503 2500 2501 + source "kernel/livepatch/Kconfig"

+2 -2

arch/arm64/include/asm/assembler.h

··· 58 58 .macro disable_step_tsk, flgs, tmp 59 59 tbz \flgs, #TIF_SINGLESTEP, 9990f 60 60 mrs \tmp, mdscr_el1 61 - bic \tmp, \tmp, #DBG_MDSCR_SS 61 + bic \tmp, \tmp, #MDSCR_EL1_SS 62 62 msr mdscr_el1, \tmp 63 63 isb // Take effect before a subsequent clear of DAIF.D 64 64 9990: ··· 68 68 .macro enable_step_tsk, flgs, tmp 69 69 tbz \flgs, #TIF_SINGLESTEP, 9990f 70 70 mrs \tmp, mdscr_el1 71 - orr \tmp, \tmp, #DBG_MDSCR_SS 71 + orr \tmp, \tmp, #MDSCR_EL1_SS 72 72 msr mdscr_el1, \tmp 73 73 9990: 74 74 .endm

+28

arch/arm64/include/asm/cpufeature.h

··· 275 275 #define ARM64_CPUCAP_OPTIONAL_FOR_LATE_CPU ((u16)BIT(5)) 276 276 /* Panic when a conflict is detected */ 277 277 #define ARM64_CPUCAP_PANIC_ON_CONFLICT ((u16)BIT(6)) 278 + /* 279 + * When paired with SCOPE_LOCAL_CPU, all early CPUs must satisfy the 280 + * condition. This is different from SCOPE_SYSTEM where the check is performed 281 + * only once at the end of the SMP boot on the sanitised ID registers. 282 + * SCOPE_SYSTEM is not suitable for cases where the capability depends on 283 + * properties local to a CPU like MIDR_EL1. 284 + */ 285 + #define ARM64_CPUCAP_MATCH_ALL_EARLY_CPUS ((u16)BIT(7)) 278 286 279 287 /* 280 288 * CPU errata workarounds that need to be enabled at boot time if one or ··· 312 304 (ARM64_CPUCAP_SCOPE_LOCAL_CPU | \ 313 305 ARM64_CPUCAP_OPTIONAL_FOR_LATE_CPU | \ 314 306 ARM64_CPUCAP_PERMITTED_FOR_LATE_CPU) 307 + /* 308 + * CPU feature detected at boot time and present on all early CPUs. Late CPUs 309 + * are permitted to have the feature even if it hasn't been enabled, although 310 + * the feature will not be used by Linux in this case. If all early CPUs have 311 + * the feature, then every late CPU must have it. 312 + */ 313 + #define ARM64_CPUCAP_EARLY_LOCAL_CPU_FEATURE \ 314 + (ARM64_CPUCAP_SCOPE_LOCAL_CPU | \ 315 + ARM64_CPUCAP_PERMITTED_FOR_LATE_CPU | \ 316 + ARM64_CPUCAP_MATCH_ALL_EARLY_CPUS) 315 317 316 318 /* 317 319 * CPU feature detected at boot time, on one or more CPUs. A late CPU ··· 407 389 static inline int cpucap_default_scope(const struct arm64_cpu_capabilities *cap) 408 390 { 409 391 return cap->type & ARM64_CPUCAP_SCOPE_MASK; 392 + } 393 + 394 + static inline bool cpucap_match_all_early_cpus(const struct arm64_cpu_capabilities *cap) 395 + { 396 + return cap->type & ARM64_CPUCAP_MATCH_ALL_EARLY_CPUS; 410 397 } 411 398 412 399 /* ··· 869 846 static inline bool system_supports_pmuv3(void) 870 847 { 871 848 return cpus_have_final_cap(ARM64_HAS_PMUV3); 849 + } 850 + 851 + static inline bool system_supports_bbml2_noabort(void) 852 + { 853 + return alternative_has_cap_unlikely(ARM64_HAS_BBML2_NOABORT); 872 854 } 873 855 874 856 int do_emulate_mrs(struct pt_regs *regs, u32 sys_reg, u32 rt);

+4 -36

arch/arm64/include/asm/debug-monitors.h

··· 13 13 #include <asm/ptrace.h> 14 14 15 15 /* Low-level stepping controls. */ 16 - #define DBG_MDSCR_SS (1 << 0) 17 16 #define DBG_SPSR_SS (1 << 21) 18 - 19 - /* MDSCR_EL1 enabling bits */ 20 - #define DBG_MDSCR_KDE (1 << 13) 21 - #define DBG_MDSCR_MDE (1 << 15) 22 - #define DBG_MDSCR_MASK ~(DBG_MDSCR_KDE | DBG_MDSCR_MDE) 23 17 24 18 #define DBG_ESR_EVT(x) (((x) >> 27) & 0x7) 25 19 ··· 56 62 #define DBG_HOOK_HANDLED 0 57 63 #define DBG_HOOK_ERROR 1 58 64 59 - struct step_hook { 60 - struct list_head node; 61 - int (*fn)(struct pt_regs *regs, unsigned long esr); 62 - }; 63 - 64 - void register_user_step_hook(struct step_hook *hook); 65 - void unregister_user_step_hook(struct step_hook *hook); 66 - 67 - void register_kernel_step_hook(struct step_hook *hook); 68 - void unregister_kernel_step_hook(struct step_hook *hook); 69 - 70 - struct break_hook { 71 - struct list_head node; 72 - int (*fn)(struct pt_regs *regs, unsigned long esr); 73 - u16 imm; 74 - u16 mask; /* These bits are ignored when comparing with imm */ 75 - }; 76 - 77 - void register_user_break_hook(struct break_hook *hook); 78 - void unregister_user_break_hook(struct break_hook *hook); 79 - 80 - void register_kernel_break_hook(struct break_hook *hook); 81 - void unregister_kernel_break_hook(struct break_hook *hook); 82 - 83 65 u8 debug_monitors_arch(void); 84 66 85 67 enum dbg_active_el { ··· 78 108 void kernel_fastforward_single_step(struct pt_regs *regs); 79 109 80 110 #ifdef CONFIG_HAVE_HW_BREAKPOINT 81 - int reinstall_suspended_bps(struct pt_regs *regs); 111 + bool try_step_suspended_breakpoints(struct pt_regs *regs); 82 112 #else 83 - static inline int reinstall_suspended_bps(struct pt_regs *regs) 113 + static inline bool try_step_suspended_breakpoints(struct pt_regs *regs) 84 114 { 85 - return -ENODEV; 115 + return false; 86 116 } 87 117 #endif 88 118 89 - int aarch32_break_handler(struct pt_regs *regs); 90 - 91 - void debug_traps_init(void); 119 + bool try_handle_aarch32_break(struct pt_regs *regs); 92 120 93 121 #endif /* __ASSEMBLY */ 94 122 #endif /* __ASM_DEBUG_MONITORS_H */

+68 -3

arch/arm64/include/asm/el2_setup.h

··· 189 189 .Lskip_set_cptr_\@: 190 190 .endm 191 191 192 + /* 193 + * Configure BRBE to permit recording cycle counts and branch mispredicts. 194 + * 195 + * At any EL, to record cycle counts BRBE requires that both BRBCR_EL2.CC=1 and 196 + * BRBCR_EL1.CC=1. 197 + * 198 + * At any EL, to record branch mispredicts BRBE requires that both 199 + * BRBCR_EL2.MPRED=1 and BRBCR_EL1.MPRED=1. 200 + * 201 + * Set {CC,MPRED} in BRBCR_EL2 in case nVHE mode is used and we are 202 + * executing in EL1. 203 + */ 204 + .macro __init_el2_brbe 205 + mrs x1, id_aa64dfr0_el1 206 + ubfx x1, x1, #ID_AA64DFR0_EL1_BRBE_SHIFT, #4 207 + cbz x1, .Lskip_brbe_\@ 208 + 209 + mov_q x0, BRBCR_ELx_CC | BRBCR_ELx_MPRED 210 + msr_s SYS_BRBCR_EL2, x0 211 + .Lskip_brbe_\@: 212 + .endm 213 + 192 214 /* Disable any fine grained traps */ 193 215 .macro __init_el2_fgt 194 216 mrs x1, id_aa64mmfr0_el1 ··· 218 196 cbz x1, .Lskip_fgt_\@ 219 197 220 198 mov x0, xzr 199 + mov x2, xzr 221 200 mrs x1, id_aa64dfr0_el1 222 201 ubfx x1, x1, #ID_AA64DFR0_EL1_PMSVer_SHIFT, #4 223 202 cmp x1, #3 224 203 b.lt .Lskip_spe_fgt_\@ 225 204 /* Disable PMSNEVFR_EL1 read and write traps */ 226 - orr x0, x0, #(1 << 62) 205 + orr x0, x0, #HDFGRTR_EL2_nPMSNEVFR_EL1_MASK 206 + orr x2, x2, #HDFGWTR_EL2_nPMSNEVFR_EL1_MASK 227 207 228 208 .Lskip_spe_fgt_\@: 209 + mrs x1, id_aa64dfr0_el1 210 + ubfx x1, x1, #ID_AA64DFR0_EL1_BRBE_SHIFT, #4 211 + cbz x1, .Lskip_brbe_fgt_\@ 212 + 213 + /* 214 + * Disable read traps for the following registers 215 + * 216 + * [BRBSRC|BRBTGT|RBINF]_EL1 217 + * [BRBSRCINJ|BRBTGTINJ|BRBINFINJ|BRBTS]_EL1 218 + */ 219 + orr x0, x0, #HDFGRTR_EL2_nBRBDATA_MASK 220 + 221 + /* 222 + * Disable write traps for the following registers 223 + * 224 + * [BRBSRCINJ|BRBTGTINJ|BRBINFINJ|BRBTS]_EL1 225 + */ 226 + orr x2, x2, #HDFGWTR_EL2_nBRBDATA_MASK 227 + 228 + /* Disable read and write traps for [BRBCR|BRBFCR]_EL1 */ 229 + orr x0, x0, #HDFGRTR_EL2_nBRBCTL_MASK 230 + orr x2, x2, #HDFGWTR_EL2_nBRBCTL_MASK 231 + 232 + /* Disable read traps for BRBIDR_EL1 */ 233 + orr x0, x0, #HDFGRTR_EL2_nBRBIDR_MASK 234 + 235 + .Lskip_brbe_fgt_\@: 229 236 230 237 .Lset_debug_fgt_\@: 231 238 msr_s SYS_HDFGRTR_EL2, x0 232 - msr_s SYS_HDFGWTR_EL2, x0 239 + msr_s SYS_HDFGWTR_EL2, x2 233 240 234 241 mov x0, xzr 242 + mov x2, xzr 243 + 244 + mrs x1, id_aa64dfr0_el1 245 + ubfx x1, x1, #ID_AA64DFR0_EL1_BRBE_SHIFT, #4 246 + cbz x1, .Lskip_brbe_insn_fgt_\@ 247 + 248 + /* Disable traps for BRBIALL instruction */ 249 + orr x2, x2, #HFGITR_EL2_nBRBIALL_MASK 250 + 251 + /* Disable traps for BRBINJ instruction */ 252 + orr x2, x2, #HFGITR_EL2_nBRBINJ_MASK 253 + 254 + .Lskip_brbe_insn_fgt_\@: 235 255 mrs x1, id_aa64pfr1_el1 236 256 ubfx x1, x1, #ID_AA64PFR1_EL1_SME_SHIFT, #4 237 257 cbz x1, .Lskip_sme_fgt_\@ ··· 314 250 .Lset_fgt_\@: 315 251 msr_s SYS_HFGRTR_EL2, x0 316 252 msr_s SYS_HFGWTR_EL2, x0 317 - msr_s SYS_HFGITR_EL2, xzr 253 + msr_s SYS_HFGITR_EL2, x2 318 254 319 255 mrs x1, id_aa64pfr0_el1 // AMU traps UNDEF without AMU 320 256 ubfx x1, x1, #ID_AA64PFR0_EL1_AMU_SHIFT, #4 ··· 364 300 __init_el2_hcrx 365 301 __init_el2_timers 366 302 __init_el2_debug 303 + __init_el2_brbe 367 304 __init_el2_lor 368 305 __init_el2_stage2 369 306 __init_el2_gicv3

+13 -1

arch/arm64/include/asm/exception.h

··· 59 59 void do_el1_bti(struct pt_regs *regs, unsigned long esr); 60 60 void do_el0_gcs(struct pt_regs *regs, unsigned long esr); 61 61 void do_el1_gcs(struct pt_regs *regs, unsigned long esr); 62 - void do_debug_exception(unsigned long addr_if_watchpoint, unsigned long esr, 62 + #ifdef CONFIG_HAVE_HW_BREAKPOINT 63 + void do_breakpoint(unsigned long esr, struct pt_regs *regs); 64 + void do_watchpoint(unsigned long addr, unsigned long esr, 63 65 struct pt_regs *regs); 66 + #else 67 + static inline void do_breakpoint(unsigned long esr, struct pt_regs *regs) {} 68 + static inline void do_watchpoint(unsigned long addr, unsigned long esr, 69 + struct pt_regs *regs) {} 70 + #endif /* CONFIG_HAVE_HW_BREAKPOINT */ 71 + void do_el0_softstep(unsigned long esr, struct pt_regs *regs); 72 + void do_el1_softstep(unsigned long esr, struct pt_regs *regs); 73 + void do_el0_brk64(unsigned long esr, struct pt_regs *regs); 74 + void do_el1_brk64(unsigned long esr, struct pt_regs *regs); 75 + void do_bkpt32(unsigned long esr, struct pt_regs *regs); 64 76 void do_fpsimd_acc(unsigned long esr, struct pt_regs *regs); 65 77 void do_sve_acc(unsigned long esr, struct pt_regs *regs); 66 78 void do_sme_acc(unsigned long esr, struct pt_regs *regs);

+1 -1

arch/arm64/include/asm/gcs.h

··· 58 58 59 59 static inline bool task_gcs_el0_enabled(struct task_struct *task) 60 60 { 61 - return current->thread.gcs_el0_mode & PR_SHADOW_STACK_ENABLE; 61 + return task->thread.gcs_el0_mode & PR_SHADOW_STACK_ENABLE; 62 62 } 63 63 64 64 void gcs_set_el0_mode(struct task_struct *task);

+2

arch/arm64/include/asm/hwcap.h

··· 176 176 #define KERNEL_HWCAP_POE __khwcap2_feature(POE) 177 177 178 178 #define __khwcap3_feature(x) (const_ilog2(HWCAP3_ ## x) + 128) 179 + #define KERNEL_HWCAP_MTE_FAR __khwcap3_feature(MTE_FAR) 180 + #define KERNEL_HWCAP_MTE_STORE_ONLY __khwcap3_feature(MTE_STORE_ONLY) 179 181 180 182 /* 181 183 * This yields a mask that user programs can use to figure out what

+12

arch/arm64/include/asm/kgdb.h

··· 24 24 extern void kgdb_handle_bus_error(void); 25 25 extern int kgdb_fault_expected; 26 26 27 + int kgdb_brk_handler(struct pt_regs *regs, unsigned long esr); 28 + int kgdb_compiled_brk_handler(struct pt_regs *regs, unsigned long esr); 29 + #ifdef CONFIG_KGDB 30 + int kgdb_single_step_handler(struct pt_regs *regs, unsigned long esr); 31 + #else 32 + static inline int kgdb_single_step_handler(struct pt_regs *regs, 33 + unsigned long esr) 34 + { 35 + return DBG_HOOK_ERROR; 36 + } 37 + #endif 38 + 27 39 #endif /* !__ASSEMBLY__ */ 28 40 29 41 /*

+8

arch/arm64/include/asm/kprobes.h

··· 41 41 void __kprobes *trampoline_probe_handler(struct pt_regs *regs); 42 42 43 43 #endif /* CONFIG_KPROBES */ 44 + 45 + int __kprobes kprobe_brk_handler(struct pt_regs *regs, 46 + unsigned long esr); 47 + int __kprobes kprobe_ss_brk_handler(struct pt_regs *regs, 48 + unsigned long esr); 49 + int __kprobes kretprobe_brk_handler(struct pt_regs *regs, 50 + unsigned long esr); 51 + 44 52 #endif /* _ARM_KPROBES_H */

+2

arch/arm64/include/asm/kvm_host.h

··· 704 704 #define KVM_HOST_DATA_FLAG_EL1_TRACING_CONFIGURED 5 705 705 #define KVM_HOST_DATA_FLAG_VCPU_IN_HYP_CONTEXT 6 706 706 #define KVM_HOST_DATA_FLAG_L1_VNCR_MAPPED 7 707 + #define KVM_HOST_DATA_FLAG_HAS_BRBE 8 707 708 unsigned long flags; 708 709 709 710 struct kvm_cpu_context host_ctxt; ··· 738 737 u64 trfcr_el1; 739 738 /* Values of trap registers for the host before guest entry. */ 740 739 u64 mdcr_el2; 740 + u64 brbcr_el1; 741 741 } host_debug_state; 742 742 743 743 /* Guest trace filter value */

+1 -5

arch/arm64/include/asm/memory.h

··· 118 118 * VMAP'd stacks are allocated at page granularity, so we must ensure that such 119 119 * stacks are a multiple of page size. 120 120 */ 121 - #if defined(CONFIG_VMAP_STACK) && (MIN_THREAD_SHIFT < PAGE_SHIFT) 121 + #if (MIN_THREAD_SHIFT < PAGE_SHIFT) 122 122 #define THREAD_SHIFT PAGE_SHIFT 123 123 #else 124 124 #define THREAD_SHIFT MIN_THREAD_SHIFT ··· 135 135 * checking sp & (1 << THREAD_SHIFT), which we can do cheaply in the entry 136 136 * assembly. 137 137 */ 138 - #ifdef CONFIG_VMAP_STACK 139 138 #define THREAD_ALIGN (2 * THREAD_SIZE) 140 - #else 141 - #define THREAD_ALIGN THREAD_SIZE 142 - #endif 143 139 144 140 #define IRQ_STACK_SIZE THREAD_SIZE 145 141

+2

arch/arm64/include/asm/processor.h

··· 23 23 #define MTE_CTRL_TCF_ASYNC (1UL << 17) 24 24 #define MTE_CTRL_TCF_ASYMM (1UL << 18) 25 25 26 + #define MTE_CTRL_STORE_ONLY (1UL << 19) 27 + 26 28 #ifndef __ASSEMBLY__ 27 29 28 30 #include <linux/build_bug.h>

+1 -5

arch/arm64/include/asm/stacktrace.h

··· 59 59 60 60 #define on_thread_stack() (on_task_stack(current, current_stack_pointer, 1)) 61 61 62 - #ifdef CONFIG_VMAP_STACK 63 62 DECLARE_PER_CPU(unsigned long [OVERFLOW_STACK_SIZE/sizeof(long)], overflow_stack); 64 63 65 64 static inline struct stack_info stackinfo_get_overflow(void) ··· 71 72 .high = high, 72 73 }; 73 74 } 74 - #else 75 - #define stackinfo_get_overflow() stackinfo_get_unknown() 76 - #endif 77 75 78 - #if defined(CONFIG_ARM_SDE_INTERFACE) && defined(CONFIG_VMAP_STACK) 76 + #if defined(CONFIG_ARM_SDE_INTERFACE) 79 77 DECLARE_PER_CPU(unsigned long *, sdei_stack_normal_ptr); 80 78 DECLARE_PER_CPU(unsigned long *, sdei_stack_critical_ptr); 81 79

+6 -10

arch/arm64/include/asm/sysreg.h

··· 202 202 #define SYS_DBGVCR32_EL2 sys_reg(2, 4, 0, 7, 0) 203 203 204 204 #define SYS_BRBINF_EL1(n) sys_reg(2, 1, 8, (n & 15), (((n & 16) >> 2) | 0)) 205 - #define SYS_BRBINFINJ_EL1 sys_reg(2, 1, 9, 1, 0) 206 205 #define SYS_BRBSRC_EL1(n) sys_reg(2, 1, 8, (n & 15), (((n & 16) >> 2) | 1)) 207 - #define SYS_BRBSRCINJ_EL1 sys_reg(2, 1, 9, 1, 1) 208 206 #define SYS_BRBTGT_EL1(n) sys_reg(2, 1, 8, (n & 15), (((n & 16) >> 2) | 2)) 209 - #define SYS_BRBTGTINJ_EL1 sys_reg(2, 1, 9, 1, 2) 210 - #define SYS_BRBTS_EL1 sys_reg(2, 1, 9, 0, 2) 211 - 212 - #define SYS_BRBCR_EL1 sys_reg(2, 1, 9, 0, 0) 213 - #define SYS_BRBFCR_EL1 sys_reg(2, 1, 9, 0, 1) 214 - #define SYS_BRBIDR0_EL1 sys_reg(2, 1, 9, 2, 0) 215 207 216 208 #define SYS_TRCITECR_EL1 sys_reg(3, 0, 1, 2, 3) 217 209 #define SYS_TRCACATR(m) sys_reg(2, 1, 2, ((m & 7) << 1), (2 | (m >> 3))) ··· 268 276 269 277 /* ETM */ 270 278 #define SYS_TRCOSLAR sys_reg(2, 1, 1, 0, 4) 271 - 272 - #define SYS_BRBCR_EL2 sys_reg(2, 4, 9, 0, 0) 273 279 274 280 #define SYS_MIDR_EL1 sys_reg(3, 0, 0, 0, 0) 275 281 #define SYS_MPIDR_EL1 sys_reg(3, 0, 0, 0, 5) ··· 810 820 #define OP_DVP_RCTX sys_insn(1, 3, 7, 3, 5) 811 821 #define OP_COSP_RCTX sys_insn(1, 3, 7, 3, 6) 812 822 #define OP_CPP_RCTX sys_insn(1, 3, 7, 3, 7) 823 + 824 + /* 825 + * BRBE Instructions 826 + */ 827 + #define BRB_IALL_INSN __emit_inst(0xd5000000 | OP_BRB_IALL | (0x1f)) 828 + #define BRB_INJ_INSN __emit_inst(0xd5000000 | OP_BRB_INJ | (0x1f)) 813 829 814 830 /* Common SCTLR_ELx flags. */ 815 831 #define SCTLR_ELx_ENTP2 (BIT(60))

-4

arch/arm64/include/asm/system_misc.h

··· 25 25 int signo, int sicode, unsigned long far, 26 26 unsigned long err); 27 27 28 - void hook_debug_fault_code(int nr, int (*fn)(unsigned long, unsigned long, 29 - struct pt_regs *), 30 - int sig, int code, const char *name); 31 - 32 28 struct mm_struct; 33 29 extern void __show_regs(struct pt_regs *); 34 30

+4 -1

arch/arm64/include/asm/thread_info.h

··· 70 70 #define TIF_SYSCALL_TRACEPOINT 10 /* syscall tracepoint for ftrace */ 71 71 #define TIF_SECCOMP 11 /* syscall secure computing */ 72 72 #define TIF_SYSCALL_EMU 12 /* syscall emulation active */ 73 + #define TIF_PATCH_PENDING 13 /* pending live patching update */ 73 74 #define TIF_MEMDIE 18 /* is terminating due to OOM killer */ 74 75 #define TIF_FREEZE 19 75 76 #define TIF_RESTORE_SIGMASK 20 ··· 97 96 #define _TIF_SYSCALL_TRACEPOINT (1 << TIF_SYSCALL_TRACEPOINT) 98 97 #define _TIF_SECCOMP (1 << TIF_SECCOMP) 99 98 #define _TIF_SYSCALL_EMU (1 << TIF_SYSCALL_EMU) 99 + #define _TIF_PATCH_PENDING (1 << TIF_PATCH_PENDING) 100 100 #define _TIF_UPROBE (1 << TIF_UPROBE) 101 101 #define _TIF_SINGLESTEP (1 << TIF_SINGLESTEP) 102 102 #define _TIF_32BIT (1 << TIF_32BIT) ··· 109 107 #define _TIF_WORK_MASK (_TIF_NEED_RESCHED | _TIF_NEED_RESCHED_LAZY | \ 110 108 _TIF_NOTIFY_RESUME | _TIF_FOREIGN_FPSTATE | \ 111 109 _TIF_UPROBE | _TIF_MTE_ASYNC_FAULT | \ 112 - _TIF_NOTIFY_SIGNAL | _TIF_SIGPENDING) 110 + _TIF_NOTIFY_SIGNAL | _TIF_SIGPENDING | \ 111 + _TIF_PATCH_PENDING) 113 112 114 113 #define _TIF_SYSCALL_WORK (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | \ 115 114 _TIF_SYSCALL_TRACEPOINT | _TIF_SECCOMP | \

+6

arch/arm64/include/asm/traps.h

··· 29 29 void arm64_force_sig_mceerr(int code, unsigned long far, short lsb, const char *str); 30 30 void arm64_force_sig_ptrace_errno_trap(int errno, unsigned long far, const char *str); 31 31 32 + int bug_brk_handler(struct pt_regs *regs, unsigned long esr); 33 + int cfi_brk_handler(struct pt_regs *regs, unsigned long esr); 34 + int reserved_fault_brk_handler(struct pt_regs *regs, unsigned long esr); 35 + int kasan_brk_handler(struct pt_regs *regs, unsigned long esr); 36 + int ubsan_brk_handler(struct pt_regs *regs, unsigned long esr); 37 + 32 38 int early_brk64(unsigned long addr, unsigned long esr, struct pt_regs *regs); 33 39 34 40 /*

+11

arch/arm64/include/asm/uprobes.h

··· 28 28 bool simulate; 29 29 }; 30 30 31 + int uprobe_brk_handler(struct pt_regs *regs, unsigned long esr); 32 + #ifdef CONFIG_UPROBES 33 + int uprobe_single_step_handler(struct pt_regs *regs, unsigned long esr); 34 + #else 35 + static inline int uprobe_single_step_handler(struct pt_regs *regs, 36 + unsigned long esr) 37 + { 38 + return DBG_HOOK_ERROR; 39 + } 40 + #endif 41 + 31 42 #endif

+2

arch/arm64/include/uapi/asm/hwcap.h

··· 143 143 /* 144 144 * HWCAP3 flags - for AT_HWCAP3 145 145 */ 146 + #define HWCAP3_MTE_FAR (1UL << 0) 147 + #define HWCAP3_MTE_STORE_ONLY (1UL << 1) 146 148 147 149 #endif /* _UAPI__ASM_HWCAP_H */

+1 -1

arch/arm64/kernel/Makefile

··· 80 80 always-$(KBUILD_BUILTIN) += vmlinux.lds 81 81 82 82 ifeq ($(CONFIG_DEBUG_EFI),y) 83 - AFLAGS_head.o += -DVMLINUX_PATH="\"$(realpath $(objtree)/vmlinux)\"" 83 + AFLAGS_head.o += -DVMLINUX_PATH="\"$(abspath vmlinux)\"" 84 84 endif 85 85 86 86 # for cleaning

+7 -3

arch/arm64/kernel/acpi.c

··· 197 197 */ 198 198 void __init acpi_boot_table_init(void) 199 199 { 200 + int ret; 201 + 200 202 /* 201 203 * Enable ACPI instead of device tree unless 202 204 * - ACPI has been disabled explicitly (acpi=off), or ··· 252 250 * behaviour, use acpi=nospcr to disable console in ACPI SPCR 253 251 * table as default serial console. 254 252 */ 255 - acpi_parse_spcr(earlycon_acpi_spcr_enable, 253 + ret = acpi_parse_spcr(earlycon_acpi_spcr_enable, 256 254 !param_acpi_nospcr); 257 - pr_info("Use ACPI SPCR as default console: %s\n", 258 - param_acpi_nospcr ? "No" : "Yes"); 255 + if (!ret || param_acpi_nospcr || !IS_ENABLED(CONFIG_ACPI_SPCR_TABLE)) 256 + pr_info("Use ACPI SPCR as default console: No\n"); 257 + else 258 + pr_info("Use ACPI SPCR as default console: Yes\n"); 259 259 260 260 if (IS_ENABLED(CONFIG_ACPI_BGRT)) 261 261 acpi_table_parse(ACPI_SIG_BGRT, acpi_parse_bgrt);

+105 -11

arch/arm64/kernel/cpufeature.c

··· 320 320 321 321 static const struct arm64_ftr_bits ftr_id_aa64pfr2[] = { 322 322 ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR2_EL1_FPMR_SHIFT, 4, 0), 323 + ARM64_FTR_BITS(FTR_VISIBLE, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR2_EL1_MTEFAR_SHIFT, 4, ID_AA64PFR2_EL1_MTEFAR_NI), 324 + ARM64_FTR_BITS(FTR_VISIBLE, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR2_EL1_MTESTOREONLY_SHIFT, 4, ID_AA64PFR2_EL1_MTESTOREONLY_NI), 323 325 ARM64_FTR_END, 324 326 }; 325 327 ··· 2215 2213 return arm64_test_sw_feature_override(ARM64_SW_FEATURE_OVERRIDE_HVHE); 2216 2214 } 2217 2215 2216 + static bool has_bbml2_noabort(const struct arm64_cpu_capabilities *caps, int scope) 2217 + { 2218 + /* 2219 + * We want to allow usage of BBML2 in as wide a range of kernel contexts 2220 + * as possible. This list is therefore an allow-list of known-good 2221 + * implementations that both support BBML2 and additionally, fulfill the 2222 + * extra constraint of never generating TLB conflict aborts when using 2223 + * the relaxed BBML2 semantics (such aborts make use of BBML2 in certain 2224 + * kernel contexts difficult to prove safe against recursive aborts). 2225 + * 2226 + * Note that implementations can only be considered "known-good" if their 2227 + * implementors attest to the fact that the implementation never raises 2228 + * TLB conflict aborts for BBML2 mapping granularity changes. 2229 + */ 2230 + static const struct midr_range supports_bbml2_noabort_list[] = { 2231 + MIDR_REV_RANGE(MIDR_CORTEX_X4, 0, 3, 0xf), 2232 + MIDR_REV_RANGE(MIDR_NEOVERSE_V3, 0, 2, 0xf), 2233 + {} 2234 + }; 2235 + 2236 + /* Does our cpu guarantee to never raise TLB conflict aborts? */ 2237 + if (!is_midr_in_range_list(supports_bbml2_noabort_list)) 2238 + return false; 2239 + 2240 + /* 2241 + * We currently ignore the ID_AA64MMFR2_EL1 register, and only care 2242 + * about whether the MIDR check passes. 2243 + */ 2244 + 2245 + return true; 2246 + } 2247 + 2218 2248 #ifdef CONFIG_ARM64_PAN 2219 2249 static void cpu_enable_pan(const struct arm64_cpu_capabilities *__unused) 2220 2250 { ··· 2908 2874 .matches = has_cpuid_feature, 2909 2875 ARM64_CPUID_FIELDS(ID_AA64PFR1_EL1, MTE, MTE3) 2910 2876 }, 2877 + { 2878 + .desc = "FAR on MTE Tag Check Fault", 2879 + .capability = ARM64_MTE_FAR, 2880 + .type = ARM64_CPUCAP_SYSTEM_FEATURE, 2881 + .matches = has_cpuid_feature, 2882 + ARM64_CPUID_FIELDS(ID_AA64PFR2_EL1, MTEFAR, IMP) 2883 + }, 2884 + { 2885 + .desc = "Store Only MTE Tag Check", 2886 + .capability = ARM64_MTE_STORE_ONLY, 2887 + .type = ARM64_CPUCAP_SYSTEM_FEATURE, 2888 + .matches = has_cpuid_feature, 2889 + ARM64_CPUID_FIELDS(ID_AA64PFR2_EL1, MTESTOREONLY, IMP) 2890 + }, 2911 2891 #endif /* CONFIG_ARM64_MTE */ 2912 2892 { 2913 2893 .desc = "RCpc load-acquire (LDAPR)", ··· 3027 2979 .type = ARM64_CPUCAP_SYSTEM_FEATURE, 3028 2980 .matches = has_cpuid_feature, 3029 2981 ARM64_CPUID_FIELDS(ID_AA64MMFR2_EL1, EVT, IMP) 2982 + }, 2983 + { 2984 + .desc = "BBM Level 2 without TLB conflict abort", 2985 + .capability = ARM64_HAS_BBML2_NOABORT, 2986 + .type = ARM64_CPUCAP_EARLY_LOCAL_CPU_FEATURE, 2987 + .matches = has_bbml2_noabort, 3030 2988 }, 3031 2989 { 3032 2990 .desc = "52-bit Virtual Addressing for KVM (LPA2)", ··· 3272 3218 #ifdef CONFIG_ARM64_MTE 3273 3219 HWCAP_CAP(ID_AA64PFR1_EL1, MTE, MTE2, CAP_HWCAP, KERNEL_HWCAP_MTE), 3274 3220 HWCAP_CAP(ID_AA64PFR1_EL1, MTE, MTE3, CAP_HWCAP, KERNEL_HWCAP_MTE3), 3221 + HWCAP_CAP(ID_AA64PFR2_EL1, MTEFAR, IMP, CAP_HWCAP, KERNEL_HWCAP_MTE_FAR), 3222 + HWCAP_CAP(ID_AA64PFR2_EL1, MTESTOREONLY, IMP, CAP_HWCAP , KERNEL_HWCAP_MTE_STORE_ONLY), 3275 3223 #endif /* CONFIG_ARM64_MTE */ 3276 3224 HWCAP_CAP(ID_AA64MMFR0_EL1, ECV, IMP, CAP_HWCAP, KERNEL_HWCAP_ECV), 3277 3225 HWCAP_CAP(ID_AA64MMFR1_EL1, AFP, IMP, CAP_HWCAP, KERNEL_HWCAP_AFP), ··· 3433 3377 3434 3378 scope_mask &= ARM64_CPUCAP_SCOPE_MASK; 3435 3379 for (i = 0; i < ARM64_NCAPS; i++) { 3380 + bool match_all = false; 3381 + bool caps_set = false; 3382 + bool boot_cpu = false; 3383 + 3436 3384 caps = cpucap_ptrs[i]; 3437 - if (!caps || !(caps->type & scope_mask) || 3438 - cpus_have_cap(caps->capability) || 3439 - !caps->matches(caps, cpucap_default_scope(caps))) 3385 + if (!caps || !(caps->type & scope_mask)) 3440 3386 continue; 3441 3387 3442 - if (caps->desc && !caps->cpus) 3388 + match_all = cpucap_match_all_early_cpus(caps); 3389 + caps_set = cpus_have_cap(caps->capability); 3390 + boot_cpu = scope_mask & SCOPE_BOOT_CPU; 3391 + 3392 + /* 3393 + * Unless it's a match-all CPUs feature, avoid probing if 3394 + * already detected. 3395 + */ 3396 + if (!match_all && caps_set) 3397 + continue; 3398 + 3399 + /* 3400 + * A match-all CPUs capability is only set when probing the 3401 + * boot CPU. It may be cleared subsequently if not detected on 3402 + * secondary ones. 3403 + */ 3404 + if (match_all && !caps_set && !boot_cpu) 3405 + continue; 3406 + 3407 + if (!caps->matches(caps, cpucap_default_scope(caps))) { 3408 + if (match_all) 3409 + __clear_bit(caps->capability, system_cpucaps); 3410 + continue; 3411 + } 3412 + 3413 + /* 3414 + * Match-all CPUs capabilities are logged later when the 3415 + * system capabilities are finalised. 3416 + */ 3417 + if (!match_all && caps->desc && !caps->cpus) 3443 3418 pr_info("detected: %s\n", caps->desc); 3444 3419 3445 3420 __set_bit(caps->capability, system_cpucaps); 3446 3421 3447 - if ((scope_mask & SCOPE_BOOT_CPU) && (caps->type & SCOPE_BOOT_CPU)) 3422 + if (boot_cpu && (caps->type & SCOPE_BOOT_CPU)) 3448 3423 set_bit(caps->capability, boot_cpucaps); 3449 3424 } 3450 3425 } ··· 3876 3789 enable_cpu_capabilities(SCOPE_ALL & ~SCOPE_BOOT_CPU); 3877 3790 apply_alternatives_all(); 3878 3791 3879 - /* 3880 - * Log any cpucaps with a cpumask as these aren't logged by 3881 - * update_cpu_capabilities(). 3882 - */ 3883 3792 for (int i = 0; i < ARM64_NCAPS; i++) { 3884 3793 const struct arm64_cpu_capabilities *caps = cpucap_ptrs[i]; 3885 3794 3886 - if (caps && caps->cpus && caps->desc && 3887 - cpumask_any(caps->cpus) < nr_cpu_ids) 3795 + if (!caps || !caps->desc) 3796 + continue; 3797 + 3798 + /* 3799 + * Log any cpucaps with a cpumask as these aren't logged by 3800 + * update_cpu_capabilities(). 3801 + */ 3802 + if (caps->cpus && cpumask_any(caps->cpus) < nr_cpu_ids) 3888 3803 pr_info("detected: %s on CPU%*pbl\n", 3889 3804 caps->desc, cpumask_pr_args(caps->cpus)); 3805 + 3806 + /* Log match-all CPUs capabilities */ 3807 + if (cpucap_match_all_early_cpus(caps) && 3808 + cpus_have_cap(caps->capability)) 3809 + pr_info("detected: %s\n", caps->desc); 3890 3810 } 3891 3811 3892 3812 /*

+2

arch/arm64/kernel/cpuinfo.c

··· 160 160 [KERNEL_HWCAP_SME_SFEXPA] = "smesfexpa", 161 161 [KERNEL_HWCAP_SME_STMOP] = "smestmop", 162 162 [KERNEL_HWCAP_SME_SMOP4] = "smesmop4", 163 + [KERNEL_HWCAP_MTE_FAR] = "mtefar", 164 + [KERNEL_HWCAP_MTE_STORE_ONLY] = "mtestoreonly", 163 165 }; 164 166 165 167 #ifdef CONFIG_COMPAT

+105 -172

arch/arm64/kernel/debug-monitors.c

··· 21 21 #include <asm/cputype.h> 22 22 #include <asm/daifflags.h> 23 23 #include <asm/debug-monitors.h> 24 + #include <asm/exception.h> 25 + #include <asm/kgdb.h> 26 + #include <asm/kprobes.h> 24 27 #include <asm/system_misc.h> 25 28 #include <asm/traps.h> 29 + #include <asm/uprobes.h> 26 30 27 31 /* Determine debug architecture. */ 28 32 u8 debug_monitors_arch(void) ··· 38 34 /* 39 35 * MDSCR access routines. 40 36 */ 41 - static void mdscr_write(u32 mdscr) 37 + static void mdscr_write(u64 mdscr) 42 38 { 43 39 unsigned long flags; 44 40 flags = local_daif_save(); ··· 47 43 } 48 44 NOKPROBE_SYMBOL(mdscr_write); 49 45 50 - static u32 mdscr_read(void) 46 + static u64 mdscr_read(void) 51 47 { 52 48 return read_sysreg(mdscr_el1); 53 49 } ··· 83 79 84 80 void enable_debug_monitors(enum dbg_active_el el) 85 81 { 86 - u32 mdscr, enable = 0; 82 + u64 mdscr, enable = 0; 87 83 88 84 WARN_ON(preemptible()); 89 85 90 86 if (this_cpu_inc_return(mde_ref_count) == 1) 91 - enable = DBG_MDSCR_MDE; 87 + enable = MDSCR_EL1_MDE; 92 88 93 89 if (el == DBG_ACTIVE_EL1 && 94 90 this_cpu_inc_return(kde_ref_count) == 1) 95 - enable |= DBG_MDSCR_KDE; 91 + enable |= MDSCR_EL1_KDE; 96 92 97 93 if (enable && debug_enabled) { 98 94 mdscr = mdscr_read(); ··· 104 100 105 101 void disable_debug_monitors(enum dbg_active_el el) 106 102 { 107 - u32 mdscr, disable = 0; 103 + u64 mdscr, disable = 0; 108 104 109 105 WARN_ON(preemptible()); 110 106 111 107 if (this_cpu_dec_return(mde_ref_count) == 0) 112 - disable = ~DBG_MDSCR_MDE; 108 + disable = ~MDSCR_EL1_MDE; 113 109 114 110 if (el == DBG_ACTIVE_EL1 && 115 111 this_cpu_dec_return(kde_ref_count) == 0) 116 - disable &= ~DBG_MDSCR_KDE; 112 + disable &= ~MDSCR_EL1_KDE; 117 113 118 114 if (disable) { 119 115 mdscr = mdscr_read(); ··· 160 156 #define set_regs_spsr_ss(r) set_user_regs_spsr_ss(&(r)->user_regs) 161 157 #define clear_regs_spsr_ss(r) clear_user_regs_spsr_ss(&(r)->user_regs) 162 158 163 - static DEFINE_SPINLOCK(debug_hook_lock); 164 - static LIST_HEAD(user_step_hook); 165 - static LIST_HEAD(kernel_step_hook); 166 - 167 - static void register_debug_hook(struct list_head *node, struct list_head *list) 168 - { 169 - spin_lock(&debug_hook_lock); 170 - list_add_rcu(node, list); 171 - spin_unlock(&debug_hook_lock); 172 - 173 - } 174 - 175 - static void unregister_debug_hook(struct list_head *node) 176 - { 177 - spin_lock(&debug_hook_lock); 178 - list_del_rcu(node); 179 - spin_unlock(&debug_hook_lock); 180 - synchronize_rcu(); 181 - } 182 - 183 - void register_user_step_hook(struct step_hook *hook) 184 - { 185 - register_debug_hook(&hook->node, &user_step_hook); 186 - } 187 - 188 - void unregister_user_step_hook(struct step_hook *hook) 189 - { 190 - unregister_debug_hook(&hook->node); 191 - } 192 - 193 - void register_kernel_step_hook(struct step_hook *hook) 194 - { 195 - register_debug_hook(&hook->node, &kernel_step_hook); 196 - } 197 - 198 - void unregister_kernel_step_hook(struct step_hook *hook) 199 - { 200 - unregister_debug_hook(&hook->node); 201 - } 202 - 203 - /* 204 - * Call registered single step handlers 205 - * There is no Syndrome info to check for determining the handler. 206 - * So we call all the registered handlers, until the right handler is 207 - * found which returns zero. 208 - */ 209 - static int call_step_hook(struct pt_regs *regs, unsigned long esr) 210 - { 211 - struct step_hook *hook; 212 - struct list_head *list; 213 - int retval = DBG_HOOK_ERROR; 214 - 215 - list = user_mode(regs) ? &user_step_hook : &kernel_step_hook; 216 - 217 - /* 218 - * Since single-step exception disables interrupt, this function is 219 - * entirely not preemptible, and we can use rcu list safely here. 220 - */ 221 - list_for_each_entry_rcu(hook, list, node) { 222 - retval = hook->fn(regs, esr); 223 - if (retval == DBG_HOOK_HANDLED) 224 - break; 225 - } 226 - 227 - return retval; 228 - } 229 - NOKPROBE_SYMBOL(call_step_hook); 230 - 231 159 static void send_user_sigtrap(int si_code) 232 160 { 233 161 struct pt_regs *regs = current_pt_regs(); ··· 174 238 "User debug trap"); 175 239 } 176 240 177 - static int single_step_handler(unsigned long unused, unsigned long esr, 178 - struct pt_regs *regs) 241 + /* 242 + * We have already unmasked interrupts and enabled preemption 243 + * when calling do_el0_softstep() from entry-common.c. 244 + */ 245 + void do_el0_softstep(unsigned long esr, struct pt_regs *regs) 179 246 { 180 - bool handler_found = false; 247 + if (uprobe_single_step_handler(regs, esr) == DBG_HOOK_HANDLED) 248 + return; 181 249 250 + send_user_sigtrap(TRAP_TRACE); 182 251 /* 183 - * If we are stepping a pending breakpoint, call the hw_breakpoint 184 - * handler first. 252 + * ptrace will disable single step unless explicitly 253 + * asked to re-enable it. For other clients, it makes 254 + * sense to leave it enabled (i.e. rewind the controls 255 + * to the active-not-pending state). 185 256 */ 186 - if (!reinstall_suspended_bps(regs)) 187 - return 0; 257 + user_rewind_single_step(current); 258 + } 188 259 189 - if (!handler_found && call_step_hook(regs, esr) == DBG_HOOK_HANDLED) 190 - handler_found = true; 260 + void do_el1_softstep(unsigned long esr, struct pt_regs *regs) 261 + { 262 + if (kgdb_single_step_handler(regs, esr) == DBG_HOOK_HANDLED) 263 + return; 191 264 192 - if (!handler_found && user_mode(regs)) { 193 - send_user_sigtrap(TRAP_TRACE); 265 + pr_warn("Unexpected kernel single-step exception at EL1\n"); 266 + /* 267 + * Re-enable stepping since we know that we will be 268 + * returning to regs. 269 + */ 270 + set_regs_spsr_ss(regs); 271 + } 272 + NOKPROBE_SYMBOL(do_el1_softstep); 194 273 195 - /* 196 - * ptrace will disable single step unless explicitly 197 - * asked to re-enable it. For other clients, it makes 198 - * sense to leave it enabled (i.e. rewind the controls 199 - * to the active-not-pending state). 200 - */ 201 - user_rewind_single_step(current); 202 - } else if (!handler_found) { 203 - pr_warn("Unexpected kernel single-step exception at EL1\n"); 204 - /* 205 - * Re-enable stepping since we know that we will be 206 - * returning to regs. 207 - */ 208 - set_regs_spsr_ss(regs); 274 + static int call_el1_break_hook(struct pt_regs *regs, unsigned long esr) 275 + { 276 + if (esr_brk_comment(esr) == BUG_BRK_IMM) 277 + return bug_brk_handler(regs, esr); 278 + 279 + if (IS_ENABLED(CONFIG_CFI_CLANG) && esr_is_cfi_brk(esr)) 280 + return cfi_brk_handler(regs, esr); 281 + 282 + if (esr_brk_comment(esr) == FAULT_BRK_IMM) 283 + return reserved_fault_brk_handler(regs, esr); 284 + 285 + if (IS_ENABLED(CONFIG_KASAN_SW_TAGS) && 286 + (esr_brk_comment(esr) & ~KASAN_BRK_MASK) == KASAN_BRK_IMM) 287 + return kasan_brk_handler(regs, esr); 288 + 289 + if (IS_ENABLED(CONFIG_UBSAN_TRAP) && esr_is_ubsan_brk(esr)) 290 + return ubsan_brk_handler(regs, esr); 291 + 292 + if (IS_ENABLED(CONFIG_KGDB)) { 293 + if (esr_brk_comment(esr) == KGDB_DYN_DBG_BRK_IMM) 294 + return kgdb_brk_handler(regs, esr); 295 + if (esr_brk_comment(esr) == KGDB_COMPILED_DBG_BRK_IMM) 296 + return kgdb_compiled_brk_handler(regs, esr); 209 297 } 210 298 211 - return 0; 212 - } 213 - NOKPROBE_SYMBOL(single_step_handler); 214 - 215 - static LIST_HEAD(user_break_hook); 216 - static LIST_HEAD(kernel_break_hook); 217 - 218 - void register_user_break_hook(struct break_hook *hook) 219 - { 220 - register_debug_hook(&hook->node, &user_break_hook); 221 - } 222 - 223 - void unregister_user_break_hook(struct break_hook *hook) 224 - { 225 - unregister_debug_hook(&hook->node); 226 - } 227 - 228 - void register_kernel_break_hook(struct break_hook *hook) 229 - { 230 - register_debug_hook(&hook->node, &kernel_break_hook); 231 - } 232 - 233 - void unregister_kernel_break_hook(struct break_hook *hook) 234 - { 235 - unregister_debug_hook(&hook->node); 236 - } 237 - 238 - static int call_break_hook(struct pt_regs *regs, unsigned long esr) 239 - { 240 - struct break_hook *hook; 241 - struct list_head *list; 242 - 243 - list = user_mode(regs) ? &user_break_hook : &kernel_break_hook; 244 - 245 - /* 246 - * Since brk exception disables interrupt, this function is 247 - * entirely not preemptible, and we can use rcu list safely here. 248 - */ 249 - list_for_each_entry_rcu(hook, list, node) { 250 - if ((esr_brk_comment(esr) & ~hook->mask) == hook->imm) 251 - return hook->fn(regs, esr); 299 + if (IS_ENABLED(CONFIG_KPROBES)) { 300 + if (esr_brk_comment(esr) == KPROBES_BRK_IMM) 301 + return kprobe_brk_handler(regs, esr); 302 + if (esr_brk_comment(esr) == KPROBES_BRK_SS_IMM) 303 + return kprobe_ss_brk_handler(regs, esr); 252 304 } 305 + 306 + if (IS_ENABLED(CONFIG_KRETPROBES) && 307 + esr_brk_comment(esr) == KRETPROBES_BRK_IMM) 308 + return kretprobe_brk_handler(regs, esr); 253 309 254 310 return DBG_HOOK_ERROR; 255 311 } 256 - NOKPROBE_SYMBOL(call_break_hook); 312 + NOKPROBE_SYMBOL(call_el1_break_hook); 257 313 258 - static int brk_handler(unsigned long unused, unsigned long esr, 259 - struct pt_regs *regs) 314 + /* 315 + * We have already unmasked interrupts and enabled preemption 316 + * when calling do_el0_brk64() from entry-common.c. 317 + */ 318 + void do_el0_brk64(unsigned long esr, struct pt_regs *regs) 260 319 { 261 - if (call_break_hook(regs, esr) == DBG_HOOK_HANDLED) 262 - return 0; 320 + if (IS_ENABLED(CONFIG_UPROBES) && 321 + esr_brk_comment(esr) == UPROBES_BRK_IMM && 322 + uprobe_brk_handler(regs, esr) == DBG_HOOK_HANDLED) 323 + return; 263 324 264 - if (user_mode(regs)) { 265 - send_user_sigtrap(TRAP_BRKPT); 266 - } else { 267 - pr_warn("Unexpected kernel BRK exception at EL1\n"); 268 - return -EFAULT; 269 - } 270 - 271 - return 0; 325 + send_user_sigtrap(TRAP_BRKPT); 272 326 } 273 - NOKPROBE_SYMBOL(brk_handler); 274 327 275 - int aarch32_break_handler(struct pt_regs *regs) 328 + void do_el1_brk64(unsigned long esr, struct pt_regs *regs) 329 + { 330 + if (call_el1_break_hook(regs, esr) == DBG_HOOK_HANDLED) 331 + return; 332 + 333 + die("Oops - BRK", regs, esr); 334 + } 335 + NOKPROBE_SYMBOL(do_el1_brk64); 336 + 337 + #ifdef CONFIG_COMPAT 338 + void do_bkpt32(unsigned long esr, struct pt_regs *regs) 339 + { 340 + arm64_notify_die("aarch32 BKPT", regs, SIGTRAP, TRAP_BRKPT, regs->pc, esr); 341 + } 342 + #endif /* CONFIG_COMPAT */ 343 + 344 + bool try_handle_aarch32_break(struct pt_regs *regs) 276 345 { 277 346 u32 arm_instr; 278 347 u16 thumb_instr; ··· 285 344 void __user *pc = (void __user *)instruction_pointer(regs); 286 345 287 346 if (!compat_user_mode(regs)) 288 - return -EFAULT; 347 + return false; 289 348 290 349 if (compat_thumb_mode(regs)) { 291 350 /* get 16-bit Thumb instruction */ ··· 309 368 } 310 369 311 370 if (!bp) 312 - return -EFAULT; 371 + return false; 313 372 314 373 send_user_sigtrap(TRAP_BRKPT); 315 - return 0; 374 + return true; 316 375 } 317 - NOKPROBE_SYMBOL(aarch32_break_handler); 318 - 319 - void __init debug_traps_init(void) 320 - { 321 - hook_debug_fault_code(DBG_ESR_EVT_HWSS, single_step_handler, SIGTRAP, 322 - TRAP_TRACE, "single-step handler"); 323 - hook_debug_fault_code(DBG_ESR_EVT_BRK, brk_handler, SIGTRAP, 324 - TRAP_BRKPT, "BRK handler"); 325 - } 376 + NOKPROBE_SYMBOL(try_handle_aarch32_break); 326 377 327 378 /* Re-enable single step for syscall restarting. */ 328 379 void user_rewind_single_step(struct task_struct *task) ··· 348 415 { 349 416 WARN_ON(!irqs_disabled()); 350 417 set_regs_spsr_ss(regs); 351 - mdscr_write(mdscr_read() | DBG_MDSCR_SS); 418 + mdscr_write(mdscr_read() | MDSCR_EL1_SS); 352 419 enable_debug_monitors(DBG_ACTIVE_EL1); 353 420 } 354 421 NOKPROBE_SYMBOL(kernel_enable_single_step); ··· 356 423 void kernel_disable_single_step(void) 357 424 { 358 425 WARN_ON(!irqs_disabled()); 359 - mdscr_write(mdscr_read() & ~DBG_MDSCR_SS); 426 + mdscr_write(mdscr_read() & ~MDSCR_EL1_SS); 360 427 disable_debug_monitors(DBG_ACTIVE_EL1); 361 428 } 362 429 NOKPROBE_SYMBOL(kernel_disable_single_step); ··· 364 431 int kernel_active_single_step(void) 365 432 { 366 433 WARN_ON(!irqs_disabled()); 367 - return mdscr_read() & DBG_MDSCR_SS; 434 + return mdscr_read() & MDSCR_EL1_SS; 368 435 } 369 436 NOKPROBE_SYMBOL(kernel_active_single_step); 370 437

-5

arch/arm64/kernel/efi.c

··· 215 215 if (!efi_enabled(EFI_RUNTIME_SERVICES)) 216 216 return 0; 217 217 218 - if (!IS_ENABLED(CONFIG_VMAP_STACK)) { 219 - clear_bit(EFI_RUNTIME_SERVICES, &efi.flags); 220 - return -ENOMEM; 221 - } 222 - 223 218 p = arch_alloc_vmap_stack(THREAD_SIZE, NUMA_NO_NODE); 224 219 if (!p) { 225 220 pr_warn("Failed to allocate EFI runtime stack\n");

+143 -13

arch/arm64/kernel/entry-common.c

··· 8 8 #include <linux/context_tracking.h> 9 9 #include <linux/kasan.h> 10 10 #include <linux/linkage.h> 11 + #include <linux/livepatch.h> 11 12 #include <linux/lockdep.h> 12 13 #include <linux/ptrace.h> 13 14 #include <linux/resume_user_mode.h> ··· 144 143 send_sig_fault(SIGSEGV, SEGV_MTEAERR, 145 144 (void __user *)NULL, current); 146 145 } 146 + 147 + if (thread_flags & _TIF_PATCH_PENDING) 148 + klp_update_patch_state(current); 147 149 148 150 if (thread_flags & (_TIF_SIGPENDING | _TIF_NOTIFY_SIGNAL)) 149 151 do_signal(regs); ··· 348 344 349 345 static void cortex_a76_erratum_1463225_svc_handler(void) 350 346 { 351 - u32 reg, val; 347 + u64 reg, val; 352 348 353 349 if (!unlikely(test_thread_flag(TIF_SINGLESTEP))) 354 350 return; ··· 358 354 359 355 __this_cpu_write(__in_cortex_a76_erratum_1463225_wa, 1); 360 356 reg = read_sysreg(mdscr_el1); 361 - val = reg | DBG_MDSCR_SS | DBG_MDSCR_KDE; 357 + val = reg | MDSCR_EL1_SS | MDSCR_EL1_KDE; 362 358 write_sysreg(val, mdscr_el1); 363 359 asm volatile("msr daifclr, #8"); 364 360 isb(); ··· 445 441 __this_cpu_write(fpsimd_last_state.to_save, FP_STATE_CURRENT); 446 442 } 447 443 444 + /* 445 + * In debug exception context, we explicitly disable preemption despite 446 + * having interrupts disabled. 447 + * This serves two purposes: it makes it much less likely that we would 448 + * accidentally schedule in exception context and it will force a warning 449 + * if we somehow manage to schedule by accident. 450 + */ 451 + static void debug_exception_enter(struct pt_regs *regs) 452 + { 453 + preempt_disable(); 454 + 455 + /* This code is a bit fragile. Test it. */ 456 + RCU_LOCKDEP_WARN(!rcu_is_watching(), "exception_enter didn't work"); 457 + } 458 + NOKPROBE_SYMBOL(debug_exception_enter); 459 + 460 + static void debug_exception_exit(struct pt_regs *regs) 461 + { 462 + preempt_enable_no_resched(); 463 + } 464 + NOKPROBE_SYMBOL(debug_exception_exit); 465 + 448 466 UNHANDLED(el1t, 64, sync) 449 467 UNHANDLED(el1t, 64, irq) 450 468 UNHANDLED(el1t, 64, fiq) ··· 530 504 exit_to_kernel_mode(regs); 531 505 } 532 506 533 - static void noinstr el1_dbg(struct pt_regs *regs, unsigned long esr) 507 + static void noinstr el1_breakpt(struct pt_regs *regs, unsigned long esr) 534 508 { 509 + arm64_enter_el1_dbg(regs); 510 + debug_exception_enter(regs); 511 + do_breakpoint(esr, regs); 512 + debug_exception_exit(regs); 513 + arm64_exit_el1_dbg(regs); 514 + } 515 + 516 + static void noinstr el1_softstp(struct pt_regs *regs, unsigned long esr) 517 + { 518 + arm64_enter_el1_dbg(regs); 519 + if (!cortex_a76_erratum_1463225_debug_handler(regs)) { 520 + debug_exception_enter(regs); 521 + /* 522 + * After handling a breakpoint, we suspend the breakpoint 523 + * and use single-step to move to the next instruction. 524 + * If we are stepping a suspended breakpoint there's nothing more to do: 525 + * the single-step is complete. 526 + */ 527 + if (!try_step_suspended_breakpoints(regs)) 528 + do_el1_softstep(esr, regs); 529 + debug_exception_exit(regs); 530 + } 531 + arm64_exit_el1_dbg(regs); 532 + } 533 + 534 + static void noinstr el1_watchpt(struct pt_regs *regs, unsigned long esr) 535 + { 536 + /* Watchpoints are the only debug exception to write FAR_EL1 */ 535 537 unsigned long far = read_sysreg(far_el1); 536 538 537 539 arm64_enter_el1_dbg(regs); 538 - if (!cortex_a76_erratum_1463225_debug_handler(regs)) 539 - do_debug_exception(far, esr, regs); 540 + debug_exception_enter(regs); 541 + do_watchpoint(far, esr, regs); 542 + debug_exception_exit(regs); 543 + arm64_exit_el1_dbg(regs); 544 + } 545 + 546 + static void noinstr el1_brk64(struct pt_regs *regs, unsigned long esr) 547 + { 548 + arm64_enter_el1_dbg(regs); 549 + debug_exception_enter(regs); 550 + do_el1_brk64(esr, regs); 551 + debug_exception_exit(regs); 540 552 arm64_exit_el1_dbg(regs); 541 553 } 542 554 ··· 617 553 el1_mops(regs, esr); 618 554 break; 619 555 case ESR_ELx_EC_BREAKPT_CUR: 556 + el1_breakpt(regs, esr); 557 + break; 620 558 case ESR_ELx_EC_SOFTSTP_CUR: 559 + el1_softstp(regs, esr); 560 + break; 621 561 case ESR_ELx_EC_WATCHPT_CUR: 562 + el1_watchpt(regs, esr); 563 + break; 622 564 case ESR_ELx_EC_BRK64: 623 - el1_dbg(regs, esr); 565 + el1_brk64(regs, esr); 624 566 break; 625 567 case ESR_ELx_EC_FPAC: 626 568 el1_fpac(regs, esr); ··· 817 747 exit_to_user_mode(regs); 818 748 } 819 749 820 - static void noinstr el0_dbg(struct pt_regs *regs, unsigned long esr) 750 + static void noinstr el0_breakpt(struct pt_regs *regs, unsigned long esr) 821 751 { 822 - /* Only watchpoints write FAR_EL1, otherwise its UNKNOWN */ 752 + if (!is_ttbr0_addr(regs->pc)) 753 + arm64_apply_bp_hardening(); 754 + 755 + enter_from_user_mode(regs); 756 + debug_exception_enter(regs); 757 + do_breakpoint(esr, regs); 758 + debug_exception_exit(regs); 759 + local_daif_restore(DAIF_PROCCTX); 760 + exit_to_user_mode(regs); 761 + } 762 + 763 + static void noinstr el0_softstp(struct pt_regs *regs, unsigned long esr) 764 + { 765 + if (!is_ttbr0_addr(regs->pc)) 766 + arm64_apply_bp_hardening(); 767 + 768 + enter_from_user_mode(regs); 769 + /* 770 + * After handling a breakpoint, we suspend the breakpoint 771 + * and use single-step to move to the next instruction. 772 + * If we are stepping a suspended breakpoint there's nothing more to do: 773 + * the single-step is complete. 774 + */ 775 + if (!try_step_suspended_breakpoints(regs)) { 776 + local_daif_restore(DAIF_PROCCTX); 777 + do_el0_softstep(esr, regs); 778 + } 779 + exit_to_user_mode(regs); 780 + } 781 + 782 + static void noinstr el0_watchpt(struct pt_regs *regs, unsigned long esr) 783 + { 784 + /* Watchpoints are the only debug exception to write FAR_EL1 */ 823 785 unsigned long far = read_sysreg(far_el1); 824 786 825 787 enter_from_user_mode(regs); 826 - do_debug_exception(far, esr, regs); 788 + debug_exception_enter(regs); 789 + do_watchpoint(far, esr, regs); 790 + debug_exception_exit(regs); 827 791 local_daif_restore(DAIF_PROCCTX); 792 + exit_to_user_mode(regs); 793 + } 794 + 795 + static void noinstr el0_brk64(struct pt_regs *regs, unsigned long esr) 796 + { 797 + enter_from_user_mode(regs); 798 + local_daif_restore(DAIF_PROCCTX); 799 + do_el0_brk64(esr, regs); 828 800 exit_to_user_mode(regs); 829 801 } 830 802 ··· 938 826 el0_gcs(regs, esr); 939 827 break; 940 828 case ESR_ELx_EC_BREAKPT_LOW: 829 + el0_breakpt(regs, esr); 830 + break; 941 831 case ESR_ELx_EC_SOFTSTP_LOW: 832 + el0_softstp(regs, esr); 833 + break; 942 834 case ESR_ELx_EC_WATCHPT_LOW: 835 + el0_watchpt(regs, esr); 836 + break; 943 837 case ESR_ELx_EC_BRK64: 944 - el0_dbg(regs, esr); 838 + el0_brk64(regs, esr); 945 839 break; 946 840 case ESR_ELx_EC_FPAC: 947 841 el0_fpac(regs, esr); ··· 1030 912 exit_to_user_mode(regs); 1031 913 } 1032 914 915 + static void noinstr el0_bkpt32(struct pt_regs *regs, unsigned long esr) 916 + { 917 + enter_from_user_mode(regs); 918 + local_daif_restore(DAIF_PROCCTX); 919 + do_bkpt32(esr, regs); 920 + exit_to_user_mode(regs); 921 + } 922 + 1033 923 asmlinkage void noinstr el0t_32_sync_handler(struct pt_regs *regs) 1034 924 { 1035 925 unsigned long esr = read_sysreg(esr_el1); ··· 1072 946 el0_cp15(regs, esr); 1073 947 break; 1074 948 case ESR_ELx_EC_BREAKPT_LOW: 949 + el0_breakpt(regs, esr); 950 + break; 1075 951 case ESR_ELx_EC_SOFTSTP_LOW: 952 + el0_softstp(regs, esr); 953 + break; 1076 954 case ESR_ELx_EC_WATCHPT_LOW: 955 + el0_watchpt(regs, esr); 956 + break; 1077 957 case ESR_ELx_EC_BKPT32: 1078 - el0_dbg(regs, esr); 958 + el0_bkpt32(regs, esr); 1079 959 break; 1080 960 default: 1081 961 el0_inv(regs, esr); ··· 1109 977 UNHANDLED(el0t, 32, error) 1110 978 #endif /* CONFIG_COMPAT */ 1111 979 1112 - #ifdef CONFIG_VMAP_STACK 1113 980 asmlinkage void noinstr __noreturn handle_bad_stack(struct pt_regs *regs) 1114 981 { 1115 982 unsigned long esr = read_sysreg(esr_el1); ··· 1117 986 arm64_enter_nmi(regs); 1118 987 panic_bad_stack(regs, esr, far); 1119 988 } 1120 - #endif /* CONFIG_VMAP_STACK */ 1121 989 1122 990 #ifdef CONFIG_ARM_SDE_INTERFACE 1123 991 asmlinkage noinstr unsigned long

-6

arch/arm64/kernel/entry.S

··· 55 55 .endif 56 56 57 57 sub sp, sp, #PT_REGS_SIZE 58 - #ifdef CONFIG_VMAP_STACK 59 58 /* 60 59 * Test whether the SP has overflowed, without corrupting a GPR. 61 60 * Task and IRQ stacks are aligned so that SP & (1 << THREAD_SHIFT) ··· 96 97 /* We were already on the overflow stack. Restore sp/x0 and carry on. */ 97 98 sub sp, sp, x0 98 99 mrs x0, tpidrro_el0 99 - #endif 100 100 b el\el\ht\()_\regsize\()_\label 101 101 .org .Lventry_start\@ + 128 // Did we overflow the ventry slot? 102 102 .endm ··· 538 540 kernel_ventry 0, t, 32, error // Error 32-bit EL0 539 541 SYM_CODE_END(vectors) 540 542 541 - #ifdef CONFIG_VMAP_STACK 542 543 SYM_CODE_START_LOCAL(__bad_stack) 543 544 /* 544 545 * We detected an overflow in kernel_ventry, which switched to the ··· 565 568 bl handle_bad_stack 566 569 ASM_BUG() 567 570 SYM_CODE_END(__bad_stack) 568 - #endif /* CONFIG_VMAP_STACK */ 569 571 570 572 571 573 .macro entry_handler el:req, ht:req, regsize:req, label:req ··· 1005 1009 1: adr_this_cpu dst=x5, sym=sdei_active_critical_event, tmp=x6 1006 1010 2: str x19, [x5] 1007 1011 1008 - #ifdef CONFIG_VMAP_STACK 1009 1012 /* 1010 1013 * entry.S may have been using sp as a scratch register, find whether 1011 1014 * this is a normal or critical event and switch to the appropriate ··· 1017 1022 2: mov x6, #SDEI_STACK_SIZE 1018 1023 add x5, x5, x6 1019 1024 mov sp, x5 1020 - #endif 1021 1025 1022 1026 #ifdef CONFIG_SHADOW_CALL_STACK 1023 1027 /* Use a separate shadow call stack for normal and critical events */

+24 -36

arch/arm64/kernel/hw_breakpoint.c

··· 22 22 #include <asm/current.h> 23 23 #include <asm/debug-monitors.h> 24 24 #include <asm/esr.h> 25 + #include <asm/exception.h> 25 26 #include <asm/hw_breakpoint.h> 26 27 #include <asm/traps.h> 27 28 #include <asm/cputype.h> ··· 619 618 /* 620 619 * Debug exception handlers. 621 620 */ 622 - static int breakpoint_handler(unsigned long unused, unsigned long esr, 623 - struct pt_regs *regs) 621 + void do_breakpoint(unsigned long esr, struct pt_regs *regs) 624 622 { 625 623 int i, step = 0, *kernel_step; 626 624 u32 ctrl_reg; ··· 662 662 } 663 663 664 664 if (!step) 665 - return 0; 665 + return; 666 666 667 667 if (user_mode(regs)) { 668 668 debug_info->bps_disabled = 1; ··· 670 670 671 671 /* If we're already stepping a watchpoint, just return. */ 672 672 if (debug_info->wps_disabled) 673 - return 0; 673 + return; 674 674 675 675 if (test_thread_flag(TIF_SINGLESTEP)) 676 676 debug_info->suspended_step = 1; ··· 681 681 kernel_step = this_cpu_ptr(&stepping_kernel_bp); 682 682 683 683 if (*kernel_step != ARM_KERNEL_STEP_NONE) 684 - return 0; 684 + return; 685 685 686 686 if (kernel_active_single_step()) { 687 687 *kernel_step = ARM_KERNEL_STEP_SUSPEND; ··· 690 690 kernel_enable_single_step(regs); 691 691 } 692 692 } 693 - 694 - return 0; 695 693 } 696 - NOKPROBE_SYMBOL(breakpoint_handler); 694 + NOKPROBE_SYMBOL(do_breakpoint); 697 695 698 696 /* 699 697 * Arm64 hardware does not always report a watchpoint hit address that matches ··· 750 752 return step; 751 753 } 752 754 753 - static int watchpoint_handler(unsigned long addr, unsigned long esr, 754 - struct pt_regs *regs) 755 + void do_watchpoint(unsigned long addr, unsigned long esr, struct pt_regs *regs) 755 756 { 756 757 int i, step = 0, *kernel_step, access, closest_match = 0; 757 758 u64 min_dist = -1, dist; ··· 805 808 rcu_read_unlock(); 806 809 807 810 if (!step) 808 - return 0; 811 + return; 809 812 810 813 /* 811 814 * We always disable EL0 watchpoints because the kernel can ··· 818 821 819 822 /* If we're already stepping a breakpoint, just return. */ 820 823 if (debug_info->bps_disabled) 821 - return 0; 824 + return; 822 825 823 826 if (test_thread_flag(TIF_SINGLESTEP)) 824 827 debug_info->suspended_step = 1; ··· 829 832 kernel_step = this_cpu_ptr(&stepping_kernel_bp); 830 833 831 834 if (*kernel_step != ARM_KERNEL_STEP_NONE) 832 - return 0; 835 + return; 833 836 834 837 if (kernel_active_single_step()) { 835 838 *kernel_step = ARM_KERNEL_STEP_SUSPEND; ··· 838 841 kernel_enable_single_step(regs); 839 842 } 840 843 } 841 - 842 - return 0; 843 844 } 844 - NOKPROBE_SYMBOL(watchpoint_handler); 845 + NOKPROBE_SYMBOL(do_watchpoint); 845 846 846 847 /* 847 848 * Handle single-step exception. 848 849 */ 849 - int reinstall_suspended_bps(struct pt_regs *regs) 850 + bool try_step_suspended_breakpoints(struct pt_regs *regs) 850 851 { 851 852 struct debug_info *debug_info = &current->thread.debug; 852 - int handled_exception = 0, *kernel_step; 853 - 854 - kernel_step = this_cpu_ptr(&stepping_kernel_bp); 853 + int *kernel_step = this_cpu_ptr(&stepping_kernel_bp); 854 + bool handled_exception = false; 855 855 856 856 /* 857 - * Called from single-step exception handler. 858 - * Return 0 if execution can resume, 1 if a SIGTRAP should be 859 - * reported. 857 + * Called from single-step exception entry. 858 + * Return true if we stepped a breakpoint and can resume execution, 859 + * false if we need to handle a single-step. 860 860 */ 861 861 if (user_mode(regs)) { 862 862 if (debug_info->bps_disabled) { 863 863 debug_info->bps_disabled = 0; 864 864 toggle_bp_registers(AARCH64_DBG_REG_BCR, DBG_ACTIVE_EL0, 1); 865 - handled_exception = 1; 865 + handled_exception = true; 866 866 } 867 867 868 868 if (debug_info->wps_disabled) { 869 869 debug_info->wps_disabled = 0; 870 870 toggle_bp_registers(AARCH64_DBG_REG_WCR, DBG_ACTIVE_EL0, 1); 871 - handled_exception = 1; 871 + handled_exception = true; 872 872 } 873 873 874 874 if (handled_exception) { 875 875 if (debug_info->suspended_step) { 876 876 debug_info->suspended_step = 0; 877 877 /* Allow exception handling to fall-through. */ 878 - handled_exception = 0; 878 + handled_exception = false; 879 879 } else { 880 880 user_disable_single_step(current); 881 881 } ··· 886 892 887 893 if (*kernel_step != ARM_KERNEL_STEP_SUSPEND) { 888 894 kernel_disable_single_step(); 889 - handled_exception = 1; 895 + handled_exception = true; 890 896 } else { 891 - handled_exception = 0; 897 + handled_exception = false; 892 898 } 893 899 894 900 *kernel_step = ARM_KERNEL_STEP_NONE; 895 901 } 896 902 897 - return !handled_exception; 903 + return handled_exception; 898 904 } 899 - NOKPROBE_SYMBOL(reinstall_suspended_bps); 905 + NOKPROBE_SYMBOL(try_step_suspended_breakpoints); 900 906 901 907 /* 902 908 * Context-switcher for restoring suspended breakpoints. ··· 980 986 981 987 pr_info("found %d breakpoint and %d watchpoint registers.\n", 982 988 core_num_brps, core_num_wrps); 983 - 984 - /* Register debug fault handlers. */ 985 - hook_debug_fault_code(DBG_ESR_EVT_HWBP, breakpoint_handler, SIGTRAP, 986 - TRAP_HWBKPT, "hw-breakpoint handler"); 987 - hook_debug_fault_code(DBG_ESR_EVT_HWWP, watchpoint_handler, SIGTRAP, 988 - TRAP_HWBKPT, "hw-watchpoint handler"); 989 989 990 990 /* 991 991 * Reset the breakpoint resources. We assume that a halting

-13

arch/arm64/kernel/irq.c

··· 51 51 scs_alloc(early_cpu_to_node(cpu)); 52 52 } 53 53 54 - #ifdef CONFIG_VMAP_STACK 55 54 static void __init init_irq_stacks(void) 56 55 { 57 56 int cpu; ··· 61 62 per_cpu(irq_stack_ptr, cpu) = p; 62 63 } 63 64 } 64 - #else 65 - /* irq stack only needs to be 16 byte aligned - not IRQ_STACK_SIZE aligned. */ 66 - DEFINE_PER_CPU_ALIGNED(unsigned long [IRQ_STACK_SIZE/sizeof(long)], irq_stack); 67 - 68 - static void init_irq_stacks(void) 69 - { 70 - int cpu; 71 - 72 - for_each_possible_cpu(cpu) 73 - per_cpu(irq_stack_ptr, cpu) = per_cpu(irq_stack, cpu); 74 - } 75 - #endif 76 65 77 66 #ifndef CONFIG_PREEMPT_RT 78 67 static void ____do_softirq(struct pt_regs *regs)

+7 -32

arch/arm64/kernel/kgdb.c

··· 234 234 return err; 235 235 } 236 236 237 - static int kgdb_brk_fn(struct pt_regs *regs, unsigned long esr) 237 + int kgdb_brk_handler(struct pt_regs *regs, unsigned long esr) 238 238 { 239 239 kgdb_handle_exception(1, SIGTRAP, 0, regs); 240 240 return DBG_HOOK_HANDLED; 241 241 } 242 - NOKPROBE_SYMBOL(kgdb_brk_fn) 242 + NOKPROBE_SYMBOL(kgdb_brk_handler) 243 243 244 - static int kgdb_compiled_brk_fn(struct pt_regs *regs, unsigned long esr) 244 + int kgdb_compiled_brk_handler(struct pt_regs *regs, unsigned long esr) 245 245 { 246 246 compiled_break = 1; 247 247 kgdb_handle_exception(1, SIGTRAP, 0, regs); 248 248 249 249 return DBG_HOOK_HANDLED; 250 250 } 251 - NOKPROBE_SYMBOL(kgdb_compiled_brk_fn); 251 + NOKPROBE_SYMBOL(kgdb_compiled_brk_handler); 252 252 253 - static int kgdb_step_brk_fn(struct pt_regs *regs, unsigned long esr) 253 + int kgdb_single_step_handler(struct pt_regs *regs, unsigned long esr) 254 254 { 255 255 if (!kgdb_single_step) 256 256 return DBG_HOOK_ERROR; ··· 258 258 kgdb_handle_exception(0, SIGTRAP, 0, regs); 259 259 return DBG_HOOK_HANDLED; 260 260 } 261 - NOKPROBE_SYMBOL(kgdb_step_brk_fn); 262 - 263 - static struct break_hook kgdb_brkpt_hook = { 264 - .fn = kgdb_brk_fn, 265 - .imm = KGDB_DYN_DBG_BRK_IMM, 266 - }; 267 - 268 - static struct break_hook kgdb_compiled_brkpt_hook = { 269 - .fn = kgdb_compiled_brk_fn, 270 - .imm = KGDB_COMPILED_DBG_BRK_IMM, 271 - }; 272 - 273 - static struct step_hook kgdb_step_hook = { 274 - .fn = kgdb_step_brk_fn 275 - }; 261 + NOKPROBE_SYMBOL(kgdb_single_step_handler); 276 262 277 263 static int __kgdb_notify(struct die_args *args, unsigned long cmd) 278 264 { ··· 297 311 */ 298 312 int kgdb_arch_init(void) 299 313 { 300 - int ret = register_die_notifier(&kgdb_notifier); 301 - 302 - if (ret != 0) 303 - return ret; 304 - 305 - register_kernel_break_hook(&kgdb_brkpt_hook); 306 - register_kernel_break_hook(&kgdb_compiled_brkpt_hook); 307 - register_kernel_step_hook(&kgdb_step_hook); 308 - return 0; 314 + return register_die_notifier(&kgdb_notifier); 309 315 } 310 316 311 317 /* ··· 307 329 */ 308 330 void kgdb_arch_exit(void) 309 331 { 310 - unregister_kernel_break_hook(&kgdb_brkpt_hook); 311 - unregister_kernel_break_hook(&kgdb_compiled_brkpt_hook); 312 - unregister_kernel_step_hook(&kgdb_step_hook); 313 332 unregister_die_notifier(&kgdb_notifier); 314 333 } 315 334

+57 -44

arch/arm64/kernel/module.c

··· 23 23 #include <asm/insn.h> 24 24 #include <asm/scs.h> 25 25 #include <asm/sections.h> 26 + #include <asm/text-patching.h> 26 27 27 28 enum aarch64_reloc_op { 28 29 RELOC_OP_NONE, ··· 49 48 return 0; 50 49 } 51 50 52 - static int reloc_data(enum aarch64_reloc_op op, void *place, u64 val, int len) 51 + #define WRITE_PLACE(place, val, mod) do { \ 52 + __typeof__(val) __val = (val); \ 53 + \ 54 + if (mod->state == MODULE_STATE_UNFORMED) \ 55 + *(place) = __val; \ 56 + else \ 57 + aarch64_insn_copy(place, &(__val), sizeof(*place)); \ 58 + } while (0) 59 + 60 + static int reloc_data(enum aarch64_reloc_op op, void *place, u64 val, int len, 61 + struct module *me) 53 62 { 54 63 s64 sval = do_reloc(op, place, val); 55 64 ··· 77 66 78 67 switch (len) { 79 68 case 16: 80 - *(s16 *)place = sval; 69 + WRITE_PLACE((s16 *)place, sval, me); 81 70 switch (op) { 82 71 case RELOC_OP_ABS: 83 72 if (sval < 0 || sval > U16_MAX) ··· 93 82 } 94 83 break; 95 84 case 32: 96 - *(s32 *)place = sval; 85 + WRITE_PLACE((s32 *)place, sval, me); 97 86 switch (op) { 98 87 case RELOC_OP_ABS: 99 88 if (sval < 0 || sval > U32_MAX) ··· 109 98 } 110 99 break; 111 100 case 64: 112 - *(s64 *)place = sval; 101 + WRITE_PLACE((s64 *)place, sval, me); 113 102 break; 114 103 default: 115 104 pr_err("Invalid length (%d) for data relocation\n", len); ··· 124 113 }; 125 114 126 115 static int reloc_insn_movw(enum aarch64_reloc_op op, __le32 *place, u64 val, 127 - int lsb, enum aarch64_insn_movw_imm_type imm_type) 116 + int lsb, enum aarch64_insn_movw_imm_type imm_type, 117 + struct module *me) 128 118 { 129 119 u64 imm; 130 120 s64 sval; ··· 157 145 158 146 /* Update the instruction with the new encoding. */ 159 147 insn = aarch64_insn_encode_immediate(AARCH64_INSN_IMM_16, insn, imm); 160 - *place = cpu_to_le32(insn); 148 + WRITE_PLACE(place, cpu_to_le32(insn), me); 161 149 162 150 if (imm > U16_MAX) 163 151 return -ERANGE; ··· 166 154 } 167 155 168 156 static int reloc_insn_imm(enum aarch64_reloc_op op, __le32 *place, u64 val, 169 - int lsb, int len, enum aarch64_insn_imm_type imm_type) 157 + int lsb, int len, enum aarch64_insn_imm_type imm_type, 158 + struct module *me) 170 159 { 171 160 u64 imm, imm_mask; 172 161 s64 sval; ··· 183 170 184 171 /* Update the instruction's immediate field. */ 185 172 insn = aarch64_insn_encode_immediate(imm_type, insn, imm); 186 - *place = cpu_to_le32(insn); 173 + WRITE_PLACE(place, cpu_to_le32(insn), me); 187 174 188 175 /* 189 176 * Extract the upper value bits (including the sign bit) and ··· 202 189 } 203 190 204 191 static int reloc_insn_adrp(struct module *mod, Elf64_Shdr *sechdrs, 205 - __le32 *place, u64 val) 192 + __le32 *place, u64 val, struct module *me) 206 193 { 207 194 u32 insn; 208 195 209 196 if (!is_forbidden_offset_for_adrp(place)) 210 197 return reloc_insn_imm(RELOC_OP_PAGE, place, val, 12, 21, 211 - AARCH64_INSN_IMM_ADR); 198 + AARCH64_INSN_IMM_ADR, me); 212 199 213 200 /* patch ADRP to ADR if it is in range */ 214 201 if (!reloc_insn_imm(RELOC_OP_PREL, place, val & ~0xfff, 0, 21, 215 - AARCH64_INSN_IMM_ADR)) { 202 + AARCH64_INSN_IMM_ADR, me)) { 216 203 insn = le32_to_cpu(*place); 217 204 insn &= ~BIT(31); 218 205 } else { ··· 224 211 AARCH64_INSN_BRANCH_NOLINK); 225 212 } 226 213 227 - *place = cpu_to_le32(insn); 214 + WRITE_PLACE(place, cpu_to_le32(insn), me); 228 215 return 0; 229 216 } 230 217 ··· 268 255 /* Data relocations. */ 269 256 case R_AARCH64_ABS64: 270 257 overflow_check = false; 271 - ovf = reloc_data(RELOC_OP_ABS, loc, val, 64); 258 + ovf = reloc_data(RELOC_OP_ABS, loc, val, 64, me); 272 259 break; 273 260 case R_AARCH64_ABS32: 274 - ovf = reloc_data(RELOC_OP_ABS, loc, val, 32); 261 + ovf = reloc_data(RELOC_OP_ABS, loc, val, 32, me); 275 262 break; 276 263 case R_AARCH64_ABS16: 277 - ovf = reloc_data(RELOC_OP_ABS, loc, val, 16); 264 + ovf = reloc_data(RELOC_OP_ABS, loc, val, 16, me); 278 265 break; 279 266 case R_AARCH64_PREL64: 280 267 overflow_check = false; 281 - ovf = reloc_data(RELOC_OP_PREL, loc, val, 64); 268 + ovf = reloc_data(RELOC_OP_PREL, loc, val, 64, me); 282 269 break; 283 270 case R_AARCH64_PREL32: 284 - ovf = reloc_data(RELOC_OP_PREL, loc, val, 32); 271 + ovf = reloc_data(RELOC_OP_PREL, loc, val, 32, me); 285 272 break; 286 273 case R_AARCH64_PREL16: 287 - ovf = reloc_data(RELOC_OP_PREL, loc, val, 16); 274 + ovf = reloc_data(RELOC_OP_PREL, loc, val, 16, me); 288 275 break; 289 276 290 277 /* MOVW instruction relocations. */ ··· 293 280 fallthrough; 294 281 case R_AARCH64_MOVW_UABS_G0: 295 282 ovf = reloc_insn_movw(RELOC_OP_ABS, loc, val, 0, 296 - AARCH64_INSN_IMM_MOVKZ); 283 + AARCH64_INSN_IMM_MOVKZ, me); 297 284 break; 298 285 case R_AARCH64_MOVW_UABS_G1_NC: 299 286 overflow_check = false; 300 287 fallthrough; 301 288 case R_AARCH64_MOVW_UABS_G1: 302 289 ovf = reloc_insn_movw(RELOC_OP_ABS, loc, val, 16, 303 - AARCH64_INSN_IMM_MOVKZ); 290 + AARCH64_INSN_IMM_MOVKZ, me); 304 291 break; 305 292 case R_AARCH64_MOVW_UABS_G2_NC: 306 293 overflow_check = false; 307 294 fallthrough; 308 295 case R_AARCH64_MOVW_UABS_G2: 309 296 ovf = reloc_insn_movw(RELOC_OP_ABS, loc, val, 32, 310 - AARCH64_INSN_IMM_MOVKZ); 297 + AARCH64_INSN_IMM_MOVKZ, me); 311 298 break; 312 299 case R_AARCH64_MOVW_UABS_G3: 313 300 /* We're using the top bits so we can't overflow. */ 314 301 overflow_check = false; 315 302 ovf = reloc_insn_movw(RELOC_OP_ABS, loc, val, 48, 316 - AARCH64_INSN_IMM_MOVKZ); 303 + AARCH64_INSN_IMM_MOVKZ, me); 317 304 break; 318 305 case R_AARCH64_MOVW_SABS_G0: 319 306 ovf = reloc_insn_movw(RELOC_OP_ABS, loc, val, 0, 320 - AARCH64_INSN_IMM_MOVNZ); 307 + AARCH64_INSN_IMM_MOVNZ, me); 321 308 break; 322 309 case R_AARCH64_MOVW_SABS_G1: 323 310 ovf = reloc_insn_movw(RELOC_OP_ABS, loc, val, 16, 324 - AARCH64_INSN_IMM_MOVNZ); 311 + AARCH64_INSN_IMM_MOVNZ, me); 325 312 break; 326 313 case R_AARCH64_MOVW_SABS_G2: 327 314 ovf = reloc_insn_movw(RELOC_OP_ABS, loc, val, 32, 328 - AARCH64_INSN_IMM_MOVNZ); 315 + AARCH64_INSN_IMM_MOVNZ, me); 329 316 break; 330 317 case R_AARCH64_MOVW_PREL_G0_NC: 331 318 overflow_check = false; 332 319 ovf = reloc_insn_movw(RELOC_OP_PREL, loc, val, 0, 333 - AARCH64_INSN_IMM_MOVKZ); 320 + AARCH64_INSN_IMM_MOVKZ, me); 334 321 break; 335 322 case R_AARCH64_MOVW_PREL_G0: 336 323 ovf = reloc_insn_movw(RELOC_OP_PREL, loc, val, 0, 337 - AARCH64_INSN_IMM_MOVNZ); 324 + AARCH64_INSN_IMM_MOVNZ, me); 338 325 break; 339 326 case R_AARCH64_MOVW_PREL_G1_NC: 340 327 overflow_check = false; 341 328 ovf = reloc_insn_movw(RELOC_OP_PREL, loc, val, 16, 342 - AARCH64_INSN_IMM_MOVKZ); 329 + AARCH64_INSN_IMM_MOVKZ, me); 343 330 break; 344 331 case R_AARCH64_MOVW_PREL_G1: 345 332 ovf = reloc_insn_movw(RELOC_OP_PREL, loc, val, 16, 346 - AARCH64_INSN_IMM_MOVNZ); 333 + AARCH64_INSN_IMM_MOVNZ, me); 347 334 break; 348 335 case R_AARCH64_MOVW_PREL_G2_NC: 349 336 overflow_check = false; 350 337 ovf = reloc_insn_movw(RELOC_OP_PREL, loc, val, 32, 351 - AARCH64_INSN_IMM_MOVKZ); 338 + AARCH64_INSN_IMM_MOVKZ, me); 352 339 break; 353 340 case R_AARCH64_MOVW_PREL_G2: 354 341 ovf = reloc_insn_movw(RELOC_OP_PREL, loc, val, 32, 355 - AARCH64_INSN_IMM_MOVNZ); 342 + AARCH64_INSN_IMM_MOVNZ, me); 356 343 break; 357 344 case R_AARCH64_MOVW_PREL_G3: 358 345 /* We're using the top bits so we can't overflow. */ 359 346 overflow_check = false; 360 347 ovf = reloc_insn_movw(RELOC_OP_PREL, loc, val, 48, 361 - AARCH64_INSN_IMM_MOVNZ); 348 + AARCH64_INSN_IMM_MOVNZ, me); 362 349 break; 363 350 364 351 /* Immediate instruction relocations. */ 365 352 case R_AARCH64_LD_PREL_LO19: 366 353 ovf = reloc_insn_imm(RELOC_OP_PREL, loc, val, 2, 19, 367 - AARCH64_INSN_IMM_19); 354 + AARCH64_INSN_IMM_19, me); 368 355 break; 369 356 case R_AARCH64_ADR_PREL_LO21: 370 357 ovf = reloc_insn_imm(RELOC_OP_PREL, loc, val, 0, 21, 371 - AARCH64_INSN_IMM_ADR); 358 + AARCH64_INSN_IMM_ADR, me); 372 359 break; 373 360 case R_AARCH64_ADR_PREL_PG_HI21_NC: 374 361 overflow_check = false; 375 362 fallthrough; 376 363 case R_AARCH64_ADR_PREL_PG_HI21: 377 - ovf = reloc_insn_adrp(me, sechdrs, loc, val); 364 + ovf = reloc_insn_adrp(me, sechdrs, loc, val, me); 378 365 if (ovf && ovf != -ERANGE) 379 366 return ovf; 380 367 break; ··· 382 369 case R_AARCH64_LDST8_ABS_LO12_NC: 383 370 overflow_check = false; 384 371 ovf = reloc_insn_imm(RELOC_OP_ABS, loc, val, 0, 12, 385 - AARCH64_INSN_IMM_12); 372 + AARCH64_INSN_IMM_12, me); 386 373 break; 387 374 case R_AARCH64_LDST16_ABS_LO12_NC: 388 375 overflow_check = false; 389 376 ovf = reloc_insn_imm(RELOC_OP_ABS, loc, val, 1, 11, 390 - AARCH64_INSN_IMM_12); 377 + AARCH64_INSN_IMM_12, me); 391 378 break; 392 379 case R_AARCH64_LDST32_ABS_LO12_NC: 393 380 overflow_check = false; 394 381 ovf = reloc_insn_imm(RELOC_OP_ABS, loc, val, 2, 10, 395 - AARCH64_INSN_IMM_12); 382 + AARCH64_INSN_IMM_12, me); 396 383 break; 397 384 case R_AARCH64_LDST64_ABS_LO12_NC: 398 385 overflow_check = false; 399 386 ovf = reloc_insn_imm(RELOC_OP_ABS, loc, val, 3, 9, 400 - AARCH64_INSN_IMM_12); 387 + AARCH64_INSN_IMM_12, me); 401 388 break; 402 389 case R_AARCH64_LDST128_ABS_LO12_NC: 403 390 overflow_check = false; 404 391 ovf = reloc_insn_imm(RELOC_OP_ABS, loc, val, 4, 8, 405 - AARCH64_INSN_IMM_12); 392 + AARCH64_INSN_IMM_12, me); 406 393 break; 407 394 case R_AARCH64_TSTBR14: 408 395 ovf = reloc_insn_imm(RELOC_OP_PREL, loc, val, 2, 14, 409 - AARCH64_INSN_IMM_14); 396 + AARCH64_INSN_IMM_14, me); 410 397 break; 411 398 case R_AARCH64_CONDBR19: 412 399 ovf = reloc_insn_imm(RELOC_OP_PREL, loc, val, 2, 19, 413 - AARCH64_INSN_IMM_19); 400 + AARCH64_INSN_IMM_19, me); 414 401 break; 415 402 case R_AARCH64_JUMP26: 416 403 case R_AARCH64_CALL26: 417 404 ovf = reloc_insn_imm(RELOC_OP_PREL, loc, val, 2, 26, 418 - AARCH64_INSN_IMM_26); 405 + AARCH64_INSN_IMM_26, me); 419 406 if (ovf == -ERANGE) { 420 407 val = module_emit_plt_entry(me, sechdrs, loc, &rel[i], sym); 421 408 if (!val) 422 409 return -ENOEXEC; 423 410 ovf = reloc_insn_imm(RELOC_OP_PREL, loc, val, 2, 424 - 26, AARCH64_INSN_IMM_26); 411 + 26, AARCH64_INSN_IMM_26, me); 425 412 } 426 413 break; 427 414

+10 -1

arch/arm64/kernel/mte.c

··· 200 200 * program requested values go with what was requested. 201 201 */ 202 202 resolved_mte_tcf = (mte_ctrl & pref) ? pref : mte_ctrl; 203 - sctlr &= ~SCTLR_EL1_TCF0_MASK; 203 + sctlr &= ~(SCTLR_EL1_TCF0_MASK | SCTLR_EL1_TCSO0_MASK); 204 204 /* 205 205 * Pick an actual setting. The order in which we check for 206 206 * set bits and map into register values determines our ··· 212 212 sctlr |= SYS_FIELD_PREP_ENUM(SCTLR_EL1, TCF0, ASYNC); 213 213 else if (resolved_mte_tcf & MTE_CTRL_TCF_SYNC) 214 214 sctlr |= SYS_FIELD_PREP_ENUM(SCTLR_EL1, TCF0, SYNC); 215 + 216 + if (mte_ctrl & MTE_CTRL_STORE_ONLY) 217 + sctlr |= SYS_FIELD_PREP(SCTLR_EL1, TCSO0, 1); 218 + 215 219 task->thread.sctlr_user = sctlr; 216 220 } 217 221 ··· 375 371 (arg & PR_MTE_TCF_SYNC)) 376 372 mte_ctrl |= MTE_CTRL_TCF_ASYMM; 377 373 374 + if (arg & PR_MTE_STORE_ONLY) 375 + mte_ctrl |= MTE_CTRL_STORE_ONLY; 376 + 378 377 task->thread.mte_ctrl = mte_ctrl; 379 378 if (task == current) { 380 379 preempt_disable(); ··· 405 398 ret |= PR_MTE_TCF_ASYNC; 406 399 if (mte_ctrl & MTE_CTRL_TCF_SYNC) 407 400 ret |= PR_MTE_TCF_SYNC; 401 + if (mte_ctrl & MTE_CTRL_STORE_ONLY) 402 + ret |= PR_MTE_STORE_ONLY; 408 403 409 404 return ret; 410 405 }

+1 -1

arch/arm64/kernel/pi/Makefile

··· 41 41 obj-$(CONFIG_RELOCATABLE) += relocate.pi.o 42 42 obj-$(CONFIG_RANDOMIZE_BASE) += kaslr_early.pi.o 43 43 obj-$(CONFIG_UNWIND_PATCH_PAC_INTO_SCS) += patch-scs.pi.o 44 - extra-y := $(patsubst %.pi.o,%.o,$(obj-y)) 44 + targets := $(patsubst %.pi.o,%.o,$(obj-y))

+6 -25

arch/arm64/kernel/probes/kprobes.c

··· 292 292 return 0; 293 293 } 294 294 295 - static int __kprobes 296 - kprobe_breakpoint_handler(struct pt_regs *regs, unsigned long esr) 295 + int __kprobes 296 + kprobe_brk_handler(struct pt_regs *regs, unsigned long esr) 297 297 { 298 298 struct kprobe *p, *cur_kprobe; 299 299 struct kprobe_ctlblk *kcb; ··· 336 336 return DBG_HOOK_HANDLED; 337 337 } 338 338 339 - static struct break_hook kprobes_break_hook = { 340 - .imm = KPROBES_BRK_IMM, 341 - .fn = kprobe_breakpoint_handler, 342 - }; 343 - 344 - static int __kprobes 345 - kprobe_breakpoint_ss_handler(struct pt_regs *regs, unsigned long esr) 339 + int __kprobes 340 + kprobe_ss_brk_handler(struct pt_regs *regs, unsigned long esr) 346 341 { 347 342 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); 348 343 unsigned long addr = instruction_pointer(regs); ··· 355 360 return DBG_HOOK_ERROR; 356 361 } 357 362 358 - static struct break_hook kprobes_break_ss_hook = { 359 - .imm = KPROBES_BRK_SS_IMM, 360 - .fn = kprobe_breakpoint_ss_handler, 361 - }; 362 - 363 - static int __kprobes 364 - kretprobe_breakpoint_handler(struct pt_regs *regs, unsigned long esr) 363 + int __kprobes 364 + kretprobe_brk_handler(struct pt_regs *regs, unsigned long esr) 365 365 { 366 366 if (regs->pc != (unsigned long)__kretprobe_trampoline) 367 367 return DBG_HOOK_ERROR; ··· 364 374 regs->pc = kretprobe_trampoline_handler(regs, (void *)regs->regs[29]); 365 375 return DBG_HOOK_HANDLED; 366 376 } 367 - 368 - static struct break_hook kretprobes_break_hook = { 369 - .imm = KRETPROBES_BRK_IMM, 370 - .fn = kretprobe_breakpoint_handler, 371 - }; 372 377 373 378 /* 374 379 * Provide a blacklist of symbols identifying ranges which cannot be kprobed. ··· 407 422 408 423 int __init arch_init_kprobes(void) 409 424 { 410 - register_kernel_break_hook(&kprobes_break_hook); 411 - register_kernel_break_hook(&kprobes_break_ss_hook); 412 - register_kernel_break_hook(&kretprobes_break_hook); 413 - 414 425 return 0; 415 426 }

+1 -1

arch/arm64/kernel/probes/kprobes_trampoline.S

··· 12 12 SYM_CODE_START(__kretprobe_trampoline) 13 13 /* 14 14 * Trigger a breakpoint exception. The PC will be adjusted by 15 - * kretprobe_breakpoint_handler(), and no subsequent instructions will 15 + * kretprobe_brk_handler(), and no subsequent instructions will 16 16 * be executed from the trampoline. 17 17 */ 18 18 brk #KRETPROBES_BRK_IMM

+2 -22

arch/arm64/kernel/probes/uprobes.c

··· 173 173 return NOTIFY_DONE; 174 174 } 175 175 176 - static int uprobe_breakpoint_handler(struct pt_regs *regs, 176 + int uprobe_brk_handler(struct pt_regs *regs, 177 177 unsigned long esr) 178 178 { 179 179 if (uprobe_pre_sstep_notifier(regs)) ··· 182 182 return DBG_HOOK_ERROR; 183 183 } 184 184 185 - static int uprobe_single_step_handler(struct pt_regs *regs, 185 + int uprobe_single_step_handler(struct pt_regs *regs, 186 186 unsigned long esr) 187 187 { 188 188 struct uprobe_task *utask = current->utask; ··· 194 194 return DBG_HOOK_ERROR; 195 195 } 196 196 197 - /* uprobe breakpoint handler hook */ 198 - static struct break_hook uprobes_break_hook = { 199 - .imm = UPROBES_BRK_IMM, 200 - .fn = uprobe_breakpoint_handler, 201 - }; 202 - 203 - /* uprobe single step handler hook */ 204 - static struct step_hook uprobes_step_hook = { 205 - .fn = uprobe_single_step_handler, 206 - }; 207 - 208 - static int __init arch_init_uprobes(void) 209 - { 210 - register_user_break_hook(&uprobes_break_hook); 211 - register_user_step_hook(&uprobes_step_hook); 212 - 213 - return 0; 214 - } 215 - 216 - device_initcall(arch_init_uprobes);

+8 -5

arch/arm64/kernel/process.c

··· 307 307 p->thread.gcs_base = 0; 308 308 p->thread.gcs_size = 0; 309 309 310 + p->thread.gcs_el0_mode = current->thread.gcs_el0_mode; 311 + p->thread.gcs_el0_locked = current->thread.gcs_el0_locked; 312 + 310 313 gcs = gcs_alloc_thread_stack(p, args); 311 314 if (IS_ERR_VALUE(gcs)) 312 315 return PTR_ERR((void *)gcs); 313 - 314 - p->thread.gcs_el0_mode = current->thread.gcs_el0_mode; 315 - p->thread.gcs_el0_locked = current->thread.gcs_el0_locked; 316 316 317 317 return 0; 318 318 } ··· 341 341 void arch_release_task_struct(struct task_struct *tsk) 342 342 { 343 343 fpsimd_release_task(tsk); 344 - gcs_free(tsk); 345 344 } 346 345 347 346 int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src) ··· 855 856 if (is_compat_thread(ti)) 856 857 return -EINVAL; 857 858 858 - if (system_supports_mte()) 859 + if (system_supports_mte()) { 859 860 valid_mask |= PR_MTE_TCF_SYNC | PR_MTE_TCF_ASYNC \ 860 861 | PR_MTE_TAG_MASK; 862 + 863 + if (cpus_have_cap(ARM64_MTE_STORE_ONLY)) 864 + valid_mask |= PR_MTE_STORE_ONLY; 865 + } 861 866 862 867 if (arg & ~valid_mask) 863 868 return -EINVAL;

+2 -6

arch/arm64/kernel/sdei.c

··· 34 34 DECLARE_PER_CPU(unsigned long *, sdei_stack_normal_ptr); 35 35 DECLARE_PER_CPU(unsigned long *, sdei_stack_critical_ptr); 36 36 37 - #ifdef CONFIG_VMAP_STACK 38 37 DEFINE_PER_CPU(unsigned long *, sdei_stack_normal_ptr); 39 38 DEFINE_PER_CPU(unsigned long *, sdei_stack_critical_ptr); 40 - #endif 41 39 42 40 DECLARE_PER_CPU(unsigned long *, sdei_shadow_call_stack_normal_ptr); 43 41 DECLARE_PER_CPU(unsigned long *, sdei_shadow_call_stack_critical_ptr); ··· 63 65 { 64 66 int cpu; 65 67 66 - if (!IS_ENABLED(CONFIG_VMAP_STACK)) 67 - return; 68 + BUILD_BUG_ON(!IS_ENABLED(CONFIG_VMAP_STACK)); 68 69 69 70 for_each_possible_cpu(cpu) { 70 71 _free_sdei_stack(&sdei_stack_normal_ptr, cpu); ··· 88 91 int cpu; 89 92 int err = 0; 90 93 91 - if (!IS_ENABLED(CONFIG_VMAP_STACK)) 92 - return 0; 94 + BUILD_BUG_ON(!IS_ENABLED(CONFIG_VMAP_STACK)); 93 95 94 96 for_each_possible_cpu(cpu) { 95 97 err = _init_sdei_stack(&sdei_stack_normal_ptr, cpu);

+5 -2

arch/arm64/kernel/signal.c

··· 95 95 96 96 ua_state->por_el0 = read_sysreg_s(SYS_POR_EL0); 97 97 write_sysreg_s(por_enable_all, SYS_POR_EL0); 98 - /* Ensure that any subsequent uaccess observes the updated value */ 99 - isb(); 98 + /* 99 + * No ISB required as we can tolerate spurious Overlay faults - 100 + * the fault handler will check again based on the new value 101 + * of POR_EL0. 102 + */ 100 103 } 101 104 } 102 105

+46 -13

arch/arm64/kernel/stacktrace.c

··· 152 152 orig_pc = kretprobe_find_ret_addr(state->task, 153 153 (void *)state->common.fp, 154 154 &state->kr_cur); 155 + if (!orig_pc) 156 + return -EINVAL; 155 157 state->common.pc = orig_pc; 156 158 state->flags.kretprobe = 1; 157 159 } ··· 279 277 280 278 typedef bool (*kunwind_consume_fn)(const struct kunwind_state *state, void *cookie); 281 279 282 - static __always_inline void 280 + static __always_inline int 283 281 do_kunwind(struct kunwind_state *state, kunwind_consume_fn consume_state, 284 282 void *cookie) 285 283 { 286 - if (kunwind_recover_return_address(state)) 287 - return; 284 + int ret; 285 + 286 + ret = kunwind_recover_return_address(state); 287 + if (ret) 288 + return ret; 288 289 289 290 while (1) { 290 - int ret; 291 - 292 291 if (!consume_state(state, cookie)) 293 - break; 292 + return -EINVAL; 294 293 ret = kunwind_next(state); 294 + if (ret == -ENOENT) 295 + return 0; 295 296 if (ret < 0) 296 - break; 297 + return ret; 297 298 } 298 299 } 299 300 ··· 329 324 : stackinfo_get_unknown(); \ 330 325 }) 331 326 332 - static __always_inline void 327 + static __always_inline int 333 328 kunwind_stack_walk(kunwind_consume_fn consume_state, 334 329 void *cookie, struct task_struct *task, 335 330 struct pt_regs *regs) ··· 337 332 struct stack_info stacks[] = { 338 333 stackinfo_get_task(task), 339 334 STACKINFO_CPU(irq), 340 - #if defined(CONFIG_VMAP_STACK) 341 335 STACKINFO_CPU(overflow), 342 - #endif 343 - #if defined(CONFIG_VMAP_STACK) && defined(CONFIG_ARM_SDE_INTERFACE) 336 + #if defined(CONFIG_ARM_SDE_INTERFACE) 344 337 STACKINFO_SDEI(normal), 345 338 STACKINFO_SDEI(critical), 346 339 #endif ··· 355 352 356 353 if (regs) { 357 354 if (task != current) 358 - return; 355 + return -EINVAL; 359 356 kunwind_init_from_regs(&state, regs); 360 357 } else if (task == current) { 361 358 kunwind_init_from_caller(&state); ··· 363 360 kunwind_init_from_task(&state, task); 364 361 } 365 362 366 - do_kunwind(&state, consume_state, cookie); 363 + return do_kunwind(&state, consume_state, cookie); 367 364 } 368 365 369 366 struct kunwind_consume_entry_data { ··· 388 385 }; 389 386 390 387 kunwind_stack_walk(arch_kunwind_consume_entry, &data, task, regs); 388 + } 389 + 390 + static __always_inline bool 391 + arch_reliable_kunwind_consume_entry(const struct kunwind_state *state, void *cookie) 392 + { 393 + /* 394 + * At an exception boundary we can reliably consume the saved PC. We do 395 + * not know whether the LR was live when the exception was taken, and 396 + * so we cannot perform the next unwind step reliably. 397 + * 398 + * All that matters is whether the *entire* unwind is reliable, so give 399 + * up as soon as we hit an exception boundary. 400 + */ 401 + if (state->source == KUNWIND_SOURCE_REGS_PC) 402 + return false; 403 + 404 + return arch_kunwind_consume_entry(state, cookie); 405 + } 406 + 407 + noinline noinstr int arch_stack_walk_reliable(stack_trace_consume_fn consume_entry, 408 + void *cookie, 409 + struct task_struct *task) 410 + { 411 + struct kunwind_consume_entry_data data = { 412 + .consume_entry = consume_entry, 413 + .cookie = cookie, 414 + }; 415 + 416 + return kunwind_stack_walk(arch_reliable_kunwind_consume_entry, &data, 417 + task, NULL); 391 418 } 392 419 393 420 struct bpf_unwind_consume_entry_data {

+8 -76

arch/arm64/kernel/traps.c

··· 454 454 u32 insn; 455 455 456 456 /* check for AArch32 breakpoint instructions */ 457 - if (!aarch32_break_handler(regs)) 457 + if (try_handle_aarch32_break(regs)) 458 458 return; 459 459 460 460 if (user_insn_read(regs, &insn)) ··· 894 894 "Bad EL0 synchronous exception"); 895 895 } 896 896 897 - #ifdef CONFIG_VMAP_STACK 898 - 899 897 DEFINE_PER_CPU(unsigned long [OVERFLOW_STACK_SIZE/sizeof(long)], overflow_stack) 900 898 __aligned(16); 901 899 ··· 925 927 nmi_panic(NULL, "kernel stack overflow"); 926 928 cpu_park_loop(); 927 929 } 928 - #endif 929 930 930 931 void __noreturn arm64_serror_panic(struct pt_regs *regs, unsigned long esr) 931 932 { 933 + add_taint(TAINT_MACHINE_CHECK, LOCKDEP_STILL_OK); 932 934 console_verbose(); 933 935 934 936 pr_crit("SError Interrupt on CPU%d, code 0x%016lx -- %s\n", ··· 985 987 int is_valid_bugaddr(unsigned long addr) 986 988 { 987 989 /* 988 - * bug_handler() only called for BRK #BUG_BRK_IMM. 990 + * bug_brk_handler() only called for BRK #BUG_BRK_IMM. 989 991 * So the answer is trivial -- any spurious instances with no 990 992 * bug table entry will be rejected by report_bug() and passed 991 993 * back to the debug-monitors code and handled as a fatal ··· 995 997 } 996 998 #endif 997 999 998 - static int bug_handler(struct pt_regs *regs, unsigned long esr) 1000 + int bug_brk_handler(struct pt_regs *regs, unsigned long esr) 999 1001 { 1000 1002 switch (report_bug(regs->pc, regs)) { 1001 1003 case BUG_TRAP_TYPE_BUG: ··· 1015 1017 return DBG_HOOK_HANDLED; 1016 1018 } 1017 1019 1018 - static struct break_hook bug_break_hook = { 1019 - .fn = bug_handler, 1020 - .imm = BUG_BRK_IMM, 1021 - }; 1022 - 1023 1020 #ifdef CONFIG_CFI_CLANG 1024 - static int cfi_handler(struct pt_regs *regs, unsigned long esr) 1021 + int cfi_brk_handler(struct pt_regs *regs, unsigned long esr) 1025 1022 { 1026 1023 unsigned long target; 1027 1024 u32 type; ··· 1039 1046 arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE); 1040 1047 return DBG_HOOK_HANDLED; 1041 1048 } 1042 - 1043 - static struct break_hook cfi_break_hook = { 1044 - .fn = cfi_handler, 1045 - .imm = CFI_BRK_IMM_BASE, 1046 - .mask = CFI_BRK_IMM_MASK, 1047 - }; 1048 1049 #endif /* CONFIG_CFI_CLANG */ 1049 1050 1050 - static int reserved_fault_handler(struct pt_regs *regs, unsigned long esr) 1051 + int reserved_fault_brk_handler(struct pt_regs *regs, unsigned long esr) 1051 1052 { 1052 1053 pr_err("%s generated an invalid instruction at %pS!\n", 1053 1054 "Kernel text patching", ··· 1051 1064 return DBG_HOOK_ERROR; 1052 1065 } 1053 1066 1054 - static struct break_hook fault_break_hook = { 1055 - .fn = reserved_fault_handler, 1056 - .imm = FAULT_BRK_IMM, 1057 - }; 1058 - 1059 1067 #ifdef CONFIG_KASAN_SW_TAGS 1060 1068 1061 1069 #define KASAN_ESR_RECOVER 0x20 ··· 1058 1076 #define KASAN_ESR_SIZE_MASK 0x0f 1059 1077 #define KASAN_ESR_SIZE(esr) (1 << ((esr) & KASAN_ESR_SIZE_MASK)) 1060 1078 1061 - static int kasan_handler(struct pt_regs *regs, unsigned long esr) 1079 + int kasan_brk_handler(struct pt_regs *regs, unsigned long esr) 1062 1080 { 1063 1081 bool recover = esr & KASAN_ESR_RECOVER; 1064 1082 bool write = esr & KASAN_ESR_WRITE; ··· 1089 1107 arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE); 1090 1108 return DBG_HOOK_HANDLED; 1091 1109 } 1092 - 1093 - static struct break_hook kasan_break_hook = { 1094 - .fn = kasan_handler, 1095 - .imm = KASAN_BRK_IMM, 1096 - .mask = KASAN_BRK_MASK, 1097 - }; 1098 1110 #endif 1099 1111 1100 1112 #ifdef CONFIG_UBSAN_TRAP 1101 - static int ubsan_handler(struct pt_regs *regs, unsigned long esr) 1113 + int ubsan_brk_handler(struct pt_regs *regs, unsigned long esr) 1102 1114 { 1103 1115 die(report_ubsan_failure(esr & UBSAN_BRK_MASK), regs, esr); 1104 1116 return DBG_HOOK_HANDLED; 1105 1117 } 1106 - 1107 - static struct break_hook ubsan_break_hook = { 1108 - .fn = ubsan_handler, 1109 - .imm = UBSAN_BRK_IMM, 1110 - .mask = UBSAN_BRK_MASK, 1111 - }; 1112 1118 #endif 1113 - 1114 - /* 1115 - * Initial handler for AArch64 BRK exceptions 1116 - * This handler only used until debug_traps_init(). 1117 - */ 1118 - int __init early_brk64(unsigned long addr, unsigned long esr, 1119 - struct pt_regs *regs) 1120 - { 1121 - #ifdef CONFIG_CFI_CLANG 1122 - if (esr_is_cfi_brk(esr)) 1123 - return cfi_handler(regs, esr) != DBG_HOOK_HANDLED; 1124 - #endif 1125 - #ifdef CONFIG_KASAN_SW_TAGS 1126 - if ((esr_brk_comment(esr) & ~KASAN_BRK_MASK) == KASAN_BRK_IMM) 1127 - return kasan_handler(regs, esr) != DBG_HOOK_HANDLED; 1128 - #endif 1129 - #ifdef CONFIG_UBSAN_TRAP 1130 - if (esr_is_ubsan_brk(esr)) 1131 - return ubsan_handler(regs, esr) != DBG_HOOK_HANDLED; 1132 - #endif 1133 - return bug_handler(regs, esr) != DBG_HOOK_HANDLED; 1134 - } 1135 - 1136 - void __init trap_init(void) 1137 - { 1138 - register_kernel_break_hook(&bug_break_hook); 1139 - #ifdef CONFIG_CFI_CLANG 1140 - register_kernel_break_hook(&cfi_break_hook); 1141 - #endif 1142 - register_kernel_break_hook(&fault_break_hook); 1143 - #ifdef CONFIG_KASAN_SW_TAGS 1144 - register_kernel_break_hook(&kasan_break_hook); 1145 - #endif 1146 - #ifdef CONFIG_UBSAN_TRAP 1147 - register_kernel_break_hook(&ubsan_break_hook); 1148 - #endif 1149 - debug_traps_init(); 1150 - }

+58

arch/arm64/kernel/watchdog_hld.c

··· 34 34 */ 35 35 return arm_pmu_irq_is_nmi(); 36 36 } 37 + 38 + static int watchdog_perf_update_period(void *data) 39 + { 40 + int cpu = smp_processor_id(); 41 + u64 max_cpu_freq, new_period; 42 + 43 + max_cpu_freq = cpufreq_get_hw_max_freq(cpu) * 1000UL; 44 + if (!max_cpu_freq) 45 + return 0; 46 + 47 + new_period = watchdog_thresh * max_cpu_freq; 48 + hardlockup_detector_perf_adjust_period(new_period); 49 + 50 + return 0; 51 + } 52 + 53 + static int watchdog_freq_notifier_callback(struct notifier_block *nb, 54 + unsigned long val, void *data) 55 + { 56 + struct cpufreq_policy *policy = data; 57 + int cpu; 58 + 59 + if (val != CPUFREQ_CREATE_POLICY) 60 + return NOTIFY_DONE; 61 + 62 + /* 63 + * Let each online CPU related to the policy update the period by their 64 + * own. This will serialize with the framework on start/stop the lockup 65 + * detector (softlockup_{start,stop}_all) and avoid potential race 66 + * condition. Otherwise we may have below theoretical race condition: 67 + * (core 0/1 share the same policy) 68 + * [core 0] [core 1] 69 + * hardlockup_detector_event_create() 70 + * hw_nmi_get_sample_period() 71 + * (cpufreq registered, notifier callback invoked) 72 + * watchdog_freq_notifier_callback() 73 + * watchdog_perf_update_period() 74 + * (since core 1's event's not yet created, 75 + * the period is not set) 76 + * perf_event_create_kernel_counter() 77 + * (event's period is SAFE_MAX_CPU_FREQ) 78 + */ 79 + for_each_cpu(cpu, policy->cpus) 80 + smp_call_on_cpu(cpu, watchdog_perf_update_period, NULL, false); 81 + 82 + return NOTIFY_DONE; 83 + } 84 + 85 + static struct notifier_block watchdog_freq_notifier = { 86 + .notifier_call = watchdog_freq_notifier_callback, 87 + }; 88 + 89 + static int __init init_watchdog_freq_notifier(void) 90 + { 91 + return cpufreq_register_notifier(&watchdog_freq_notifier, 92 + CPUFREQ_POLICY_NOTIFIER); 93 + } 94 + core_initcall(init_watchdog_freq_notifier);

+4

arch/arm64/kvm/debug.c

··· 81 81 !(read_sysreg_s(SYS_PMBIDR_EL1) & PMBIDR_EL1_P)) 82 82 host_data_set_flag(HAS_SPE); 83 83 84 + /* Check if we have BRBE implemented and available at the host */ 85 + if (cpuid_feature_extract_unsigned_field(dfr0, ID_AA64DFR0_EL1_BRBE_SHIFT)) 86 + host_data_set_flag(HAS_BRBE); 87 + 84 88 if (cpuid_feature_extract_unsigned_field(dfr0, ID_AA64DFR0_EL1_TraceFilt_SHIFT)) { 85 89 /* Force disable trace in protected mode in case of no TRBE */ 86 90 if (is_protected_kvm_enabled())

+32

arch/arm64/kvm/hyp/nvhe/debug-sr.c

··· 92 92 *host_data_ptr(host_debug_state.trfcr_el1)); 93 93 } 94 94 95 + static void __debug_save_brbe(u64 *brbcr_el1) 96 + { 97 + *brbcr_el1 = 0; 98 + 99 + /* Check if the BRBE is enabled */ 100 + if (!(read_sysreg_el1(SYS_BRBCR) & (BRBCR_ELx_E0BRE | BRBCR_ELx_ExBRE))) 101 + return; 102 + 103 + /* 104 + * Prohibit branch record generation while we are in guest. 105 + * Since access to BRBCR_EL1 is trapped, the guest can't 106 + * modify the filtering set by the host. 107 + */ 108 + *brbcr_el1 = read_sysreg_el1(SYS_BRBCR); 109 + write_sysreg_el1(0, SYS_BRBCR); 110 + } 111 + 112 + static void __debug_restore_brbe(u64 brbcr_el1) 113 + { 114 + if (!brbcr_el1) 115 + return; 116 + 117 + /* Restore BRBE controls */ 118 + write_sysreg_el1(brbcr_el1, SYS_BRBCR); 119 + } 120 + 95 121 void __debug_save_host_buffers_nvhe(struct kvm_vcpu *vcpu) 96 122 { 97 123 /* Disable and flush SPE data generation */ 98 124 if (host_data_test_flag(HAS_SPE)) 99 125 __debug_save_spe(host_data_ptr(host_debug_state.pmscr_el1)); 126 + 127 + /* Disable BRBE branch records */ 128 + if (host_data_test_flag(HAS_BRBE)) 129 + __debug_save_brbe(host_data_ptr(host_debug_state.brbcr_el1)); 100 130 101 131 if (__trace_needs_switch()) 102 132 __trace_switch_to_guest(); ··· 141 111 { 142 112 if (host_data_test_flag(HAS_SPE)) 143 113 __debug_restore_spe(*host_data_ptr(host_debug_state.pmscr_el1)); 114 + if (host_data_test_flag(HAS_BRBE)) 115 + __debug_restore_brbe(*host_data_ptr(host_debug_state.brbcr_el1)); 144 116 if (__trace_needs_switch()) 145 117 __trace_switch_to_host(); 146 118 }

+1 -1

arch/arm64/kvm/hyp/nvhe/switch.c

··· 272 272 * We're about to restore some new MMU state. Make sure 273 273 * ongoing page-table walks that have started before we 274 274 * trapped to EL2 have completed. This also synchronises the 275 - * above disabling of SPE and TRBE. 275 + * above disabling of BRBE, SPE and TRBE. 276 276 * 277 277 * See DDI0487I.a D8.1.5 "Out-of-context translation regimes", 278 278 * rule R_LFHQG and subsequent information statements.

+8 -3

arch/arm64/kvm/sys_regs.c

··· 1617 1617 val &= ~ARM64_FEATURE_MASK(ID_AA64PFR1_EL1_MPAM_frac); 1618 1618 break; 1619 1619 case SYS_ID_AA64PFR2_EL1: 1620 - /* We only expose FPMR */ 1621 - val &= ID_AA64PFR2_EL1_FPMR; 1620 + val &= ID_AA64PFR2_EL1_FPMR | 1621 + (kvm_has_mte(vcpu->kvm) ? 1622 + ID_AA64PFR2_EL1_MTEFAR | ID_AA64PFR2_EL1_MTESTOREONLY : 1623 + 0); 1622 1624 break; 1623 1625 case SYS_ID_AA64ISAR1_EL1: 1624 1626 if (!vcpu_has_ptrauth(vcpu)) ··· 2880 2878 ID_AA64PFR1_EL1_MPAM_frac | 2881 2879 ID_AA64PFR1_EL1_RAS_frac | 2882 2880 ID_AA64PFR1_EL1_MTE)), 2883 - ID_WRITABLE(ID_AA64PFR2_EL1, ID_AA64PFR2_EL1_FPMR), 2881 + ID_WRITABLE(ID_AA64PFR2_EL1, 2882 + ID_AA64PFR2_EL1_FPMR | 2883 + ID_AA64PFR2_EL1_MTEFAR | 2884 + ID_AA64PFR2_EL1_MTESTOREONLY), 2884 2885 ID_UNALLOCATED(4,3), 2885 2886 ID_WRITABLE(ID_AA64ZFR0_EL1, ~ID_AA64ZFR0_EL1_RES0), 2886 2887 ID_HIDDEN(ID_AA64SMFR0_EL1),

+201 -10

arch/arm64/mm/contpte.c

··· 68 68 pte = pte_mkyoung(pte); 69 69 } 70 70 71 - __flush_tlb_range(&vma, start_addr, addr, PAGE_SIZE, true, 3); 71 + /* 72 + * On eliding the __tlb_flush_range() under BBML2+noabort: 73 + * 74 + * NOTE: Instead of using N=16 as the contiguous block length, we use 75 + * N=4 for clarity. 76 + * 77 + * NOTE: 'n' and 'c' are used to denote the "contiguous bit" being 78 + * unset and set, respectively. 79 + * 80 + * We worry about two cases where contiguous bit is used: 81 + * - When folding N smaller non-contiguous ptes as 1 contiguous block. 82 + * - When unfolding a contiguous block into N smaller non-contiguous ptes. 83 + * 84 + * Currently, the BBML0 folding case looks as follows: 85 + * 86 + * 0) Initial page-table layout: 87 + * 88 + * +----+----+----+----+ 89 + * |RO,n|RO,n|RO,n|RW,n| <--- last page being set as RO 90 + * +----+----+----+----+ 91 + * 92 + * 1) Aggregate AF + dirty flags using __ptep_get_and_clear(): 93 + * 94 + * +----+----+----+----+ 95 + * | 0 | 0 | 0 | 0 | 96 + * +----+----+----+----+ 97 + * 98 + * 2) __flush_tlb_range(): 99 + * 100 + * |____ tlbi + dsb ____| 101 + * 102 + * 3) __set_ptes() to repaint contiguous block: 103 + * 104 + * +----+----+----+----+ 105 + * |RO,c|RO,c|RO,c|RO,c| 106 + * +----+----+----+----+ 107 + * 108 + * 4) The kernel will eventually __flush_tlb() for changed page: 109 + * 110 + * |____| <--- tlbi + dsb 111 + * 112 + * As expected, the intermediate tlbi+dsb ensures that other PEs 113 + * only ever see an invalid (0) entry, or the new contiguous TLB entry. 114 + * The final tlbi+dsb will always throw away the newly installed 115 + * contiguous TLB entry, which is a micro-optimisation opportunity, 116 + * but does not affect correctness. 117 + * 118 + * In the BBML2 case, the change is avoiding the intermediate tlbi+dsb. 119 + * This means a few things, but notably other PEs will still "see" any 120 + * stale cached TLB entries. This could lead to a "contiguous bit 121 + * misprogramming" issue until the final tlbi+dsb of the changed page, 122 + * which would clear out both the stale (RW,n) entry and the new (RO,c) 123 + * contiguous entry installed in its place. 124 + * 125 + * What this is saying, is the following: 126 + * 127 + * +----+----+----+----+ 128 + * |RO,n|RO,n|RO,n|RW,n| <--- old page tables, all non-contiguous 129 + * +----+----+----+----+ 130 + * 131 + * +----+----+----+----+ 132 + * |RO,c|RO,c|RO,c|RO,c| <--- new page tables, all contiguous 133 + * +----+----+----+----+ 134 + * /\ 135 + * || 136 + * 137 + * If both the old single (RW,n) and new contiguous (RO,c) TLB entries 138 + * are present, and a write is made to this address, do we fault or 139 + * is the write permitted (via amalgamation)? 140 + * 141 + * The relevant Arm ARM DDI 0487L.a requirements are RNGLXZ and RJQQTC, 142 + * and together state that when BBML1 or BBML2 are implemented, either 143 + * a TLB conflict abort is raised (which we expressly forbid), or will 144 + * "produce an OA, access permissions, and memory attributes that are 145 + * consistent with any of the programmed translation table values". 146 + * 147 + * That is to say, will either raise a TLB conflict, or produce one of 148 + * the cached TLB entries, but never amalgamate. 149 + * 150 + * Thus, as the page tables are only considered "consistent" after 151 + * the final tlbi+dsb (which evicts both the single stale (RW,n) TLB 152 + * entry as well as the new contiguous (RO,c) TLB entry), omitting the 153 + * initial tlbi+dsb is correct. 154 + * 155 + * It is also important to note that at the end of the BBML2 folding 156 + * case, we are still left with potentially all N TLB entries still 157 + * cached (the N-1 non-contiguous ptes, and the single contiguous 158 + * block). However, over time, natural TLB pressure will cause the 159 + * non-contiguous pte TLB entries to be flushed, leaving only the 160 + * contiguous block TLB entry. This means that omitting the tlbi+dsb is 161 + * not only correct, but also keeps our eventual performance benefits. 162 + * 163 + * For the unfolding case, BBML0 looks as follows: 164 + * 165 + * 0) Initial page-table layout: 166 + * 167 + * +----+----+----+----+ 168 + * |RW,c|RW,c|RW,c|RW,c| <--- last page being set as RO 169 + * +----+----+----+----+ 170 + * 171 + * 1) Aggregate AF + dirty flags using __ptep_get_and_clear(): 172 + * 173 + * +----+----+----+----+ 174 + * | 0 | 0 | 0 | 0 | 175 + * +----+----+----+----+ 176 + * 177 + * 2) __flush_tlb_range(): 178 + * 179 + * |____ tlbi + dsb ____| 180 + * 181 + * 3) __set_ptes() to repaint as non-contiguous: 182 + * 183 + * +----+----+----+----+ 184 + * |RW,n|RW,n|RW,n|RW,n| 185 + * +----+----+----+----+ 186 + * 187 + * 4) Update changed page permissions: 188 + * 189 + * +----+----+----+----+ 190 + * |RW,n|RW,n|RW,n|RO,n| <--- last page permissions set 191 + * +----+----+----+----+ 192 + * 193 + * 5) The kernel will eventually __flush_tlb() for changed page: 194 + * 195 + * |____| <--- tlbi + dsb 196 + * 197 + * For BBML2, we again remove the intermediate tlbi+dsb. Here, there 198 + * are no issues, as the final tlbi+dsb covering the changed page is 199 + * guaranteed to remove the original large contiguous (RW,c) TLB entry, 200 + * as well as the intermediate (RW,n) TLB entry; the next access will 201 + * install the new (RO,n) TLB entry and the page tables are only 202 + * considered "consistent" after the final tlbi+dsb, so software must 203 + * be prepared for this inconsistency prior to finishing the mm dance 204 + * regardless. 205 + */ 206 + 207 + if (!system_supports_bbml2_noabort()) 208 + __flush_tlb_range(&vma, start_addr, addr, PAGE_SIZE, true, 3); 72 209 73 210 __set_ptes(mm, start_addr, start_ptep, pte, CONT_PTES); 74 211 } ··· 306 169 for (i = 0; i < CONT_PTES; i++, ptep++) { 307 170 pte = __ptep_get(ptep); 308 171 309 - if (pte_dirty(pte)) 172 + if (pte_dirty(pte)) { 310 173 orig_pte = pte_mkdirty(orig_pte); 174 + for (; i < CONT_PTES; i++, ptep++) { 175 + pte = __ptep_get(ptep); 176 + if (pte_young(pte)) { 177 + orig_pte = pte_mkyoung(orig_pte); 178 + break; 179 + } 180 + } 181 + break; 182 + } 311 183 312 - if (pte_young(pte)) 184 + if (pte_young(pte)) { 313 185 orig_pte = pte_mkyoung(orig_pte); 186 + i++; 187 + ptep++; 188 + for (; i < CONT_PTES; i++, ptep++) { 189 + pte = __ptep_get(ptep); 190 + if (pte_dirty(pte)) { 191 + orig_pte = pte_mkdirty(orig_pte); 192 + break; 193 + } 194 + } 195 + break; 196 + } 314 197 } 315 198 316 199 return orig_pte; 317 200 } 318 201 EXPORT_SYMBOL_GPL(contpte_ptep_get); 202 + 203 + static inline bool contpte_is_consistent(pte_t pte, unsigned long pfn, 204 + pgprot_t orig_prot) 205 + { 206 + pgprot_t prot = pte_pgprot(pte_mkold(pte_mkclean(pte))); 207 + 208 + return pte_valid_cont(pte) && pte_pfn(pte) == pfn && 209 + pgprot_val(prot) == pgprot_val(orig_prot); 210 + } 319 211 320 212 pte_t contpte_ptep_get_lockless(pte_t *orig_ptep) 321 213 { ··· 368 202 pgprot_t orig_prot; 369 203 unsigned long pfn; 370 204 pte_t orig_pte; 371 - pgprot_t prot; 372 205 pte_t *ptep; 373 206 pte_t pte; 374 207 int i; ··· 384 219 385 220 for (i = 0; i < CONT_PTES; i++, ptep++, pfn++) { 386 221 pte = __ptep_get(ptep); 387 - prot = pte_pgprot(pte_mkold(pte_mkclean(pte))); 388 222 389 - if (!pte_valid_cont(pte) || 390 - pte_pfn(pte) != pfn || 391 - pgprot_val(prot) != pgprot_val(orig_prot)) 223 + if (!contpte_is_consistent(pte, pfn, orig_prot)) 392 224 goto retry; 393 225 394 - if (pte_dirty(pte)) 226 + if (pte_dirty(pte)) { 395 227 orig_pte = pte_mkdirty(orig_pte); 228 + for (; i < CONT_PTES; i++, ptep++, pfn++) { 229 + pte = __ptep_get(ptep); 396 230 397 - if (pte_young(pte)) 231 + if (!contpte_is_consistent(pte, pfn, orig_prot)) 232 + goto retry; 233 + 234 + if (pte_young(pte)) { 235 + orig_pte = pte_mkyoung(orig_pte); 236 + break; 237 + } 238 + } 239 + break; 240 + } 241 + 242 + if (pte_young(pte)) { 398 243 orig_pte = pte_mkyoung(orig_pte); 244 + i++; 245 + ptep++; 246 + pfn++; 247 + for (; i < CONT_PTES; i++, ptep++, pfn++) { 248 + pte = __ptep_get(ptep); 249 + 250 + if (!contpte_is_consistent(pte, pfn, orig_prot)) 251 + goto retry; 252 + 253 + if (pte_dirty(pte)) { 254 + orig_pte = pte_mkdirty(orig_pte); 255 + break; 256 + } 257 + } 258 + break; 259 + } 399 260 } 400 261 401 262 return orig_pte;

+6 -77

arch/arm64/mm/fault.c

··· 53 53 }; 54 54 55 55 static const struct fault_info fault_info[]; 56 - static struct fault_info debug_fault_info[]; 57 56 58 57 static inline const struct fault_info *esr_to_fault_info(unsigned long esr) 59 58 { 60 59 return fault_info + (esr & ESR_ELx_FSC); 61 - } 62 - 63 - static inline const struct fault_info *esr_to_debug_fault_info(unsigned long esr) 64 - { 65 - return debug_fault_info + DBG_ESR_EVT(esr); 66 60 } 67 61 68 62 static void data_abort_decode(unsigned long esr) ··· 832 838 */ 833 839 siaddr = untagged_addr(far); 834 840 } 841 + add_taint(TAINT_MACHINE_CHECK, LOCKDEP_STILL_OK); 835 842 arm64_notify_die(inf->name, regs, inf->sig, inf->code, siaddr, esr); 836 843 837 844 return 0; ··· 844 849 /* 845 850 * The architecture specifies that bits 63:60 of FAR_EL1 are UNKNOWN 846 851 * for tag check faults. Set them to corresponding bits in the untagged 847 - * address. 852 + * address if ARM64_MTE_FAR isn't supported. 853 + * Otherwise, bits 63:60 of FAR_EL1 are not UNKNOWN. 848 854 */ 849 - far = (__untagged_addr(far) & ~MTE_TAG_MASK) | (far & MTE_TAG_MASK); 855 + if (!cpus_have_cap(ARM64_MTE_FAR)) 856 + far = (__untagged_addr(far) & ~MTE_TAG_MASK) | (far & MTE_TAG_MASK); 857 + 850 858 do_bad_area(far, esr, regs); 851 859 return 0; 852 860 } ··· 947 949 addr, esr); 948 950 } 949 951 NOKPROBE_SYMBOL(do_sp_pc_abort); 950 - 951 - /* 952 - * __refdata because early_brk64 is __init, but the reference to it is 953 - * clobbered at arch_initcall time. 954 - * See traps.c and debug-monitors.c:debug_traps_init(). 955 - */ 956 - static struct fault_info __refdata debug_fault_info[] = { 957 - { do_bad, SIGTRAP, TRAP_HWBKPT, "hardware breakpoint" }, 958 - { do_bad, SIGTRAP, TRAP_HWBKPT, "hardware single-step" }, 959 - { do_bad, SIGTRAP, TRAP_HWBKPT, "hardware watchpoint" }, 960 - { do_bad, SIGKILL, SI_KERNEL, "unknown 3" }, 961 - { do_bad, SIGTRAP, TRAP_BRKPT, "aarch32 BKPT" }, 962 - { do_bad, SIGKILL, SI_KERNEL, "aarch32 vector catch" }, 963 - { early_brk64, SIGTRAP, TRAP_BRKPT, "aarch64 BRK" }, 964 - { do_bad, SIGKILL, SI_KERNEL, "unknown 7" }, 965 - }; 966 - 967 - void __init hook_debug_fault_code(int nr, 968 - int (*fn)(unsigned long, unsigned long, struct pt_regs *), 969 - int sig, int code, const char *name) 970 - { 971 - BUG_ON(nr < 0 || nr >= ARRAY_SIZE(debug_fault_info)); 972 - 973 - debug_fault_info[nr].fn = fn; 974 - debug_fault_info[nr].sig = sig; 975 - debug_fault_info[nr].code = code; 976 - debug_fault_info[nr].name = name; 977 - } 978 - 979 - /* 980 - * In debug exception context, we explicitly disable preemption despite 981 - * having interrupts disabled. 982 - * This serves two purposes: it makes it much less likely that we would 983 - * accidentally schedule in exception context and it will force a warning 984 - * if we somehow manage to schedule by accident. 985 - */ 986 - static void debug_exception_enter(struct pt_regs *regs) 987 - { 988 - preempt_disable(); 989 - 990 - /* This code is a bit fragile. Test it. */ 991 - RCU_LOCKDEP_WARN(!rcu_is_watching(), "exception_enter didn't work"); 992 - } 993 - NOKPROBE_SYMBOL(debug_exception_enter); 994 - 995 - static void debug_exception_exit(struct pt_regs *regs) 996 - { 997 - preempt_enable_no_resched(); 998 - } 999 - NOKPROBE_SYMBOL(debug_exception_exit); 1000 - 1001 - void do_debug_exception(unsigned long addr_if_watchpoint, unsigned long esr, 1002 - struct pt_regs *regs) 1003 - { 1004 - const struct fault_info *inf = esr_to_debug_fault_info(esr); 1005 - unsigned long pc = instruction_pointer(regs); 1006 - 1007 - debug_exception_enter(regs); 1008 - 1009 - if (user_mode(regs) && !is_ttbr0_addr(pc)) 1010 - arm64_apply_bp_hardening(); 1011 - 1012 - if (inf->fn(addr_if_watchpoint, esr, regs)) { 1013 - arm64_notify_die(inf->name, regs, inf->sig, inf->code, pc, esr); 1014 - } 1015 - 1016 - debug_exception_exit(regs); 1017 - } 1018 - NOKPROBE_SYMBOL(do_debug_exception); 1019 952 1020 953 /* 1021 954 * Used during anonymous page fault handling.

-6

arch/arm64/mm/gcs.c

··· 157 157 if (!system_supports_gcs()) 158 158 return; 159 159 160 - /* 161 - * When fork() with CLONE_VM fails, the child (tsk) already 162 - * has a GCS allocated, and exit_thread() calls this function 163 - * to free it. In this case the parent (current) and the 164 - * child share the same mm struct. 165 - */ 166 160 if (!task->mm || task->mm != current->mm) 167 161 return; 168 162

+1 -1

arch/arm64/mm/hugetlbpage.c

··· 225 225 ncontig = num_contig_ptes(sz, &pgsize); 226 226 227 227 if (!pte_present(pte)) { 228 - for (i = 0; i < ncontig; i++, ptep++, addr += pgsize) 228 + for (i = 0; i < ncontig; i++, ptep++) 229 229 __set_ptes_anysz(mm, ptep, pte, 1, pgsize); 230 230 return; 231 231 }

+1 -1

arch/arm64/mm/proc.S

··· 454 454 dsb nsh 455 455 456 456 msr cpacr_el1, xzr // Reset cpacr_el1 457 - mov x1, #1 << 12 // Reset mdscr_el1 and disable 457 + mov x1, MDSCR_EL1_TDCC // Reset mdscr_el1 and disable 458 458 msr mdscr_el1, x1 // access to the DCC from EL0 459 459 reset_pmuserenr_el0 x1 // Disable PMU access from EL0 460 460 reset_amuserenr_el0 x1 // Disable AMU access from EL0

+3

arch/arm64/tools/cpucaps

··· 45 45 HAS_LSE_ATOMICS 46 46 HAS_MOPS 47 47 HAS_NESTED_VIRT 48 + HAS_BBML2_NOABORT 48 49 HAS_PAN 49 50 HAS_PMUV3 50 51 HAS_S1PIE ··· 69 68 MPAM_HCR 70 69 MTE 71 70 MTE_ASYMM 71 + MTE_FAR 72 + MTE_STORE_ONLY 72 73 SME 73 74 SME_FA64 74 75 SME2

+132

arch/arm64/tools/sysreg

··· 1329 1329 EndEnum 1330 1330 EndSysreg 1331 1331 1332 + 1333 + SysregFields BRBINFx_EL1 1334 + Res0 63:47 1335 + Field 46 CCU 1336 + Field 45:40 CC_EXP 1337 + Field 39:32 CC_MANT 1338 + Res0 31:18 1339 + Field 17 LASTFAILED 1340 + Field 16 T 1341 + Res0 15:14 1342 + Enum 13:8 TYPE 1343 + 0b000000 DIRECT_UNCOND 1344 + 0b000001 INDIRECT 1345 + 0b000010 DIRECT_LINK 1346 + 0b000011 INDIRECT_LINK 1347 + 0b000101 RET 1348 + 0b000111 ERET 1349 + 0b001000 DIRECT_COND 1350 + 0b100001 DEBUG_HALT 1351 + 0b100010 CALL 1352 + 0b100011 TRAP 1353 + 0b100100 SERROR 1354 + 0b100110 INSN_DEBUG 1355 + 0b100111 DATA_DEBUG 1356 + 0b101010 ALIGN_FAULT 1357 + 0b101011 INSN_FAULT 1358 + 0b101100 DATA_FAULT 1359 + 0b101110 IRQ 1360 + 0b101111 FIQ 1361 + 0b110000 IMPDEF_TRAP_EL3 1362 + 0b111001 DEBUG_EXIT 1363 + EndEnum 1364 + Enum 7:6 EL 1365 + 0b00 EL0 1366 + 0b01 EL1 1367 + 0b10 EL2 1368 + 0b11 EL3 1369 + EndEnum 1370 + Field 5 MPRED 1371 + Res0 4:2 1372 + Enum 1:0 VALID 1373 + 0b00 NONE 1374 + 0b01 TARGET 1375 + 0b10 SOURCE 1376 + 0b11 FULL 1377 + EndEnum 1378 + EndSysregFields 1379 + 1380 + SysregFields BRBCR_ELx 1381 + Res0 63:24 1382 + Field 23 EXCEPTION 1383 + Field 22 ERTN 1384 + Res0 21:10 1385 + Field 9 FZPSS 1386 + Field 8 FZP 1387 + Res0 7 1388 + Enum 6:5 TS 1389 + 0b01 VIRTUAL 1390 + 0b10 GUEST_PHYSICAL 1391 + 0b11 PHYSICAL 1392 + EndEnum 1393 + Field 4 MPRED 1394 + Field 3 CC 1395 + Res0 2 1396 + Field 1 ExBRE 1397 + Field 0 E0BRE 1398 + EndSysregFields 1399 + 1400 + Sysreg BRBCR_EL1 2 1 9 0 0 1401 + Fields BRBCR_ELx 1402 + EndSysreg 1403 + 1404 + Sysreg BRBFCR_EL1 2 1 9 0 1 1405 + Res0 63:30 1406 + Enum 29:28 BANK 1407 + 0b00 BANK_0 1408 + 0b01 BANK_1 1409 + EndEnum 1410 + Res0 27:23 1411 + Field 22 CONDDIR 1412 + Field 21 DIRCALL 1413 + Field 20 INDCALL 1414 + Field 19 RTN 1415 + Field 18 INDIRECT 1416 + Field 17 DIRECT 1417 + Field 16 EnI 1418 + Res0 15:8 1419 + Field 7 PAUSED 1420 + Field 6 LASTFAILED 1421 + Res0 5:0 1422 + EndSysreg 1423 + 1424 + Sysreg BRBTS_EL1 2 1 9 0 2 1425 + Field 63:0 TS 1426 + EndSysreg 1427 + 1428 + Sysreg BRBINFINJ_EL1 2 1 9 1 0 1429 + Fields BRBINFx_EL1 1430 + EndSysreg 1431 + 1432 + Sysreg BRBSRCINJ_EL1 2 1 9 1 1 1433 + Field 63:0 ADDRESS 1434 + EndSysreg 1435 + 1436 + Sysreg BRBTGTINJ_EL1 2 1 9 1 2 1437 + Field 63:0 ADDRESS 1438 + EndSysreg 1439 + 1440 + Sysreg BRBIDR0_EL1 2 1 9 2 0 1441 + Res0 63:16 1442 + Enum 15:12 CC 1443 + 0b0101 20_BIT 1444 + EndEnum 1445 + Enum 11:8 FORMAT 1446 + 0b0000 FORMAT_0 1447 + EndEnum 1448 + Enum 7:0 NUMREC 1449 + 0b00001000 8 1450 + 0b00010000 16 1451 + 0b00100000 32 1452 + 0b01000000 64 1453 + EndEnum 1454 + EndSysreg 1455 + 1456 + Sysreg BRBCR_EL2 2 4 9 0 0 1457 + Fields BRBCR_ELx 1458 + EndSysreg 1459 + 1460 + Sysreg BRBCR_EL12 2 5 9 0 0 1461 + Fields BRBCR_ELx 1462 + EndSysreg 1463 + 1332 1464 Sysreg ID_AA64ZFR0_EL1 3 0 0 4 4 1333 1465 Res0 63:60 1334 1466 UnsignedEnum 59:56 F64MM

+1 -1

drivers/firmware/efi/libstub/Makefile.zboot

··· 36 36 -DPE_DLL_CHAR_EX=IMAGE_DLLCHARACTERISTICS_EX_FORWARD_CFI_COMPAT 37 37 38 38 AFLAGS_zboot-header.o += -DMACHINE_TYPE=IMAGE_FILE_MACHINE_$(EFI_ZBOOT_MACH_TYPE) \ 39 - -DZBOOT_EFI_PATH="\"$(realpath $(obj)/vmlinuz.efi.elf)\"" \ 39 + -DZBOOT_EFI_PATH="\"$(abspath $(obj)/vmlinuz.efi.elf)\"" \ 40 40 -DZBOOT_SIZE_LEN=$(zboot-size-len-y) \ 41 41 -DCOMP_TYPE="\"$(comp-type-y)\"" \ 42 42 $(aflags-zboot-header-y)

+3

drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c

··· 220 220 feat_mask |= ARM_SMMU_FEAT_VAX; 221 221 } 222 222 223 + if (system_supports_bbml2_noabort()) 224 + feat_mask |= ARM_SMMU_FEAT_BBML2; 225 + 223 226 if ((smmu->features & feat_mask) != feat_mask) 224 227 return false; 225 228

+3

drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c

··· 4457 4457 if (FIELD_GET(IDR3_FWB, reg)) 4458 4458 smmu->features |= ARM_SMMU_FEAT_S2FWB; 4459 4459 4460 + if (FIELD_GET(IDR3_BBM, reg) == 2) 4461 + smmu->features |= ARM_SMMU_FEAT_BBML2; 4462 + 4460 4463 /* IDR5 */ 4461 4464 reg = readl_relaxed(smmu->base + ARM_SMMU_IDR5); 4462 4465

+2

drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h

··· 60 60 #define ARM_SMMU_IDR3 0xc 61 61 #define IDR3_FWB (1 << 8) 62 62 #define IDR3_RIL (1 << 10) 63 + #define IDR3_BBM GENMASK(12, 11) 63 64 64 65 #define ARM_SMMU_IDR5 0x14 65 66 #define IDR5_STALL_MAX GENMASK(31, 16) ··· 756 755 #define ARM_SMMU_FEAT_HA (1 << 21) 757 756 #define ARM_SMMU_FEAT_HD (1 << 22) 758 757 #define ARM_SMMU_FEAT_S2FWB (1 << 23) 758 + #define ARM_SMMU_FEAT_BBML2 (1 << 24) 759 759 u32 features; 760 760 761 761 #define ARM_SMMU_OPT_SKIP_PREFETCH (1 << 0)

+11

drivers/perf/Kconfig

··· 223 223 Extension, which provides periodic sampling of operations in 224 224 the CPU pipeline and reports this via the perf AUX interface. 225 225 226 + config ARM64_BRBE 227 + bool "Enable support for branch stack sampling using FEAT_BRBE" 228 + depends on ARM_PMUV3 && ARM64 229 + default y 230 + help 231 + Enable perf support for Branch Record Buffer Extension (BRBE) which 232 + records all branches taken in an execution path. This supports some 233 + branch types and privilege based filtering. It captures additional 234 + relevant information such as cycle count, misprediction and branch 235 + type, branch privilege level etc. 236 + 226 237 config ARM_DMC620_PMU 227 238 tristate "Enable PMU support for the ARM DMC-620 memory controller" 228 239 depends on (ARM64 && ACPI) || COMPILE_TEST

+1

drivers/perf/Makefile

··· 23 23 obj-$(CONFIG_THUNDERX2_PMU) += thunderx2_pmu.o 24 24 obj-$(CONFIG_XGENE_PMU) += xgene_pmu.o 25 25 obj-$(CONFIG_ARM_SPE_PMU) += arm_spe_pmu.o 26 + obj-$(CONFIG_ARM64_BRBE) += arm_brbe.o 26 27 obj-$(CONFIG_ARM_DMC620_PMU) += arm_dmc620_pmu.o 27 28 obj-$(CONFIG_MARVELL_CN10K_TAD_PMU) += marvell_cn10k_tad_pmu.o 28 29 obj-$(CONFIG_MARVELL_CN10K_DDR_PMU) += marvell_cn10k_ddr_pmu.o

+11 -9

drivers/perf/arm-cmn.c

··· 1 1 // SPDX-License-Identifier: GPL-2.0 2 2 // Copyright (C) 2016-2020 Arm Limited 3 - // CMN-600 Coherent Mesh Network PMU driver 3 + // ARM CMN/CI interconnect PMU driver 4 4 5 5 #include <linux/acpi.h> 6 6 #include <linux/bitfield.h> ··· 2245 2245 2246 2246 static int arm_cmn_discover(struct arm_cmn *cmn, unsigned int rgn_offset) 2247 2247 { 2248 - void __iomem *cfg_region; 2248 + void __iomem *cfg_region, __iomem *xp_region; 2249 2249 struct arm_cmn_node cfg, *dn; 2250 2250 struct arm_cmn_dtm *dtm; 2251 2251 enum cmn_part part; 2252 2252 u16 child_count, child_poff; 2253 - u32 xp_offset[CMN_MAX_XPS]; 2254 2253 u64 reg; 2255 2254 int i, j; 2256 2255 size_t sz; ··· 2301 2302 cmn->num_dns = cmn->num_xps; 2302 2303 2303 2304 /* Pass 1: visit the XPs, enumerate their children */ 2305 + cfg_region += child_poff; 2304 2306 for (i = 0; i < cmn->num_xps; i++) { 2305 - reg = readq_relaxed(cfg_region + child_poff + i * 8); 2306 - xp_offset[i] = reg & CMN_CHILD_NODE_ADDR; 2307 + reg = readq_relaxed(cfg_region + i * 8); 2308 + xp_region = cmn->base + (reg & CMN_CHILD_NODE_ADDR); 2307 2309 2308 - reg = readq_relaxed(cmn->base + xp_offset[i] + CMN_CHILD_INFO); 2310 + reg = readq_relaxed(xp_region + CMN_CHILD_INFO); 2309 2311 cmn->num_dns += FIELD_GET(CMN_CI_CHILD_COUNT, reg); 2310 2312 } 2311 2313 ··· 2332 2332 cmn->dns = dn; 2333 2333 cmn->dtms = dtm; 2334 2334 for (i = 0; i < cmn->num_xps; i++) { 2335 - void __iomem *xp_region = cmn->base + xp_offset[i]; 2336 2335 struct arm_cmn_node *xp = dn++; 2337 2336 unsigned int xp_ports = 0; 2338 2337 2339 - arm_cmn_init_node_info(cmn, xp_offset[i], xp); 2338 + reg = readq_relaxed(cfg_region + i * 8); 2339 + xp_region = cmn->base + (reg & CMN_CHILD_NODE_ADDR); 2340 + arm_cmn_init_node_info(cmn, reg & CMN_CHILD_NODE_ADDR, xp); 2340 2341 /* 2341 2342 * Thanks to the order in which XP logical IDs seem to be 2342 2343 * assigned, we can handily infer the mesh X dimension by ··· 2656 2655 .name = "arm-cmn", 2657 2656 .of_match_table = of_match_ptr(arm_cmn_of_match), 2658 2657 .acpi_match_table = ACPI_PTR(arm_cmn_acpi_match), 2658 + .suppress_bind_attrs = true, 2659 2659 }, 2660 2660 .probe = arm_cmn_probe, 2661 2661 .remove = arm_cmn_remove, ··· 2695 2693 module_exit(arm_cmn_exit); 2696 2694 2697 2695 MODULE_AUTHOR("Robin Murphy <robin.murphy@arm.com>"); 2698 - MODULE_DESCRIPTION("Arm CMN-600 PMU driver"); 2696 + MODULE_DESCRIPTION("Arm CMN/CI interconnect PMU driver"); 2699 2697 MODULE_LICENSE("GPL v2");

+89 -64

drivers/perf/arm-ni.c

··· 102 102 struct arm_ni_cd { 103 103 void __iomem *pmu_base; 104 104 u16 id; 105 + s8 irq_friend; 105 106 int num_units; 106 107 int irq; 107 - int cpu; 108 - struct hlist_node cpuhp_node; 109 108 struct pmu pmu; 110 109 struct arm_ni_unit *units; 111 110 struct perf_event *evcnt[NI_NUM_COUNTERS]; ··· 116 117 void __iomem *base; 117 118 enum ni_part part; 118 119 int id; 120 + int cpu; 119 121 int num_cds; 122 + struct hlist_node cpuhp_node; 120 123 struct arm_ni_cd cds[] __counted_by(num_cds); 121 124 }; 122 125 123 126 #define cd_to_ni(cd) container_of((cd), struct arm_ni, cds[(cd)->id]) 124 127 #define pmu_to_cd(p) container_of((p), struct arm_ni_cd, pmu) 128 + 129 + #define ni_for_each_cd(n, c) \ 130 + for (struct arm_ni_cd *c = n->cds; c < n->cds + n->num_cds; c++) if (c->pmu_base) 125 131 126 132 #define cd_for_each_unit(cd, u) \ 127 133 for (struct arm_ni_unit *u = cd->units; u < cd->units + cd->num_units; u++) ··· 222 218 static ssize_t arm_ni_cpumask_show(struct device *dev, 223 219 struct device_attribute *attr, char *buf) 224 220 { 225 - struct arm_ni_cd *cd = pmu_to_cd(dev_get_drvdata(dev)); 221 + struct arm_ni *ni = cd_to_ni(pmu_to_cd(dev_get_drvdata(dev))); 226 222 227 - return cpumap_print_to_pagebuf(true, buf, cpumask_of(cd->cpu)); 223 + return cpumap_print_to_pagebuf(true, buf, cpumask_of(ni->cpu)); 228 224 } 229 225 230 226 static struct device_attribute arm_ni_cpumask_attr = ··· 318 314 if (is_sampling_event(event)) 319 315 return -EINVAL; 320 316 321 - event->cpu = cd->cpu; 317 + event->cpu = cd_to_ni(cd)->cpu; 322 318 if (NI_EVENT_TYPE(event) == NI_PMU) 323 319 return arm_ni_validate_group(event); 324 320 ··· 449 445 { 450 446 struct arm_ni_cd *cd = dev_id; 451 447 irqreturn_t ret = IRQ_NONE; 452 - u32 reg = readl_relaxed(cd->pmu_base + NI_PMOVSCLR); 453 448 454 - if (reg & (1U << NI_CCNT_IDX)) { 455 - ret = IRQ_HANDLED; 456 - if (!(WARN_ON(!cd->ccnt))) { 457 - arm_ni_event_read(cd->ccnt); 458 - arm_ni_init_ccnt(cd); 449 + for (;;) { 450 + u32 reg = readl_relaxed(cd->pmu_base + NI_PMOVSCLR); 451 + 452 + if (reg & (1U << NI_CCNT_IDX)) { 453 + ret = IRQ_HANDLED; 454 + if (!(WARN_ON(!cd->ccnt))) { 455 + arm_ni_event_read(cd->ccnt); 456 + arm_ni_init_ccnt(cd); 457 + } 459 458 } 460 - } 461 - for (int i = 0; i < NI_NUM_COUNTERS; i++) { 462 - if (!(reg & (1U << i))) 463 - continue; 464 - ret = IRQ_HANDLED; 465 - if (!(WARN_ON(!cd->evcnt[i]))) { 466 - arm_ni_event_read(cd->evcnt[i]); 467 - arm_ni_init_evcnt(cd, i); 459 + for (int i = 0; i < NI_NUM_COUNTERS; i++) { 460 + if (!(reg & (1U << i))) 461 + continue; 462 + ret = IRQ_HANDLED; 463 + if (!(WARN_ON(!cd->evcnt[i]))) { 464 + arm_ni_event_read(cd->evcnt[i]); 465 + arm_ni_init_evcnt(cd, i); 466 + } 468 467 } 468 + writel_relaxed(reg, cd->pmu_base + NI_PMOVSCLR); 469 + if (!cd->irq_friend) 470 + return ret; 471 + cd += cd->irq_friend; 469 472 } 470 - writel_relaxed(reg, cd->pmu_base + NI_PMOVSCLR); 471 - return ret; 472 473 } 473 474 474 475 static int arm_ni_init_cd(struct arm_ni *ni, struct arm_ni_node *node, u64 res_start) 475 476 { 476 477 struct arm_ni_cd *cd = ni->cds + node->id; 477 478 const char *name; 478 - int err; 479 479 480 480 cd->id = node->id; 481 481 cd->num_units = node->num_components; ··· 539 531 cd->pmu_base + NI_PMCR); 540 532 writel_relaxed(U32_MAX, cd->pmu_base + NI_PMCNTENCLR); 541 533 writel_relaxed(U32_MAX, cd->pmu_base + NI_PMOVSCLR); 542 - writel_relaxed(U32_MAX, cd->pmu_base + NI_PMINTENSET); 543 534 544 535 cd->irq = platform_get_irq(to_platform_device(ni->dev), cd->id); 545 536 if (cd->irq < 0) 546 537 return cd->irq; 547 538 548 - err = devm_request_irq(ni->dev, cd->irq, arm_ni_handle_irq, 549 - IRQF_NOBALANCING | IRQF_NO_THREAD, 550 - dev_name(ni->dev), cd); 551 - if (err) 552 - return err; 553 - 554 - cd->cpu = cpumask_local_spread(0, dev_to_node(ni->dev)); 555 539 cd->pmu = (struct pmu) { 556 540 .module = THIS_MODULE, 557 541 .parent = ni->dev, ··· 564 564 if (!name) 565 565 return -ENOMEM; 566 566 567 - err = cpuhp_state_add_instance_nocalls(arm_ni_hp_state, &cd->cpuhp_node); 568 - if (err) 569 - return err; 570 - 571 - err = perf_pmu_register(&cd->pmu, name, -1); 572 - if (err) 573 - cpuhp_state_remove_instance_nocalls(arm_ni_hp_state, &cd->cpuhp_node); 574 - 575 - return err; 567 + return perf_pmu_register(&cd->pmu, name, -1); 576 568 } 577 569 578 570 static void arm_ni_remove(struct platform_device *pdev) 579 571 { 580 572 struct arm_ni *ni = platform_get_drvdata(pdev); 581 573 582 - for (int i = 0; i < ni->num_cds; i++) { 583 - struct arm_ni_cd *cd = ni->cds + i; 584 - 585 - if (!cd->pmu_base) 586 - continue; 587 - 574 + ni_for_each_cd(ni, cd) { 588 575 writel_relaxed(0, cd->pmu_base + NI_PMCR); 589 576 writel_relaxed(U32_MAX, cd->pmu_base + NI_PMINTENCLR); 590 577 perf_pmu_unregister(&cd->pmu); 591 - cpuhp_state_remove_instance_nocalls(arm_ni_hp_state, &cd->cpuhp_node); 592 578 } 579 + cpuhp_state_remove_instance_nocalls(arm_ni_hp_state, &ni->cpuhp_node); 593 580 } 594 581 595 582 static void arm_ni_probe_domain(void __iomem *base, struct arm_ni_node *node) ··· 589 602 node->num_components = readl_relaxed(base + NI_CHILD_NODE_INFO); 590 603 } 591 604 605 + static int arm_ni_init_irqs(struct arm_ni *ni) 606 + { 607 + int err; 608 + 609 + ni_for_each_cd(ni, cd) { 610 + for (struct arm_ni_cd *prev = cd; prev-- > ni->cds; ) { 611 + if (prev->irq == cd->irq) { 612 + prev->irq_friend = cd - prev; 613 + goto set_inten; 614 + } 615 + } 616 + err = devm_request_irq(ni->dev, cd->irq, arm_ni_handle_irq, 617 + IRQF_NOBALANCING | IRQF_NO_THREAD | IRQF_NO_AUTOEN, 618 + dev_name(ni->dev), cd); 619 + if (err) 620 + return err; 621 + 622 + irq_set_affinity(cd->irq, cpumask_of(ni->cpu)); 623 + set_inten: 624 + writel_relaxed(U32_MAX, cd->pmu_base + NI_PMINTENSET); 625 + } 626 + 627 + ni_for_each_cd(ni, cd) 628 + if (!cd->irq_friend) 629 + enable_irq(cd->irq); 630 + return 0; 631 + } 632 + 592 633 static int arm_ni_probe(struct platform_device *pdev) 593 634 { 594 635 struct arm_ni_node cfg, vd, pd, cd; ··· 624 609 struct resource *res; 625 610 void __iomem *base; 626 611 static atomic_t id; 627 - int num_cds; 612 + int ret, num_cds; 628 613 u32 reg, part; 629 614 630 615 /* ··· 675 660 ni->num_cds = num_cds; 676 661 ni->part = part; 677 662 ni->id = atomic_fetch_inc(&id); 663 + ni->cpu = cpumask_local_spread(0, dev_to_node(ni->dev)); 678 664 platform_set_drvdata(pdev, ni); 665 + 666 + ret = cpuhp_state_add_instance_nocalls(arm_ni_hp_state, &ni->cpuhp_node); 667 + if (ret) 668 + return ret; 679 669 680 670 for (int v = 0; v < cfg.num_components; v++) { 681 671 reg = readl_relaxed(cfg.base + NI_CHILD_PTR(v)); ··· 689 669 reg = readl_relaxed(vd.base + NI_CHILD_PTR(p)); 690 670 arm_ni_probe_domain(base + reg, &pd); 691 671 for (int c = 0; c < pd.num_components; c++) { 692 - int ret; 693 - 694 672 reg = readl_relaxed(pd.base + NI_CHILD_PTR(c)); 695 673 arm_ni_probe_domain(base + reg, &cd); 696 674 ret = arm_ni_init_cd(ni, &cd, res->start); ··· 701 683 } 702 684 } 703 685 704 - return 0; 686 + ret = arm_ni_init_irqs(ni); 687 + if (ret) 688 + arm_ni_remove(pdev); 689 + 690 + return ret; 705 691 } 706 692 707 693 #ifdef CONFIG_OF ··· 729 707 .name = "arm-ni", 730 708 .of_match_table = of_match_ptr(arm_ni_of_match), 731 709 .acpi_match_table = ACPI_PTR(arm_ni_acpi_match), 710 + .suppress_bind_attrs = true, 732 711 }, 733 712 .probe = arm_ni_probe, 734 713 .remove = arm_ni_remove, 735 714 }; 736 715 737 - static void arm_ni_pmu_migrate(struct arm_ni_cd *cd, unsigned int cpu) 716 + static void arm_ni_pmu_migrate(struct arm_ni *ni, unsigned int cpu) 738 717 { 739 - perf_pmu_migrate_context(&cd->pmu, cd->cpu, cpu); 740 - irq_set_affinity(cd->irq, cpumask_of(cpu)); 741 - cd->cpu = cpu; 718 + ni_for_each_cd(ni, cd) { 719 + perf_pmu_migrate_context(&cd->pmu, ni->cpu, cpu); 720 + irq_set_affinity(cd->irq, cpumask_of(cpu)); 721 + } 722 + ni->cpu = cpu; 742 723 } 743 724 744 725 static int arm_ni_pmu_online_cpu(unsigned int cpu, struct hlist_node *cpuhp_node) 745 726 { 746 - struct arm_ni_cd *cd; 727 + struct arm_ni *ni; 747 728 int node; 748 729 749 - cd = hlist_entry_safe(cpuhp_node, struct arm_ni_cd, cpuhp_node); 750 - node = dev_to_node(cd_to_ni(cd)->dev); 751 - if (cpu_to_node(cd->cpu) != node && cpu_to_node(cpu) == node) 752 - arm_ni_pmu_migrate(cd, cpu); 730 + ni = hlist_entry_safe(cpuhp_node, struct arm_ni, cpuhp_node); 731 + node = dev_to_node(ni->dev); 732 + if (cpu_to_node(ni->cpu) != node && cpu_to_node(cpu) == node) 733 + arm_ni_pmu_migrate(ni, cpu); 753 734 return 0; 754 735 } 755 736 756 737 static int arm_ni_pmu_offline_cpu(unsigned int cpu, struct hlist_node *cpuhp_node) 757 738 { 758 - struct arm_ni_cd *cd; 739 + struct arm_ni *ni; 759 740 unsigned int target; 760 741 int node; 761 742 762 - cd = hlist_entry_safe(cpuhp_node, struct arm_ni_cd, cpuhp_node); 763 - if (cpu != cd->cpu) 743 + ni = hlist_entry_safe(cpuhp_node, struct arm_ni, cpuhp_node); 744 + if (cpu != ni->cpu) 764 745 return 0; 765 746 766 - node = dev_to_node(cd_to_ni(cd)->dev); 747 + node = dev_to_node(ni->dev); 767 748 target = cpumask_any_and_but(cpumask_of_node(node), cpu_online_mask, cpu); 768 749 if (target >= nr_cpu_ids) 769 750 target = cpumask_any_but(cpu_online_mask, cpu); 770 751 771 752 if (target < nr_cpu_ids) 772 - arm_ni_pmu_migrate(cd, target); 753 + arm_ni_pmu_migrate(ni, target); 773 754 return 0; 774 755 } 775 756

+805

drivers/perf/arm_brbe.c

··· 1 + // SPDX-License-Identifier: GPL-2.0-only 2 + /* 3 + * Branch Record Buffer Extension Driver. 4 + * 5 + * Copyright (C) 2022-2025 ARM Limited 6 + * 7 + * Author: Anshuman Khandual <anshuman.khandual@arm.com> 8 + */ 9 + #include <linux/types.h> 10 + #include <linux/bitmap.h> 11 + #include <linux/perf/arm_pmu.h> 12 + #include "arm_brbe.h" 13 + 14 + #define BRBFCR_EL1_BRANCH_FILTERS (BRBFCR_EL1_DIRECT | \ 15 + BRBFCR_EL1_INDIRECT | \ 16 + BRBFCR_EL1_RTN | \ 17 + BRBFCR_EL1_INDCALL | \ 18 + BRBFCR_EL1_DIRCALL | \ 19 + BRBFCR_EL1_CONDDIR) 20 + 21 + /* 22 + * BRBTS_EL1 is currently not used for branch stack implementation 23 + * purpose but BRBCR_ELx.TS needs to have a valid value from all 24 + * available options. BRBCR_ELx_TS_VIRTUAL is selected for this. 25 + */ 26 + #define BRBCR_ELx_DEFAULT_TS FIELD_PREP(BRBCR_ELx_TS_MASK, BRBCR_ELx_TS_VIRTUAL) 27 + 28 + /* 29 + * BRBE Buffer Organization 30 + * 31 + * BRBE buffer is arranged as multiple banks of 32 branch record 32 + * entries each. An individual branch record in a given bank could 33 + * be accessed, after selecting the bank in BRBFCR_EL1.BANK and 34 + * accessing the registers i.e [BRBSRC, BRBTGT, BRBINF] set with 35 + * indices [0..31]. 36 + * 37 + * Bank 0 38 + * 39 + * --------------------------------- ------ 40 + * | 00 | BRBSRC | BRBTGT | BRBINF | | 00 | 41 + * --------------------------------- ------ 42 + * | 01 | BRBSRC | BRBTGT | BRBINF | | 01 | 43 + * --------------------------------- ------ 44 + * | .. | BRBSRC | BRBTGT | BRBINF | | .. | 45 + * --------------------------------- ------ 46 + * | 31 | BRBSRC | BRBTGT | BRBINF | | 31 | 47 + * --------------------------------- ------ 48 + * 49 + * Bank 1 50 + * 51 + * --------------------------------- ------ 52 + * | 32 | BRBSRC | BRBTGT | BRBINF | | 00 | 53 + * --------------------------------- ------ 54 + * | 33 | BRBSRC | BRBTGT | BRBINF | | 01 | 55 + * --------------------------------- ------ 56 + * | .. | BRBSRC | BRBTGT | BRBINF | | .. | 57 + * --------------------------------- ------ 58 + * | 63 | BRBSRC | BRBTGT | BRBINF | | 31 | 59 + * --------------------------------- ------ 60 + */ 61 + #define BRBE_BANK_MAX_ENTRIES 32 62 + 63 + struct brbe_regset { 64 + u64 brbsrc; 65 + u64 brbtgt; 66 + u64 brbinf; 67 + }; 68 + 69 + #define PERF_BR_ARM64_MAX (PERF_BR_MAX + PERF_BR_NEW_MAX) 70 + 71 + struct brbe_hw_attr { 72 + int brbe_version; 73 + int brbe_cc; 74 + int brbe_nr; 75 + int brbe_format; 76 + }; 77 + 78 + #define BRBE_REGN_CASE(n, case_macro) \ 79 + case n: case_macro(n); break 80 + 81 + #define BRBE_REGN_SWITCH(x, case_macro) \ 82 + do { \ 83 + switch (x) { \ 84 + BRBE_REGN_CASE(0, case_macro); \ 85 + BRBE_REGN_CASE(1, case_macro); \ 86 + BRBE_REGN_CASE(2, case_macro); \ 87 + BRBE_REGN_CASE(3, case_macro); \ 88 + BRBE_REGN_CASE(4, case_macro); \ 89 + BRBE_REGN_CASE(5, case_macro); \ 90 + BRBE_REGN_CASE(6, case_macro); \ 91 + BRBE_REGN_CASE(7, case_macro); \ 92 + BRBE_REGN_CASE(8, case_macro); \ 93 + BRBE_REGN_CASE(9, case_macro); \ 94 + BRBE_REGN_CASE(10, case_macro); \ 95 + BRBE_REGN_CASE(11, case_macro); \ 96 + BRBE_REGN_CASE(12, case_macro); \ 97 + BRBE_REGN_CASE(13, case_macro); \ 98 + BRBE_REGN_CASE(14, case_macro); \ 99 + BRBE_REGN_CASE(15, case_macro); \ 100 + BRBE_REGN_CASE(16, case_macro); \ 101 + BRBE_REGN_CASE(17, case_macro); \ 102 + BRBE_REGN_CASE(18, case_macro); \ 103 + BRBE_REGN_CASE(19, case_macro); \ 104 + BRBE_REGN_CASE(20, case_macro); \ 105 + BRBE_REGN_CASE(21, case_macro); \ 106 + BRBE_REGN_CASE(22, case_macro); \ 107 + BRBE_REGN_CASE(23, case_macro); \ 108 + BRBE_REGN_CASE(24, case_macro); \ 109 + BRBE_REGN_CASE(25, case_macro); \ 110 + BRBE_REGN_CASE(26, case_macro); \ 111 + BRBE_REGN_CASE(27, case_macro); \ 112 + BRBE_REGN_CASE(28, case_macro); \ 113 + BRBE_REGN_CASE(29, case_macro); \ 114 + BRBE_REGN_CASE(30, case_macro); \ 115 + BRBE_REGN_CASE(31, case_macro); \ 116 + default: WARN(1, "Invalid BRB* index %d\n", x); \ 117 + } \ 118 + } while (0) 119 + 120 + #define RETURN_READ_BRBSRCN(n) \ 121 + return read_sysreg_s(SYS_BRBSRC_EL1(n)) 122 + static inline u64 get_brbsrc_reg(int idx) 123 + { 124 + BRBE_REGN_SWITCH(idx, RETURN_READ_BRBSRCN); 125 + return 0; 126 + } 127 + 128 + #define RETURN_READ_BRBTGTN(n) \ 129 + return read_sysreg_s(SYS_BRBTGT_EL1(n)) 130 + static u64 get_brbtgt_reg(int idx) 131 + { 132 + BRBE_REGN_SWITCH(idx, RETURN_READ_BRBTGTN); 133 + return 0; 134 + } 135 + 136 + #define RETURN_READ_BRBINFN(n) \ 137 + return read_sysreg_s(SYS_BRBINF_EL1(n)) 138 + static u64 get_brbinf_reg(int idx) 139 + { 140 + BRBE_REGN_SWITCH(idx, RETURN_READ_BRBINFN); 141 + return 0; 142 + } 143 + 144 + static u64 brbe_record_valid(u64 brbinf) 145 + { 146 + return FIELD_GET(BRBINFx_EL1_VALID_MASK, brbinf); 147 + } 148 + 149 + static bool brbe_invalid(u64 brbinf) 150 + { 151 + return brbe_record_valid(brbinf) == BRBINFx_EL1_VALID_NONE; 152 + } 153 + 154 + static bool brbe_record_is_complete(u64 brbinf) 155 + { 156 + return brbe_record_valid(brbinf) == BRBINFx_EL1_VALID_FULL; 157 + } 158 + 159 + static bool brbe_record_is_source_only(u64 brbinf) 160 + { 161 + return brbe_record_valid(brbinf) == BRBINFx_EL1_VALID_SOURCE; 162 + } 163 + 164 + static bool brbe_record_is_target_only(u64 brbinf) 165 + { 166 + return brbe_record_valid(brbinf) == BRBINFx_EL1_VALID_TARGET; 167 + } 168 + 169 + static int brbinf_get_in_tx(u64 brbinf) 170 + { 171 + return FIELD_GET(BRBINFx_EL1_T_MASK, brbinf); 172 + } 173 + 174 + static int brbinf_get_mispredict(u64 brbinf) 175 + { 176 + return FIELD_GET(BRBINFx_EL1_MPRED_MASK, brbinf); 177 + } 178 + 179 + static int brbinf_get_lastfailed(u64 brbinf) 180 + { 181 + return FIELD_GET(BRBINFx_EL1_LASTFAILED_MASK, brbinf); 182 + } 183 + 184 + static u16 brbinf_get_cycles(u64 brbinf) 185 + { 186 + u32 exp, mant, cycles; 187 + /* 188 + * Captured cycle count is unknown and hence 189 + * should not be passed on to userspace. 190 + */ 191 + if (brbinf & BRBINFx_EL1_CCU) 192 + return 0; 193 + 194 + exp = FIELD_GET(BRBINFx_EL1_CC_EXP_MASK, brbinf); 195 + mant = FIELD_GET(BRBINFx_EL1_CC_MANT_MASK, brbinf); 196 + 197 + if (!exp) 198 + return mant; 199 + 200 + cycles = (mant | 0x100) << (exp - 1); 201 + 202 + return min(cycles, U16_MAX); 203 + } 204 + 205 + static int brbinf_get_type(u64 brbinf) 206 + { 207 + return FIELD_GET(BRBINFx_EL1_TYPE_MASK, brbinf); 208 + } 209 + 210 + static int brbinf_get_el(u64 brbinf) 211 + { 212 + return FIELD_GET(BRBINFx_EL1_EL_MASK, brbinf); 213 + } 214 + 215 + void brbe_invalidate(void) 216 + { 217 + /* Ensure all branches before this point are recorded */ 218 + isb(); 219 + asm volatile(BRB_IALL_INSN); 220 + /* Ensure all branch records are invalidated after this point */ 221 + isb(); 222 + } 223 + 224 + static bool valid_brbe_nr(int brbe_nr) 225 + { 226 + return brbe_nr == BRBIDR0_EL1_NUMREC_8 || 227 + brbe_nr == BRBIDR0_EL1_NUMREC_16 || 228 + brbe_nr == BRBIDR0_EL1_NUMREC_32 || 229 + brbe_nr == BRBIDR0_EL1_NUMREC_64; 230 + } 231 + 232 + static bool valid_brbe_cc(int brbe_cc) 233 + { 234 + return brbe_cc == BRBIDR0_EL1_CC_20_BIT; 235 + } 236 + 237 + static bool valid_brbe_format(int brbe_format) 238 + { 239 + return brbe_format == BRBIDR0_EL1_FORMAT_FORMAT_0; 240 + } 241 + 242 + static bool valid_brbidr(u64 brbidr) 243 + { 244 + int brbe_format, brbe_cc, brbe_nr; 245 + 246 + brbe_format = FIELD_GET(BRBIDR0_EL1_FORMAT_MASK, brbidr); 247 + brbe_cc = FIELD_GET(BRBIDR0_EL1_CC_MASK, brbidr); 248 + brbe_nr = FIELD_GET(BRBIDR0_EL1_NUMREC_MASK, brbidr); 249 + 250 + return valid_brbe_format(brbe_format) && valid_brbe_cc(brbe_cc) && valid_brbe_nr(brbe_nr); 251 + } 252 + 253 + static bool valid_brbe_version(int brbe_version) 254 + { 255 + return brbe_version == ID_AA64DFR0_EL1_BRBE_IMP || 256 + brbe_version == ID_AA64DFR0_EL1_BRBE_BRBE_V1P1; 257 + } 258 + 259 + static void select_brbe_bank(int bank) 260 + { 261 + u64 brbfcr; 262 + 263 + brbfcr = read_sysreg_s(SYS_BRBFCR_EL1); 264 + brbfcr &= ~BRBFCR_EL1_BANK_MASK; 265 + brbfcr |= SYS_FIELD_PREP(BRBFCR_EL1, BANK, bank); 266 + write_sysreg_s(brbfcr, SYS_BRBFCR_EL1); 267 + /* 268 + * Arm ARM (DDI 0487K.a) D.18.4 rule PPBZP requires explicit sync 269 + * between setting BANK and accessing branch records. 270 + */ 271 + isb(); 272 + } 273 + 274 + static bool __read_brbe_regset(struct brbe_regset *entry, int idx) 275 + { 276 + entry->brbinf = get_brbinf_reg(idx); 277 + 278 + if (brbe_invalid(entry->brbinf)) 279 + return false; 280 + 281 + entry->brbsrc = get_brbsrc_reg(idx); 282 + entry->brbtgt = get_brbtgt_reg(idx); 283 + return true; 284 + } 285 + 286 + /* 287 + * Generic perf branch filters supported on BRBE 288 + * 289 + * New branch filters need to be evaluated whether they could be supported on 290 + * BRBE. This ensures that such branch filters would not just be accepted, to 291 + * fail silently. PERF_SAMPLE_BRANCH_HV is a special case that is selectively 292 + * supported only on platforms where kernel is in hyp mode. 293 + */ 294 + #define BRBE_EXCLUDE_BRANCH_FILTERS (PERF_SAMPLE_BRANCH_ABORT_TX | \ 295 + PERF_SAMPLE_BRANCH_IN_TX | \ 296 + PERF_SAMPLE_BRANCH_NO_TX | \ 297 + PERF_SAMPLE_BRANCH_CALL_STACK | \ 298 + PERF_SAMPLE_BRANCH_COUNTERS) 299 + 300 + #define BRBE_ALLOWED_BRANCH_TYPES (PERF_SAMPLE_BRANCH_ANY | \ 301 + PERF_SAMPLE_BRANCH_ANY_CALL | \ 302 + PERF_SAMPLE_BRANCH_ANY_RETURN | \ 303 + PERF_SAMPLE_BRANCH_IND_CALL | \ 304 + PERF_SAMPLE_BRANCH_COND | \ 305 + PERF_SAMPLE_BRANCH_IND_JUMP | \ 306 + PERF_SAMPLE_BRANCH_CALL) 307 + 308 + 309 + #define BRBE_ALLOWED_BRANCH_FILTERS (PERF_SAMPLE_BRANCH_USER | \ 310 + PERF_SAMPLE_BRANCH_KERNEL | \ 311 + PERF_SAMPLE_BRANCH_HV | \ 312 + BRBE_ALLOWED_BRANCH_TYPES | \ 313 + PERF_SAMPLE_BRANCH_NO_FLAGS | \ 314 + PERF_SAMPLE_BRANCH_NO_CYCLES | \ 315 + PERF_SAMPLE_BRANCH_TYPE_SAVE | \ 316 + PERF_SAMPLE_BRANCH_HW_INDEX | \ 317 + PERF_SAMPLE_BRANCH_PRIV_SAVE) 318 + 319 + #define BRBE_PERF_BRANCH_FILTERS (BRBE_ALLOWED_BRANCH_FILTERS | \ 320 + BRBE_EXCLUDE_BRANCH_FILTERS) 321 + 322 + /* 323 + * BRBE supports the following functional branch type filters while 324 + * generating branch records. These branch filters can be enabled, 325 + * either individually or as a group i.e ORing multiple filters 326 + * with each other. 327 + * 328 + * BRBFCR_EL1_CONDDIR - Conditional direct branch 329 + * BRBFCR_EL1_DIRCALL - Direct call 330 + * BRBFCR_EL1_INDCALL - Indirect call 331 + * BRBFCR_EL1_INDIRECT - Indirect branch 332 + * BRBFCR_EL1_DIRECT - Direct branch 333 + * BRBFCR_EL1_RTN - Subroutine return 334 + */ 335 + static u64 branch_type_to_brbfcr(int branch_type) 336 + { 337 + u64 brbfcr = 0; 338 + 339 + if (branch_type & PERF_SAMPLE_BRANCH_ANY) { 340 + brbfcr |= BRBFCR_EL1_BRANCH_FILTERS; 341 + return brbfcr; 342 + } 343 + 344 + if (branch_type & PERF_SAMPLE_BRANCH_ANY_CALL) { 345 + brbfcr |= BRBFCR_EL1_INDCALL; 346 + brbfcr |= BRBFCR_EL1_DIRCALL; 347 + } 348 + 349 + if (branch_type & PERF_SAMPLE_BRANCH_ANY_RETURN) 350 + brbfcr |= BRBFCR_EL1_RTN; 351 + 352 + if (branch_type & PERF_SAMPLE_BRANCH_IND_CALL) 353 + brbfcr |= BRBFCR_EL1_INDCALL; 354 + 355 + if (branch_type & PERF_SAMPLE_BRANCH_COND) 356 + brbfcr |= BRBFCR_EL1_CONDDIR; 357 + 358 + if (branch_type & PERF_SAMPLE_BRANCH_IND_JUMP) 359 + brbfcr |= BRBFCR_EL1_INDIRECT; 360 + 361 + if (branch_type & PERF_SAMPLE_BRANCH_CALL) 362 + brbfcr |= BRBFCR_EL1_DIRCALL; 363 + 364 + return brbfcr; 365 + } 366 + 367 + /* 368 + * BRBE supports the following privilege mode filters while generating 369 + * branch records. 370 + * 371 + * BRBCR_ELx_E0BRE - EL0 branch records 372 + * BRBCR_ELx_ExBRE - EL1/EL2 branch records 373 + * 374 + * BRBE also supports the following additional functional branch type 375 + * filters while generating branch records. 376 + * 377 + * BRBCR_ELx_EXCEPTION - Exception 378 + * BRBCR_ELx_ERTN - Exception return 379 + */ 380 + static u64 branch_type_to_brbcr(int branch_type) 381 + { 382 + u64 brbcr = BRBCR_ELx_FZP | BRBCR_ELx_DEFAULT_TS; 383 + 384 + if (branch_type & PERF_SAMPLE_BRANCH_USER) 385 + brbcr |= BRBCR_ELx_E0BRE; 386 + 387 + /* 388 + * When running in the hyp mode, writing into BRBCR_EL1 389 + * actually writes into BRBCR_EL2 instead. Field E2BRE 390 + * is also at the same position as E1BRE. 391 + */ 392 + if (branch_type & PERF_SAMPLE_BRANCH_KERNEL) 393 + brbcr |= BRBCR_ELx_ExBRE; 394 + 395 + if (branch_type & PERF_SAMPLE_BRANCH_HV) { 396 + if (is_kernel_in_hyp_mode()) 397 + brbcr |= BRBCR_ELx_ExBRE; 398 + } 399 + 400 + if (!(branch_type & PERF_SAMPLE_BRANCH_NO_CYCLES)) 401 + brbcr |= BRBCR_ELx_CC; 402 + 403 + if (!(branch_type & PERF_SAMPLE_BRANCH_NO_FLAGS)) 404 + brbcr |= BRBCR_ELx_MPRED; 405 + 406 + /* 407 + * The exception and exception return branches could be 408 + * captured, irrespective of the perf event's privilege. 409 + * If the perf event does not have enough privilege for 410 + * a given exception level, then addresses which falls 411 + * under that exception level will be reported as zero 412 + * for the captured branch record, creating source only 413 + * or target only records. 414 + */ 415 + if (branch_type & PERF_SAMPLE_BRANCH_KERNEL) { 416 + if (branch_type & PERF_SAMPLE_BRANCH_ANY) { 417 + brbcr |= BRBCR_ELx_EXCEPTION; 418 + brbcr |= BRBCR_ELx_ERTN; 419 + } 420 + 421 + if (branch_type & PERF_SAMPLE_BRANCH_ANY_CALL) 422 + brbcr |= BRBCR_ELx_EXCEPTION; 423 + 424 + if (branch_type & PERF_SAMPLE_BRANCH_ANY_RETURN) 425 + brbcr |= BRBCR_ELx_ERTN; 426 + } 427 + return brbcr; 428 + } 429 + 430 + bool brbe_branch_attr_valid(struct perf_event *event) 431 + { 432 + u64 branch_type = event->attr.branch_sample_type; 433 + 434 + /* 435 + * Ensure both perf branch filter allowed and exclude 436 + * masks are always in sync with the generic perf ABI. 437 + */ 438 + BUILD_BUG_ON(BRBE_PERF_BRANCH_FILTERS != (PERF_SAMPLE_BRANCH_MAX - 1)); 439 + 440 + if (branch_type & BRBE_EXCLUDE_BRANCH_FILTERS) { 441 + pr_debug("requested branch filter not supported 0x%llx\n", branch_type); 442 + return false; 443 + } 444 + 445 + /* Ensure at least 1 branch type is enabled */ 446 + if (!(branch_type & BRBE_ALLOWED_BRANCH_TYPES)) { 447 + pr_debug("no branch type enabled 0x%llx\n", branch_type); 448 + return false; 449 + } 450 + 451 + /* 452 + * No branches are recorded in guests nor nVHE hypervisors, so 453 + * excluding the host or both kernel and user is invalid. 454 + * 455 + * Ideally we'd just require exclude_guest and exclude_hv, but setting 456 + * event filters with perf for kernel or user don't set exclude_guest. 457 + * So effectively, exclude_guest and exclude_hv are ignored. 458 + */ 459 + if (event->attr.exclude_host || (event->attr.exclude_user && event->attr.exclude_kernel)) { 460 + pr_debug("branch filter in hypervisor or guest only not supported 0x%llx\n", branch_type); 461 + return false; 462 + } 463 + 464 + event->hw.branch_reg.config = branch_type_to_brbfcr(event->attr.branch_sample_type); 465 + event->hw.extra_reg.config = branch_type_to_brbcr(event->attr.branch_sample_type); 466 + 467 + return true; 468 + } 469 + 470 + unsigned int brbe_num_branch_records(const struct arm_pmu *armpmu) 471 + { 472 + return FIELD_GET(BRBIDR0_EL1_NUMREC_MASK, armpmu->reg_brbidr); 473 + } 474 + 475 + void brbe_probe(struct arm_pmu *armpmu) 476 + { 477 + u64 brbidr, aa64dfr0 = read_sysreg_s(SYS_ID_AA64DFR0_EL1); 478 + u32 brbe; 479 + 480 + brbe = cpuid_feature_extract_unsigned_field(aa64dfr0, ID_AA64DFR0_EL1_BRBE_SHIFT); 481 + if (!valid_brbe_version(brbe)) 482 + return; 483 + 484 + brbidr = read_sysreg_s(SYS_BRBIDR0_EL1); 485 + if (!valid_brbidr(brbidr)) 486 + return; 487 + 488 + armpmu->reg_brbidr = brbidr; 489 + } 490 + 491 + /* 492 + * BRBE is assumed to be disabled/paused on entry 493 + */ 494 + void brbe_enable(const struct arm_pmu *arm_pmu) 495 + { 496 + struct pmu_hw_events *cpuc = this_cpu_ptr(arm_pmu->hw_events); 497 + u64 brbfcr = 0, brbcr = 0; 498 + 499 + /* 500 + * Discard existing records to avoid a discontinuity, e.g. records 501 + * missed during handling an overflow. 502 + */ 503 + brbe_invalidate(); 504 + 505 + /* 506 + * Merge the permitted branch filters of all events. 507 + */ 508 + for (int i = 0; i < ARMPMU_MAX_HWEVENTS; i++) { 509 + struct perf_event *event = cpuc->events[i]; 510 + 511 + if (event && has_branch_stack(event)) { 512 + brbfcr |= event->hw.branch_reg.config; 513 + brbcr |= event->hw.extra_reg.config; 514 + } 515 + } 516 + 517 + /* 518 + * In VHE mode with MDCR_EL2.HPMN equal to PMCR_EL0.N, BRBCR_EL1.FZP 519 + * controls freezing the branch records on counter overflow rather than 520 + * BRBCR_EL2.FZP (which writes to BRBCR_EL1 are redirected to). 521 + * The exception levels are enabled/disabled in BRBCR_EL2, so keep EL1 522 + * and EL0 recording disabled for guests. 523 + * 524 + * As BRBCR_EL1 CC and MPRED bits also need to match, use the same 525 + * value for both registers just masking the exception levels. 526 + */ 527 + if (is_kernel_in_hyp_mode()) 528 + write_sysreg_s(brbcr & ~(BRBCR_ELx_ExBRE | BRBCR_ELx_E0BRE), SYS_BRBCR_EL12); 529 + write_sysreg_s(brbcr, SYS_BRBCR_EL1); 530 + /* Ensure BRBCR_ELx settings take effect before unpausing */ 531 + isb(); 532 + 533 + /* Finally write SYS_BRBFCR_EL to unpause BRBE */ 534 + write_sysreg_s(brbfcr, SYS_BRBFCR_EL1); 535 + /* Synchronization in PMCR write ensures ordering WRT PMU enabling */ 536 + } 537 + 538 + void brbe_disable(void) 539 + { 540 + /* 541 + * No need for synchronization here as synchronization in PMCR write 542 + * ensures ordering and in the interrupt handler this is a NOP as 543 + * we're already paused. 544 + */ 545 + write_sysreg_s(BRBFCR_EL1_PAUSED, SYS_BRBFCR_EL1); 546 + write_sysreg_s(0, SYS_BRBCR_EL1); 547 + } 548 + 549 + static const int brbe_type_to_perf_type_map[BRBINFx_EL1_TYPE_DEBUG_EXIT + 1][2] = { 550 + [BRBINFx_EL1_TYPE_DIRECT_UNCOND] = { PERF_BR_UNCOND, 0 }, 551 + [BRBINFx_EL1_TYPE_INDIRECT] = { PERF_BR_IND, 0 }, 552 + [BRBINFx_EL1_TYPE_DIRECT_LINK] = { PERF_BR_CALL, 0 }, 553 + [BRBINFx_EL1_TYPE_INDIRECT_LINK] = { PERF_BR_IND_CALL, 0 }, 554 + [BRBINFx_EL1_TYPE_RET] = { PERF_BR_RET, 0 }, 555 + [BRBINFx_EL1_TYPE_DIRECT_COND] = { PERF_BR_COND, 0 }, 556 + [BRBINFx_EL1_TYPE_CALL] = { PERF_BR_SYSCALL, 0 }, 557 + [BRBINFx_EL1_TYPE_ERET] = { PERF_BR_ERET, 0 }, 558 + [BRBINFx_EL1_TYPE_IRQ] = { PERF_BR_IRQ, 0 }, 559 + [BRBINFx_EL1_TYPE_TRAP] = { PERF_BR_IRQ, 0 }, 560 + [BRBINFx_EL1_TYPE_SERROR] = { PERF_BR_SERROR, 0 }, 561 + [BRBINFx_EL1_TYPE_ALIGN_FAULT] = { PERF_BR_EXTEND_ABI, PERF_BR_NEW_FAULT_ALGN }, 562 + [BRBINFx_EL1_TYPE_INSN_FAULT] = { PERF_BR_EXTEND_ABI, PERF_BR_NEW_FAULT_INST }, 563 + [BRBINFx_EL1_TYPE_DATA_FAULT] = { PERF_BR_EXTEND_ABI, PERF_BR_NEW_FAULT_DATA }, 564 + }; 565 + 566 + static void brbe_set_perf_entry_type(struct perf_branch_entry *entry, u64 brbinf) 567 + { 568 + int brbe_type = brbinf_get_type(brbinf); 569 + 570 + if (brbe_type <= BRBINFx_EL1_TYPE_DEBUG_EXIT) { 571 + const int *br_type = brbe_type_to_perf_type_map[brbe_type]; 572 + 573 + entry->type = br_type[0]; 574 + entry->new_type = br_type[1]; 575 + } 576 + } 577 + 578 + static int brbinf_get_perf_priv(u64 brbinf) 579 + { 580 + int brbe_el = brbinf_get_el(brbinf); 581 + 582 + switch (brbe_el) { 583 + case BRBINFx_EL1_EL_EL0: 584 + return PERF_BR_PRIV_USER; 585 + case BRBINFx_EL1_EL_EL1: 586 + return PERF_BR_PRIV_KERNEL; 587 + case BRBINFx_EL1_EL_EL2: 588 + if (is_kernel_in_hyp_mode()) 589 + return PERF_BR_PRIV_KERNEL; 590 + return PERF_BR_PRIV_HV; 591 + default: 592 + pr_warn_once("%d - unknown branch privilege captured\n", brbe_el); 593 + return PERF_BR_PRIV_UNKNOWN; 594 + } 595 + } 596 + 597 + static bool perf_entry_from_brbe_regset(int index, struct perf_branch_entry *entry, 598 + const struct perf_event *event) 599 + { 600 + struct brbe_regset bregs; 601 + u64 brbinf; 602 + 603 + if (!__read_brbe_regset(&bregs, index)) 604 + return false; 605 + 606 + brbinf = bregs.brbinf; 607 + perf_clear_branch_entry_bitfields(entry); 608 + if (brbe_record_is_complete(brbinf)) { 609 + entry->from = bregs.brbsrc; 610 + entry->to = bregs.brbtgt; 611 + } else if (brbe_record_is_source_only(brbinf)) { 612 + entry->from = bregs.brbsrc; 613 + entry->to = 0; 614 + } else if (brbe_record_is_target_only(brbinf)) { 615 + entry->from = 0; 616 + entry->to = bregs.brbtgt; 617 + } 618 + 619 + brbe_set_perf_entry_type(entry, brbinf); 620 + 621 + if (!branch_sample_no_cycles(event)) 622 + entry->cycles = brbinf_get_cycles(brbinf); 623 + 624 + if (!branch_sample_no_flags(event)) { 625 + /* Mispredict info is available for source only and complete branch records. */ 626 + if (!brbe_record_is_target_only(brbinf)) { 627 + entry->mispred = brbinf_get_mispredict(brbinf); 628 + entry->predicted = !entry->mispred; 629 + } 630 + 631 + /* 632 + * Currently TME feature is neither implemented in any hardware 633 + * nor it is being supported in the kernel. Just warn here once 634 + * if TME related information shows up rather unexpectedly. 635 + */ 636 + if (brbinf_get_lastfailed(brbinf) || brbinf_get_in_tx(brbinf)) 637 + pr_warn_once("Unknown transaction states\n"); 638 + } 639 + 640 + /* 641 + * Branch privilege level is available for target only and complete 642 + * branch records. 643 + */ 644 + if (!brbe_record_is_source_only(brbinf)) 645 + entry->priv = brbinf_get_perf_priv(brbinf); 646 + 647 + return true; 648 + } 649 + 650 + #define PERF_BR_ARM64_ALL ( \ 651 + BIT(PERF_BR_COND) | \ 652 + BIT(PERF_BR_UNCOND) | \ 653 + BIT(PERF_BR_IND) | \ 654 + BIT(PERF_BR_CALL) | \ 655 + BIT(PERF_BR_IND_CALL) | \ 656 + BIT(PERF_BR_RET)) 657 + 658 + #define PERF_BR_ARM64_ALL_KERNEL ( \ 659 + BIT(PERF_BR_SYSCALL) | \ 660 + BIT(PERF_BR_IRQ) | \ 661 + BIT(PERF_BR_SERROR) | \ 662 + BIT(PERF_BR_MAX + PERF_BR_NEW_FAULT_ALGN) | \ 663 + BIT(PERF_BR_MAX + PERF_BR_NEW_FAULT_DATA) | \ 664 + BIT(PERF_BR_MAX + PERF_BR_NEW_FAULT_INST)) 665 + 666 + static void prepare_event_branch_type_mask(u64 branch_sample, 667 + unsigned long *event_type_mask) 668 + { 669 + if (branch_sample & PERF_SAMPLE_BRANCH_ANY) { 670 + if (branch_sample & PERF_SAMPLE_BRANCH_KERNEL) 671 + bitmap_from_u64(event_type_mask, 672 + BIT(PERF_BR_ERET) | PERF_BR_ARM64_ALL | 673 + PERF_BR_ARM64_ALL_KERNEL); 674 + else 675 + bitmap_from_u64(event_type_mask, PERF_BR_ARM64_ALL); 676 + return; 677 + } 678 + 679 + bitmap_zero(event_type_mask, PERF_BR_ARM64_MAX); 680 + 681 + if (branch_sample & PERF_SAMPLE_BRANCH_ANY_CALL) { 682 + if (branch_sample & PERF_SAMPLE_BRANCH_KERNEL) 683 + bitmap_from_u64(event_type_mask, PERF_BR_ARM64_ALL_KERNEL); 684 + 685 + set_bit(PERF_BR_CALL, event_type_mask); 686 + set_bit(PERF_BR_IND_CALL, event_type_mask); 687 + } 688 + 689 + if (branch_sample & PERF_SAMPLE_BRANCH_IND_JUMP) 690 + set_bit(PERF_BR_IND, event_type_mask); 691 + 692 + if (branch_sample & PERF_SAMPLE_BRANCH_COND) 693 + set_bit(PERF_BR_COND, event_type_mask); 694 + 695 + if (branch_sample & PERF_SAMPLE_BRANCH_CALL) 696 + set_bit(PERF_BR_CALL, event_type_mask); 697 + 698 + if (branch_sample & PERF_SAMPLE_BRANCH_IND_CALL) 699 + set_bit(PERF_BR_IND_CALL, event_type_mask); 700 + 701 + if (branch_sample & PERF_SAMPLE_BRANCH_ANY_RETURN) { 702 + set_bit(PERF_BR_RET, event_type_mask); 703 + 704 + if (branch_sample & PERF_SAMPLE_BRANCH_KERNEL) 705 + set_bit(PERF_BR_ERET, event_type_mask); 706 + } 707 + } 708 + 709 + /* 710 + * BRBE is configured with an OR of permissions from all events, so there may 711 + * be events which have to be dropped or events where just the source or target 712 + * address has to be zeroed. 713 + */ 714 + static bool filter_branch_privilege(struct perf_branch_entry *entry, u64 branch_sample_type) 715 + { 716 + bool from_user = access_ok((void __user *)(unsigned long)entry->from, 4); 717 + bool to_user = access_ok((void __user *)(unsigned long)entry->to, 4); 718 + bool exclude_kernel = !((branch_sample_type & PERF_SAMPLE_BRANCH_KERNEL) || 719 + (is_kernel_in_hyp_mode() && (branch_sample_type & PERF_SAMPLE_BRANCH_HV))); 720 + 721 + /* We can only have a half record if permissions have not been expanded */ 722 + if (!entry->from || !entry->to) 723 + return true; 724 + 725 + /* 726 + * If record is within a single exception level, just need to either 727 + * drop or keep the entire record. 728 + */ 729 + if (from_user == to_user) 730 + return ((entry->priv == PERF_BR_PRIV_KERNEL) && !exclude_kernel) || 731 + ((entry->priv == PERF_BR_PRIV_USER) && 732 + (branch_sample_type & PERF_SAMPLE_BRANCH_USER)); 733 + 734 + /* 735 + * Record is across exception levels, mask addresses for the exception 736 + * level we're not capturing. 737 + */ 738 + if (!(branch_sample_type & PERF_SAMPLE_BRANCH_USER)) { 739 + if (from_user) 740 + entry->from = 0; 741 + if (to_user) 742 + entry->to = 0; 743 + } 744 + 745 + if (exclude_kernel) { 746 + if (!from_user) 747 + entry->from = 0; 748 + if (!to_user) 749 + entry->to = 0; 750 + } 751 + 752 + return true; 753 + } 754 + 755 + static bool filter_branch_type(struct perf_branch_entry *entry, 756 + const unsigned long *event_type_mask) 757 + { 758 + if (entry->type == PERF_BR_EXTEND_ABI) 759 + return test_bit(PERF_BR_MAX + entry->new_type, event_type_mask); 760 + else 761 + return test_bit(entry->type, event_type_mask); 762 + } 763 + 764 + static bool filter_branch_record(struct perf_branch_entry *entry, 765 + u64 branch_sample, 766 + const unsigned long *event_type_mask) 767 + { 768 + return filter_branch_type(entry, event_type_mask) && 769 + filter_branch_privilege(entry, branch_sample); 770 + } 771 + 772 + void brbe_read_filtered_entries(struct perf_branch_stack *branch_stack, 773 + const struct perf_event *event) 774 + { 775 + struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu); 776 + int nr_hw = brbe_num_branch_records(cpu_pmu); 777 + int nr_banks = DIV_ROUND_UP(nr_hw, BRBE_BANK_MAX_ENTRIES); 778 + int nr_filtered = 0; 779 + u64 branch_sample_type = event->attr.branch_sample_type; 780 + DECLARE_BITMAP(event_type_mask, PERF_BR_ARM64_MAX); 781 + 782 + prepare_event_branch_type_mask(branch_sample_type, event_type_mask); 783 + 784 + for (int bank = 0; bank < nr_banks; bank++) { 785 + int nr_remaining = nr_hw - (bank * BRBE_BANK_MAX_ENTRIES); 786 + int nr_this_bank = min(nr_remaining, BRBE_BANK_MAX_ENTRIES); 787 + 788 + select_brbe_bank(bank); 789 + 790 + for (int i = 0; i < nr_this_bank; i++) { 791 + struct perf_branch_entry *pbe = &branch_stack->entries[nr_filtered]; 792 + 793 + if (!perf_entry_from_brbe_regset(i, pbe, event)) 794 + goto done; 795 + 796 + if (!filter_branch_record(pbe, branch_sample_type, event_type_mask)) 797 + continue; 798 + 799 + nr_filtered++; 800 + } 801 + } 802 + 803 + done: 804 + branch_stack->nr = nr_filtered; 805 + }

+47

drivers/perf/arm_brbe.h

··· 1 + /* SPDX-License-Identifier: GPL-2.0-only */ 2 + /* 3 + * Branch Record Buffer Extension Helpers. 4 + * 5 + * Copyright (C) 2022-2025 ARM Limited 6 + * 7 + * Author: Anshuman Khandual <anshuman.khandual@arm.com> 8 + */ 9 + 10 + struct arm_pmu; 11 + struct perf_branch_stack; 12 + struct perf_event; 13 + 14 + #ifdef CONFIG_ARM64_BRBE 15 + void brbe_probe(struct arm_pmu *arm_pmu); 16 + unsigned int brbe_num_branch_records(const struct arm_pmu *armpmu); 17 + void brbe_invalidate(void); 18 + 19 + void brbe_enable(const struct arm_pmu *arm_pmu); 20 + void brbe_disable(void); 21 + 22 + bool brbe_branch_attr_valid(struct perf_event *event); 23 + void brbe_read_filtered_entries(struct perf_branch_stack *branch_stack, 24 + const struct perf_event *event); 25 + #else 26 + static inline void brbe_probe(struct arm_pmu *arm_pmu) { } 27 + static inline unsigned int brbe_num_branch_records(const struct arm_pmu *armpmu) 28 + { 29 + return 0; 30 + } 31 + 32 + static inline void brbe_invalidate(void) { } 33 + 34 + static inline void brbe_enable(const struct arm_pmu *arm_pmu) { }; 35 + static inline void brbe_disable(void) { }; 36 + 37 + static inline bool brbe_branch_attr_valid(struct perf_event *event) 38 + { 39 + WARN_ON_ONCE(!has_branch_stack(event)); 40 + return false; 41 + } 42 + 43 + static void brbe_read_filtered_entries(struct perf_branch_stack *branch_stack, 44 + const struct perf_event *event) 45 + { 46 + } 47 + #endif

+13 -3

drivers/perf/arm_pmu.c

··· 99 99 .free_pmuirq = armpmu_free_percpu_pmunmi 100 100 }; 101 101 102 - static DEFINE_PER_CPU(struct arm_pmu *, cpu_armpmu); 102 + DEFINE_PER_CPU(struct arm_pmu *, cpu_armpmu); 103 103 static DEFINE_PER_CPU(int, cpu_irq); 104 104 static DEFINE_PER_CPU(const struct pmu_irq_ops *, cpu_irq_ops); 105 105 ··· 318 318 int idx = hwc->idx; 319 319 320 320 armpmu_stop(event, PERF_EF_UPDATE); 321 + 322 + if (has_branch_stack(event)) { 323 + hw_events->branch_users--; 324 + perf_sched_cb_dec(event->pmu); 325 + } 326 + 321 327 hw_events->events[idx] = NULL; 322 328 armpmu->clear_event_idx(hw_events, event); 323 329 perf_event_update_userpage(event); ··· 350 344 351 345 /* The newly-allocated counter should be empty */ 352 346 WARN_ON_ONCE(hw_events->events[idx]); 347 + 348 + if (has_branch_stack(event)) { 349 + hw_events->branch_users++; 350 + perf_sched_cb_inc(event->pmu); 351 + } 353 352 354 353 event->hw.idx = idx; 355 354 hw_events->events[idx] = event; ··· 520 509 !cpumask_test_cpu(event->cpu, &armpmu->supported_cpus)) 521 510 return -ENOENT; 522 511 523 - /* does not support taken branch sampling */ 524 - if (has_branch_stack(event)) 512 + if (has_branch_stack(event) && !armpmu->reg_brbidr) 525 513 return -EOPNOTSUPP; 526 514 527 515 return __hw_perf_event_init(event);

+105 -2

drivers/perf/arm_pmuv3.c

··· 25 25 #include <linux/smp.h> 26 26 #include <linux/nmi.h> 27 27 28 + #include "arm_brbe.h" 29 + 28 30 /* ARMv8 Cortex-A53 specific event types. */ 29 31 #define ARMV8_A53_PERFCTR_PREF_LINEFILL 0xC2 30 32 ··· 440 438 441 439 static DEVICE_ATTR_RO(threshold_max); 442 440 441 + static ssize_t branches_show(struct device *dev, 442 + struct device_attribute *attr, char *page) 443 + { 444 + struct pmu *pmu = dev_get_drvdata(dev); 445 + struct arm_pmu *cpu_pmu = container_of(pmu, struct arm_pmu, pmu); 446 + 447 + return sysfs_emit(page, "%d\n", brbe_num_branch_records(cpu_pmu)); 448 + } 449 + 450 + static DEVICE_ATTR_RO(branches); 451 + 443 452 static struct attribute *armv8_pmuv3_caps_attrs[] = { 453 + &dev_attr_branches.attr, 444 454 &dev_attr_slots.attr, 445 455 &dev_attr_bus_slots.attr, 446 456 &dev_attr_bus_width.attr, ··· 460 446 NULL, 461 447 }; 462 448 449 + static umode_t caps_is_visible(struct kobject *kobj, struct attribute *attr, int i) 450 + { 451 + struct device *dev = kobj_to_dev(kobj); 452 + struct pmu *pmu = dev_get_drvdata(dev); 453 + struct arm_pmu *cpu_pmu = container_of(pmu, struct arm_pmu, pmu); 454 + 455 + if (i == 0) 456 + return brbe_num_branch_records(cpu_pmu) ? attr->mode : 0; 457 + 458 + return attr->mode; 459 + } 460 + 463 461 static const struct attribute_group armv8_pmuv3_caps_attr_group = { 464 462 .name = "caps", 465 463 .attrs = armv8_pmuv3_caps_attrs, 464 + .is_visible = caps_is_visible, 466 465 }; 467 466 468 467 /* ··· 836 809 static void armv8pmu_start(struct arm_pmu *cpu_pmu) 837 810 { 838 811 struct perf_event_context *ctx; 812 + struct pmu_hw_events *hw_events = this_cpu_ptr(cpu_pmu->hw_events); 839 813 int nr_user = 0; 840 814 841 815 ctx = perf_cpu_task_ctx(); ··· 850 822 851 823 kvm_vcpu_pmu_resync_el0(); 852 824 825 + if (hw_events->branch_users) 826 + brbe_enable(cpu_pmu); 827 + 853 828 /* Enable all counters */ 854 829 armv8pmu_pmcr_write(armv8pmu_pmcr_read() | ARMV8_PMU_PMCR_E); 855 830 } 856 831 857 832 static void armv8pmu_stop(struct arm_pmu *cpu_pmu) 858 833 { 834 + struct pmu_hw_events *hw_events = this_cpu_ptr(cpu_pmu->hw_events); 835 + 836 + if (hw_events->branch_users) 837 + brbe_disable(); 838 + 859 839 /* Disable all counters */ 860 840 armv8pmu_pmcr_write(armv8pmu_pmcr_read() & ~ARMV8_PMU_PMCR_E); 841 + } 842 + 843 + static void read_branch_records(struct pmu_hw_events *cpuc, 844 + struct perf_event *event, 845 + struct perf_sample_data *data) 846 + { 847 + struct perf_branch_stack *branch_stack = cpuc->branch_stack; 848 + 849 + brbe_read_filtered_entries(branch_stack, event); 850 + perf_sample_save_brstack(data, event, branch_stack, NULL); 861 851 } 862 852 863 853 static irqreturn_t armv8pmu_handle_irq(struct arm_pmu *cpu_pmu) ··· 927 881 perf_sample_data_init(&data, 0, hwc->last_period); 928 882 if (!armpmu_event_set_period(event)) 929 883 continue; 884 + 885 + if (has_branch_stack(event)) 886 + read_branch_records(cpuc, event, &data); 930 887 931 888 /* 932 889 * Perf event overflow will queue the processing of the event as ··· 987 938 988 939 /* Always prefer to place a cycle counter into the cycle counter. */ 989 940 if ((evtype == ARMV8_PMUV3_PERFCTR_CPU_CYCLES) && 990 - !armv8pmu_event_get_threshold(&event->attr)) { 941 + !armv8pmu_event_get_threshold(&event->attr) && !has_branch_stack(event)) { 991 942 if (!test_and_set_bit(ARMV8_PMU_CYCLE_IDX, cpuc->used_mask)) 992 943 return ARMV8_PMU_CYCLE_IDX; 993 944 else if (armv8pmu_event_is_64bit(event) && ··· 1036 987 return event->hw.idx + 1; 1037 988 } 1038 989 990 + static void armv8pmu_sched_task(struct perf_event_pmu_context *pmu_ctx, 991 + struct task_struct *task, bool sched_in) 992 + { 993 + struct arm_pmu *armpmu = *this_cpu_ptr(&cpu_armpmu); 994 + struct pmu_hw_events *hw_events = this_cpu_ptr(armpmu->hw_events); 995 + 996 + if (!hw_events->branch_users) 997 + return; 998 + 999 + if (sched_in) 1000 + brbe_invalidate(); 1001 + } 1002 + 1039 1003 /* 1040 1004 * Add an event filter to a given event. 1041 1005 */ ··· 1064 1002 if (attr->exclude_idle) { 1065 1003 pr_debug("ARM performance counters do not support mode exclusion\n"); 1066 1004 return -EOPNOTSUPP; 1005 + } 1006 + 1007 + if (has_branch_stack(perf_event)) { 1008 + if (!brbe_num_branch_records(cpu_pmu) || !brbe_branch_attr_valid(perf_event)) 1009 + return -EOPNOTSUPP; 1010 + 1011 + perf_event->attach_state |= PERF_ATTACH_SCHED_CB; 1067 1012 } 1068 1013 1069 1014 /* ··· 1138 1069 1139 1070 /* Clear the counters we flip at guest entry/exit */ 1140 1071 kvm_clr_pmu_events(mask); 1072 + 1073 + if (brbe_num_branch_records(cpu_pmu)) { 1074 + brbe_disable(); 1075 + brbe_invalidate(); 1076 + } 1141 1077 1142 1078 /* 1143 1079 * Initialize & Reset PMNC. Request overflow interrupt for ··· 1312 1238 cpu_pmu->reg_pmmir = read_pmmir(); 1313 1239 else 1314 1240 cpu_pmu->reg_pmmir = 0; 1241 + 1242 + brbe_probe(cpu_pmu); 1243 + } 1244 + 1245 + static int branch_records_alloc(struct arm_pmu *armpmu) 1246 + { 1247 + size_t size = struct_size_t(struct perf_branch_stack, entries, 1248 + brbe_num_branch_records(armpmu)); 1249 + int cpu; 1250 + 1251 + for_each_cpu(cpu, &armpmu->supported_cpus) { 1252 + struct pmu_hw_events *events_cpu; 1253 + 1254 + events_cpu = per_cpu_ptr(armpmu->hw_events, cpu); 1255 + events_cpu->branch_stack = kmalloc(size, GFP_KERNEL); 1256 + if (!events_cpu->branch_stack) 1257 + return -ENOMEM; 1258 + } 1259 + return 0; 1315 1260 } 1316 1261 1317 1262 static int armv8pmu_probe_pmu(struct arm_pmu *cpu_pmu) ··· 1347 1254 if (ret) 1348 1255 return ret; 1349 1256 1350 - return probe.present ? 0 : -ENODEV; 1257 + if (!probe.present) 1258 + return -ENODEV; 1259 + 1260 + if (brbe_num_branch_records(cpu_pmu)) { 1261 + ret = branch_records_alloc(cpu_pmu); 1262 + if (ret) 1263 + return ret; 1264 + } 1265 + return 0; 1351 1266 } 1352 1267 1353 1268 static void armv8pmu_disable_user_access_ipi(void *unused) ··· 1414 1313 cpu_pmu->set_event_filter = armv8pmu_set_event_filter; 1415 1314 1416 1315 cpu_pmu->pmu.event_idx = armv8pmu_user_event_idx; 1316 + if (brbe_num_branch_records(cpu_pmu)) 1317 + cpu_pmu->pmu.sched_task = armv8pmu_sched_task; 1417 1318 1418 1319 cpu_pmu->name = name; 1419 1320 cpu_pmu->map_event = map_event;

+11 -7

drivers/perf/arm_spe_pmu.c

··· 308 308 309 309 static void arm_spe_event_sanitise_period(struct perf_event *event) 310 310 { 311 - struct arm_spe_pmu *spe_pmu = to_spe_pmu(event->pmu); 312 311 u64 period = event->hw.sample_period; 313 312 u64 max_period = PMSIRR_EL1_INTERVAL_MASK; 314 313 315 - if (period < spe_pmu->min_period) 316 - period = spe_pmu->min_period; 317 - else if (period > max_period) 318 - period = max_period; 319 - else 320 - period &= max_period; 314 + /* 315 + * The PMSIDR_EL1.Interval field (stored in spe_pmu->min_period) is a 316 + * recommendation for the minimum interval, not a hardware limitation. 317 + * 318 + * According to the Arm ARM (DDI 0487 L.a), section D24.7.12 PMSIRR_EL1, 319 + * Sampling Interval Reload Register, the INTERVAL field (bits [31:8]) 320 + * states: "Software must set this to a nonzero value". Use 1 as the 321 + * minimum value. 322 + */ 323 + u64 min_period = FIELD_PREP(PMSIRR_EL1_INTERVAL_MASK, 1); 321 324 325 + period = clamp_t(u64, period, min_period, max_period) & max_period; 322 326 event->hw.sample_period = period; 323 327 } 324 328

+6 -6

drivers/perf/cxl_pmu.c

··· 113 113 114 114 /* 115 115 * All CPMU counters are discoverable via the Event Capabilities Registers. 116 - * Each Event Capability register contains a a VID / GroupID. 116 + * Each Event Capability register contains a VID / GroupID. 117 117 * A counter may then count any combination (by summing) of events in 118 118 * that group which are in the Supported Events Bitmask. 119 119 * However, there are some complexities to the scheme. ··· 406 406 CXL_PMU_EVENT_CXL_ATTR(s2m_bisnp_curblk, CXL_PMU_GID_S2M_BISNP, BIT(4)), 407 407 CXL_PMU_EVENT_CXL_ATTR(s2m_bisnp_datblk, CXL_PMU_GID_S2M_BISNP, BIT(5)), 408 408 CXL_PMU_EVENT_CXL_ATTR(s2m_bisnp_invblk, CXL_PMU_GID_S2M_BISNP, BIT(6)), 409 - /* CXL rev 3.1 Table 3-50 S2M NDR Opcopdes */ 409 + /* CXL rev 3.1 Table 3-50 S2M NDR Opcodes */ 410 410 CXL_PMU_EVENT_CXL_ATTR(s2m_ndr_cmp, CXL_PMU_GID_S2M_NDR, BIT(0)), 411 411 CXL_PMU_EVENT_CXL_ATTR(s2m_ndr_cmps, CXL_PMU_GID_S2M_NDR, BIT(1)), 412 412 CXL_PMU_EVENT_CXL_ATTR(s2m_ndr_cmpe, CXL_PMU_GID_S2M_NDR, BIT(2)), ··· 627 627 hwc->state = 0; 628 628 629 629 /* 630 - * Currently only hdm filter control is implemnted, this code will 630 + * Currently only hdm filter control is implemented, this code will 631 631 * want generalizing when more filters are added. 632 632 */ 633 633 if (info->filter_hdm) { ··· 834 834 if (rc) 835 835 return rc; 836 836 837 - info->hw_events = devm_kcalloc(dev, sizeof(*info->hw_events), 838 - info->num_counters, GFP_KERNEL); 837 + info->hw_events = devm_kcalloc(dev, info->num_counters, 838 + sizeof(*info->hw_events), GFP_KERNEL); 839 839 if (!info->hw_events) 840 840 return -ENOMEM; 841 841 ··· 873 873 return rc; 874 874 irq = rc; 875 875 876 - irq_name = devm_kasprintf(dev, GFP_KERNEL, "%s_overflow\n", dev_name); 876 + irq_name = devm_kasprintf(dev, GFP_KERNEL, "%s_overflow", dev_name); 877 877 if (!irq_name) 878 878 return -ENOMEM; 879 879

+5 -3

drivers/perf/fsl_imx9_ddr_perf.c

··· 461 461 int counter, int axi_id, int axi_mask) 462 462 { 463 463 u32 pmcfg1, pmcfg2; 464 - u32 mask[] = { MX93_PMCFG1_RD_TRANS_FILT_EN, 465 - MX93_PMCFG1_WR_TRANS_FILT_EN, 466 - MX93_PMCFG1_RD_BT_FILT_EN }; 464 + static const u32 mask[] = { 465 + MX93_PMCFG1_RD_TRANS_FILT_EN, 466 + MX93_PMCFG1_WR_TRANS_FILT_EN, 467 + MX93_PMCFG1_RD_BT_FILT_EN 468 + }; 467 469 468 470 pmcfg1 = readl_relaxed(pmu->base + PMCFG1); 469 471

+165 -189

drivers/perf/hisilicon/hisi_uncore_ddrc_pmu.c

··· 43 43 #define DDRC_V2_EVENT_TYPE 0xe74 44 44 #define DDRC_V2_PERF_CTRL 0xeA0 45 45 46 + /* DDRC interrupt registers definition in v3 */ 47 + #define DDRC_V3_INT_MASK 0x534 48 + #define DDRC_V3_INT_STATUS 0x538 49 + #define DDRC_V3_INT_CLEAR 0x53C 50 + 46 51 /* DDRC has 8-counters */ 47 52 #define DDRC_NR_COUNTERS 0x8 48 53 #define DDRC_V1_PERF_CTRL_EN 0x2 49 54 #define DDRC_V2_PERF_CTRL_EN 0x1 50 55 #define DDRC_V1_NR_EVENTS 0x7 51 - #define DDRC_V2_NR_EVENTS 0x90 56 + #define DDRC_V2_NR_EVENTS 0xFF 57 + 58 + #define DDRC_EVENT_CNTn(base, n) ((base) + (n) * 8) 59 + #define DDRC_EVENT_TYPEn(base, n) ((base) + (n) * 4) 60 + #define DDRC_UNIMPLEMENTED_REG GENMASK(31, 0) 52 61 53 62 /* 54 63 * For PMU v1, there are eight-events and every event has been mapped ··· 72 63 DDRC_PRE_CMD, DDRC_ACT_CMD, DDRC_RNK_CHG, DDRC_RW_CHG 73 64 }; 74 65 75 - /* 76 - * Select the counter register offset using the counter index. 77 - * In PMU v1, there are no programmable counter, the count 78 - * is read form the statistics counter register itself. 79 - */ 80 - static u32 hisi_ddrc_pmu_v1_get_counter_offset(int cntr_idx) 81 - { 82 - return ddrc_reg_off[cntr_idx]; 83 - } 66 + struct hisi_ddrc_pmu_regs { 67 + u32 event_cnt; 68 + u32 event_ctrl; 69 + u32 event_type; 70 + u32 perf_ctrl; 71 + u32 perf_ctrl_en; 72 + u32 int_mask; 73 + u32 int_clear; 74 + u32 int_status; 75 + }; 84 76 85 - static u32 hisi_ddrc_pmu_v2_get_counter_offset(int cntr_idx) 86 - { 87 - return DDRC_V2_EVENT_CNT + cntr_idx * 8; 88 - } 89 - 90 - static u64 hisi_ddrc_pmu_v1_read_counter(struct hisi_pmu *ddrc_pmu, 77 + static u64 hisi_ddrc_pmu_read_counter(struct hisi_pmu *ddrc_pmu, 91 78 struct hw_perf_event *hwc) 92 79 { 93 - return readl(ddrc_pmu->base + 94 - hisi_ddrc_pmu_v1_get_counter_offset(hwc->idx)); 80 + struct hisi_ddrc_pmu_regs *regs = ddrc_pmu->dev_info->private; 81 + 82 + if (regs->event_cnt == DDRC_UNIMPLEMENTED_REG) 83 + return readl(ddrc_pmu->base + ddrc_reg_off[hwc->idx]); 84 + 85 + return readq(ddrc_pmu->base + DDRC_EVENT_CNTn(regs->event_cnt, hwc->idx)); 95 86 } 96 87 97 - static void hisi_ddrc_pmu_v1_write_counter(struct hisi_pmu *ddrc_pmu, 88 + static void hisi_ddrc_pmu_write_counter(struct hisi_pmu *ddrc_pmu, 98 89 struct hw_perf_event *hwc, u64 val) 99 90 { 100 - writel((u32)val, 101 - ddrc_pmu->base + hisi_ddrc_pmu_v1_get_counter_offset(hwc->idx)); 102 - } 91 + struct hisi_ddrc_pmu_regs *regs = ddrc_pmu->dev_info->private; 103 92 104 - static u64 hisi_ddrc_pmu_v2_read_counter(struct hisi_pmu *ddrc_pmu, 105 - struct hw_perf_event *hwc) 106 - { 107 - return readq(ddrc_pmu->base + 108 - hisi_ddrc_pmu_v2_get_counter_offset(hwc->idx)); 109 - } 110 - 111 - static void hisi_ddrc_pmu_v2_write_counter(struct hisi_pmu *ddrc_pmu, 112 - struct hw_perf_event *hwc, u64 val) 113 - { 114 - writeq(val, 115 - ddrc_pmu->base + hisi_ddrc_pmu_v2_get_counter_offset(hwc->idx)); 93 + if (regs->event_cnt == DDRC_UNIMPLEMENTED_REG) 94 + writel((u32)val, ddrc_pmu->base + ddrc_reg_off[hwc->idx]); 95 + else 96 + writeq(val, ddrc_pmu->base + DDRC_EVENT_CNTn(regs->event_cnt, hwc->idx)); 116 97 } 117 98 118 99 /* ··· 113 114 static void hisi_ddrc_pmu_write_evtype(struct hisi_pmu *ddrc_pmu, int idx, 114 115 u32 type) 115 116 { 116 - u32 offset; 117 + struct hisi_ddrc_pmu_regs *regs = ddrc_pmu->dev_info->private; 117 118 118 - if (ddrc_pmu->identifier >= HISI_PMU_V2) { 119 - offset = DDRC_V2_EVENT_TYPE + 4 * idx; 120 - writel(type, ddrc_pmu->base + offset); 121 - } 122 - } 119 + if (regs->event_type == DDRC_UNIMPLEMENTED_REG) 120 + return; 123 121 124 - static void hisi_ddrc_pmu_v1_start_counters(struct hisi_pmu *ddrc_pmu) 125 - { 126 - u32 val; 127 - 128 - /* Set perf_enable in DDRC_PERF_CTRL to start event counting */ 129 - val = readl(ddrc_pmu->base + DDRC_PERF_CTRL); 130 - val |= DDRC_V1_PERF_CTRL_EN; 131 - writel(val, ddrc_pmu->base + DDRC_PERF_CTRL); 132 - } 133 - 134 - static void hisi_ddrc_pmu_v1_stop_counters(struct hisi_pmu *ddrc_pmu) 135 - { 136 - u32 val; 137 - 138 - /* Clear perf_enable in DDRC_PERF_CTRL to stop event counting */ 139 - val = readl(ddrc_pmu->base + DDRC_PERF_CTRL); 140 - val &= ~DDRC_V1_PERF_CTRL_EN; 141 - writel(val, ddrc_pmu->base + DDRC_PERF_CTRL); 142 - } 143 - 144 - static void hisi_ddrc_pmu_v1_enable_counter(struct hisi_pmu *ddrc_pmu, 145 - struct hw_perf_event *hwc) 146 - { 147 - u32 val; 148 - 149 - /* Set counter index(event code) in DDRC_EVENT_CTRL register */ 150 - val = readl(ddrc_pmu->base + DDRC_EVENT_CTRL); 151 - val |= (1 << GET_DDRC_EVENTID(hwc)); 152 - writel(val, ddrc_pmu->base + DDRC_EVENT_CTRL); 153 - } 154 - 155 - static void hisi_ddrc_pmu_v1_disable_counter(struct hisi_pmu *ddrc_pmu, 156 - struct hw_perf_event *hwc) 157 - { 158 - u32 val; 159 - 160 - /* Clear counter index(event code) in DDRC_EVENT_CTRL register */ 161 - val = readl(ddrc_pmu->base + DDRC_EVENT_CTRL); 162 - val &= ~(1 << GET_DDRC_EVENTID(hwc)); 163 - writel(val, ddrc_pmu->base + DDRC_EVENT_CTRL); 122 + writel(type, ddrc_pmu->base + DDRC_EVENT_TYPEn(regs->event_type, idx)); 164 123 } 165 124 166 125 static int hisi_ddrc_pmu_v1_get_event_idx(struct perf_event *event) ··· 137 180 return idx; 138 181 } 139 182 140 - static int hisi_ddrc_pmu_v2_get_event_idx(struct perf_event *event) 183 + static int hisi_ddrc_pmu_get_event_idx(struct perf_event *event) 141 184 { 185 + struct hisi_pmu *ddrc_pmu = to_hisi_pmu(event->pmu); 186 + struct hisi_ddrc_pmu_regs *regs = ddrc_pmu->dev_info->private; 187 + 188 + if (regs->event_type == DDRC_UNIMPLEMENTED_REG) 189 + return hisi_ddrc_pmu_v1_get_event_idx(event); 190 + 142 191 return hisi_uncore_pmu_get_event_idx(event); 143 192 } 144 193 145 - static void hisi_ddrc_pmu_v2_start_counters(struct hisi_pmu *ddrc_pmu) 194 + static void hisi_ddrc_pmu_start_counters(struct hisi_pmu *ddrc_pmu) 146 195 { 196 + struct hisi_ddrc_pmu_regs *regs = ddrc_pmu->dev_info->private; 147 197 u32 val; 148 198 149 - val = readl(ddrc_pmu->base + DDRC_V2_PERF_CTRL); 150 - val |= DDRC_V2_PERF_CTRL_EN; 151 - writel(val, ddrc_pmu->base + DDRC_V2_PERF_CTRL); 199 + val = readl(ddrc_pmu->base + regs->perf_ctrl); 200 + val |= regs->perf_ctrl_en; 201 + writel(val, ddrc_pmu->base + regs->perf_ctrl); 152 202 } 153 203 154 - static void hisi_ddrc_pmu_v2_stop_counters(struct hisi_pmu *ddrc_pmu) 204 + static void hisi_ddrc_pmu_stop_counters(struct hisi_pmu *ddrc_pmu) 155 205 { 206 + struct hisi_ddrc_pmu_regs *regs = ddrc_pmu->dev_info->private; 156 207 u32 val; 157 208 158 - val = readl(ddrc_pmu->base + DDRC_V2_PERF_CTRL); 159 - val &= ~DDRC_V2_PERF_CTRL_EN; 160 - writel(val, ddrc_pmu->base + DDRC_V2_PERF_CTRL); 209 + val = readl(ddrc_pmu->base + regs->perf_ctrl); 210 + val &= ~regs->perf_ctrl_en; 211 + writel(val, ddrc_pmu->base + regs->perf_ctrl); 161 212 } 162 213 163 - static void hisi_ddrc_pmu_v2_enable_counter(struct hisi_pmu *ddrc_pmu, 214 + static void hisi_ddrc_pmu_enable_counter(struct hisi_pmu *ddrc_pmu, 164 215 struct hw_perf_event *hwc) 165 216 { 217 + struct hisi_ddrc_pmu_regs *regs = ddrc_pmu->dev_info->private; 166 218 u32 val; 167 219 168 - val = readl(ddrc_pmu->base + DDRC_V2_EVENT_CTRL); 169 - val |= 1 << hwc->idx; 170 - writel(val, ddrc_pmu->base + DDRC_V2_EVENT_CTRL); 220 + val = readl(ddrc_pmu->base + regs->event_ctrl); 221 + val |= BIT_ULL(hwc->idx); 222 + writel(val, ddrc_pmu->base + regs->event_ctrl); 171 223 } 172 224 173 - static void hisi_ddrc_pmu_v2_disable_counter(struct hisi_pmu *ddrc_pmu, 225 + static void hisi_ddrc_pmu_disable_counter(struct hisi_pmu *ddrc_pmu, 174 226 struct hw_perf_event *hwc) 175 227 { 228 + struct hisi_ddrc_pmu_regs *regs = ddrc_pmu->dev_info->private; 176 229 u32 val; 177 230 178 - val = readl(ddrc_pmu->base + DDRC_V2_EVENT_CTRL); 179 - val &= ~(1 << hwc->idx); 180 - writel(val, ddrc_pmu->base + DDRC_V2_EVENT_CTRL); 231 + val = readl(ddrc_pmu->base + regs->event_ctrl); 232 + val &= ~BIT_ULL(hwc->idx); 233 + writel(val, ddrc_pmu->base + regs->event_ctrl); 181 234 } 182 235 183 - static void hisi_ddrc_pmu_v1_enable_counter_int(struct hisi_pmu *ddrc_pmu, 184 - struct hw_perf_event *hwc) 236 + static void hisi_ddrc_pmu_enable_counter_int(struct hisi_pmu *ddrc_pmu, 237 + struct hw_perf_event *hwc) 185 238 { 239 + struct hisi_ddrc_pmu_regs *regs = ddrc_pmu->dev_info->private; 186 240 u32 val; 187 241 188 - /* Write 0 to enable interrupt */ 189 - val = readl(ddrc_pmu->base + DDRC_INT_MASK); 190 - val &= ~(1 << hwc->idx); 191 - writel(val, ddrc_pmu->base + DDRC_INT_MASK); 242 + val = readl(ddrc_pmu->base + regs->int_mask); 243 + val &= ~BIT_ULL(hwc->idx); 244 + writel(val, ddrc_pmu->base + regs->int_mask); 192 245 } 193 246 194 - static void hisi_ddrc_pmu_v1_disable_counter_int(struct hisi_pmu *ddrc_pmu, 195 - struct hw_perf_event *hwc) 247 + static void hisi_ddrc_pmu_disable_counter_int(struct hisi_pmu *ddrc_pmu, 248 + struct hw_perf_event *hwc) 196 249 { 250 + struct hisi_ddrc_pmu_regs *regs = ddrc_pmu->dev_info->private; 197 251 u32 val; 198 252 199 - /* Write 1 to mask interrupt */ 200 - val = readl(ddrc_pmu->base + DDRC_INT_MASK); 201 - val |= 1 << hwc->idx; 202 - writel(val, ddrc_pmu->base + DDRC_INT_MASK); 253 + val = readl(ddrc_pmu->base + regs->int_mask); 254 + val |= BIT_ULL(hwc->idx); 255 + writel(val, ddrc_pmu->base + regs->int_mask); 203 256 } 204 257 205 - static void hisi_ddrc_pmu_v2_enable_counter_int(struct hisi_pmu *ddrc_pmu, 206 - struct hw_perf_event *hwc) 258 + static u32 hisi_ddrc_pmu_get_int_status(struct hisi_pmu *ddrc_pmu) 207 259 { 208 - u32 val; 260 + struct hisi_ddrc_pmu_regs *regs = ddrc_pmu->dev_info->private; 209 261 210 - val = readl(ddrc_pmu->base + DDRC_V2_INT_MASK); 211 - val &= ~(1 << hwc->idx); 212 - writel(val, ddrc_pmu->base + DDRC_V2_INT_MASK); 262 + return readl(ddrc_pmu->base + regs->int_status); 213 263 } 214 264 215 - static void hisi_ddrc_pmu_v2_disable_counter_int(struct hisi_pmu *ddrc_pmu, 216 - struct hw_perf_event *hwc) 265 + static void hisi_ddrc_pmu_clear_int_status(struct hisi_pmu *ddrc_pmu, 266 + int idx) 217 267 { 218 - u32 val; 268 + struct hisi_ddrc_pmu_regs *regs = ddrc_pmu->dev_info->private; 219 269 220 - val = readl(ddrc_pmu->base + DDRC_V2_INT_MASK); 221 - val |= 1 << hwc->idx; 222 - writel(val, ddrc_pmu->base + DDRC_V2_INT_MASK); 270 + writel(1 << idx, ddrc_pmu->base + regs->int_clear); 223 271 } 224 - 225 - static u32 hisi_ddrc_pmu_v1_get_int_status(struct hisi_pmu *ddrc_pmu) 226 - { 227 - return readl(ddrc_pmu->base + DDRC_INT_STATUS); 228 - } 229 - 230 - static void hisi_ddrc_pmu_v1_clear_int_status(struct hisi_pmu *ddrc_pmu, 231 - int idx) 232 - { 233 - writel(1 << idx, ddrc_pmu->base + DDRC_INT_CLEAR); 234 - } 235 - 236 - static u32 hisi_ddrc_pmu_v2_get_int_status(struct hisi_pmu *ddrc_pmu) 237 - { 238 - return readl(ddrc_pmu->base + DDRC_V2_INT_STATUS); 239 - } 240 - 241 - static void hisi_ddrc_pmu_v2_clear_int_status(struct hisi_pmu *ddrc_pmu, 242 - int idx) 243 - { 244 - writel(1 << idx, ddrc_pmu->base + DDRC_V2_INT_CLEAR); 245 - } 246 - 247 - static const struct acpi_device_id hisi_ddrc_pmu_acpi_match[] = { 248 - { "HISI0233", }, 249 - { "HISI0234", }, 250 - {} 251 - }; 252 - MODULE_DEVICE_TABLE(acpi, hisi_ddrc_pmu_acpi_match); 253 272 254 273 static int hisi_ddrc_pmu_init_data(struct platform_device *pdev, 255 274 struct hisi_pmu *ddrc_pmu) ··· 246 313 dev_err(&pdev->dev, "Can not read ddrc sccl-id!\n"); 247 314 return -EINVAL; 248 315 } 316 + 317 + ddrc_pmu->dev_info = device_get_match_data(&pdev->dev); 318 + if (!ddrc_pmu->dev_info) 319 + return -ENODEV; 249 320 250 321 ddrc_pmu->base = devm_platform_ioremap_resource(pdev, 0); 251 322 if (IS_ERR(ddrc_pmu->base)) { ··· 333 396 NULL 334 397 }; 335 398 336 - static const struct hisi_uncore_ops hisi_uncore_ddrc_v1_ops = { 399 + static const struct hisi_uncore_ops hisi_uncore_ddrc_ops = { 337 400 .write_evtype = hisi_ddrc_pmu_write_evtype, 338 - .get_event_idx = hisi_ddrc_pmu_v1_get_event_idx, 339 - .start_counters = hisi_ddrc_pmu_v1_start_counters, 340 - .stop_counters = hisi_ddrc_pmu_v1_stop_counters, 341 - .enable_counter = hisi_ddrc_pmu_v1_enable_counter, 342 - .disable_counter = hisi_ddrc_pmu_v1_disable_counter, 343 - .enable_counter_int = hisi_ddrc_pmu_v1_enable_counter_int, 344 - .disable_counter_int = hisi_ddrc_pmu_v1_disable_counter_int, 345 - .write_counter = hisi_ddrc_pmu_v1_write_counter, 346 - .read_counter = hisi_ddrc_pmu_v1_read_counter, 347 - .get_int_status = hisi_ddrc_pmu_v1_get_int_status, 348 - .clear_int_status = hisi_ddrc_pmu_v1_clear_int_status, 349 - }; 350 - 351 - static const struct hisi_uncore_ops hisi_uncore_ddrc_v2_ops = { 352 - .write_evtype = hisi_ddrc_pmu_write_evtype, 353 - .get_event_idx = hisi_ddrc_pmu_v2_get_event_idx, 354 - .start_counters = hisi_ddrc_pmu_v2_start_counters, 355 - .stop_counters = hisi_ddrc_pmu_v2_stop_counters, 356 - .enable_counter = hisi_ddrc_pmu_v2_enable_counter, 357 - .disable_counter = hisi_ddrc_pmu_v2_disable_counter, 358 - .enable_counter_int = hisi_ddrc_pmu_v2_enable_counter_int, 359 - .disable_counter_int = hisi_ddrc_pmu_v2_disable_counter_int, 360 - .write_counter = hisi_ddrc_pmu_v2_write_counter, 361 - .read_counter = hisi_ddrc_pmu_v2_read_counter, 362 - .get_int_status = hisi_ddrc_pmu_v2_get_int_status, 363 - .clear_int_status = hisi_ddrc_pmu_v2_clear_int_status, 401 + .get_event_idx = hisi_ddrc_pmu_get_event_idx, 402 + .start_counters = hisi_ddrc_pmu_start_counters, 403 + .stop_counters = hisi_ddrc_pmu_stop_counters, 404 + .enable_counter = hisi_ddrc_pmu_enable_counter, 405 + .disable_counter = hisi_ddrc_pmu_disable_counter, 406 + .enable_counter_int = hisi_ddrc_pmu_enable_counter_int, 407 + .disable_counter_int = hisi_ddrc_pmu_disable_counter_int, 408 + .write_counter = hisi_ddrc_pmu_write_counter, 409 + .read_counter = hisi_ddrc_pmu_read_counter, 410 + .get_int_status = hisi_ddrc_pmu_get_int_status, 411 + .clear_int_status = hisi_ddrc_pmu_clear_int_status, 364 412 }; 365 413 366 414 static int hisi_ddrc_pmu_dev_probe(struct platform_device *pdev, ··· 361 439 if (ret) 362 440 return ret; 363 441 364 - if (ddrc_pmu->identifier >= HISI_PMU_V2) { 365 - ddrc_pmu->counter_bits = 48; 366 - ddrc_pmu->check_event = DDRC_V2_NR_EVENTS; 367 - ddrc_pmu->pmu_events.attr_groups = hisi_ddrc_pmu_v2_attr_groups; 368 - ddrc_pmu->ops = &hisi_uncore_ddrc_v2_ops; 369 - } else { 370 - ddrc_pmu->counter_bits = 32; 371 - ddrc_pmu->check_event = DDRC_V1_NR_EVENTS; 372 - ddrc_pmu->pmu_events.attr_groups = hisi_ddrc_pmu_v1_attr_groups; 373 - ddrc_pmu->ops = &hisi_uncore_ddrc_v1_ops; 374 - } 375 - 442 + ddrc_pmu->pmu_events.attr_groups = ddrc_pmu->dev_info->attr_groups; 443 + ddrc_pmu->counter_bits = ddrc_pmu->dev_info->counter_bits; 444 + ddrc_pmu->check_event = ddrc_pmu->dev_info->check_event; 445 + ddrc_pmu->ops = &hisi_uncore_ddrc_ops; 376 446 ddrc_pmu->num_counters = DDRC_NR_COUNTERS; 377 447 ddrc_pmu->dev = &pdev->dev; 378 448 ddrc_pmu->on_cpu = -1; ··· 428 514 cpuhp_state_remove_instance_nocalls(CPUHP_AP_PERF_ARM_HISI_DDRC_ONLINE, 429 515 &ddrc_pmu->node); 430 516 } 517 + 518 + static struct hisi_ddrc_pmu_regs hisi_ddrc_v1_pmu_regs = { 519 + .event_cnt = DDRC_UNIMPLEMENTED_REG, 520 + .event_ctrl = DDRC_EVENT_CTRL, 521 + .event_type = DDRC_UNIMPLEMENTED_REG, 522 + .perf_ctrl = DDRC_PERF_CTRL, 523 + .perf_ctrl_en = DDRC_V1_PERF_CTRL_EN, 524 + .int_mask = DDRC_INT_MASK, 525 + .int_clear = DDRC_INT_CLEAR, 526 + .int_status = DDRC_INT_STATUS, 527 + }; 528 + 529 + static const struct hisi_pmu_dev_info hisi_ddrc_v1 = { 530 + .counter_bits = 32, 531 + .check_event = DDRC_V1_NR_EVENTS, 532 + .attr_groups = hisi_ddrc_pmu_v1_attr_groups, 533 + .private = &hisi_ddrc_v1_pmu_regs, 534 + }; 535 + 536 + static struct hisi_ddrc_pmu_regs hisi_ddrc_v2_pmu_regs = { 537 + .event_cnt = DDRC_V2_EVENT_CNT, 538 + .event_ctrl = DDRC_V2_EVENT_CTRL, 539 + .event_type = DDRC_V2_EVENT_TYPE, 540 + .perf_ctrl = DDRC_V2_PERF_CTRL, 541 + .perf_ctrl_en = DDRC_V2_PERF_CTRL_EN, 542 + .int_mask = DDRC_V2_INT_MASK, 543 + .int_clear = DDRC_V2_INT_CLEAR, 544 + .int_status = DDRC_V2_INT_STATUS, 545 + }; 546 + 547 + static const struct hisi_pmu_dev_info hisi_ddrc_v2 = { 548 + .counter_bits = 48, 549 + .check_event = DDRC_V2_NR_EVENTS, 550 + .attr_groups = hisi_ddrc_pmu_v2_attr_groups, 551 + .private = &hisi_ddrc_v2_pmu_regs, 552 + }; 553 + 554 + static struct hisi_ddrc_pmu_regs hisi_ddrc_v3_pmu_regs = { 555 + .event_cnt = DDRC_V2_EVENT_CNT, 556 + .event_ctrl = DDRC_V2_EVENT_CTRL, 557 + .event_type = DDRC_V2_EVENT_TYPE, 558 + .perf_ctrl = DDRC_V2_PERF_CTRL, 559 + .perf_ctrl_en = DDRC_V2_PERF_CTRL_EN, 560 + .int_mask = DDRC_V3_INT_MASK, 561 + .int_clear = DDRC_V3_INT_CLEAR, 562 + .int_status = DDRC_V3_INT_STATUS, 563 + }; 564 + 565 + static const struct hisi_pmu_dev_info hisi_ddrc_v3 = { 566 + .counter_bits = 48, 567 + .check_event = DDRC_V2_NR_EVENTS, 568 + .attr_groups = hisi_ddrc_pmu_v2_attr_groups, 569 + .private = &hisi_ddrc_v3_pmu_regs, 570 + }; 571 + 572 + static const struct acpi_device_id hisi_ddrc_pmu_acpi_match[] = { 573 + { "HISI0233", (kernel_ulong_t)&hisi_ddrc_v1 }, 574 + { "HISI0234", (kernel_ulong_t)&hisi_ddrc_v2 }, 575 + { "HISI0235", (kernel_ulong_t)&hisi_ddrc_v3 }, 576 + {} 577 + }; 578 + MODULE_DEVICE_TABLE(acpi, hisi_ddrc_pmu_acpi_match); 431 579 432 580 static struct platform_driver hisi_ddrc_pmu_driver = { 433 581 .driver = {

+3 -3

drivers/perf/hisilicon/hisi_uncore_hha_pmu.c

··· 47 47 #define HHA_SRCID_CMD GENMASK(16, 6) 48 48 #define HHA_SRCID_MSK GENMASK(30, 20) 49 49 #define HHA_DATSRC_SKT_EN BIT(23) 50 - #define HHA_EVTYPE_NONE 0xff 50 + #define HHA_EVTYPE_MASK GENMASK(7, 0) 51 51 #define HHA_V1_NR_EVENT 0x65 52 - #define HHA_V2_NR_EVENT 0xCE 52 + #define HHA_V2_NR_EVENT 0xFF 53 53 54 54 HISI_PMU_EVENT_ATTR_EXTRACTOR(srcid_cmd, config1, 10, 0); 55 55 HISI_PMU_EVENT_ATTR_EXTRACTOR(srcid_msk, config1, 21, 11); ··· 197 197 198 198 /* Write event code to HHA_EVENT_TYPEx register */ 199 199 val = readl(hha_pmu->base + reg); 200 - val &= ~(HHA_EVTYPE_NONE << shift); 200 + val &= ~(HHA_EVTYPE_MASK << shift); 201 201 val |= (type << shift); 202 202 writel(val, hha_pmu->base + reg); 203 203 }

+1 -1

drivers/perf/hisilicon/hisi_uncore_pa_pmu.c

··· 440 440 pa_pmu->pmu_events.attr_groups = pa_pmu->dev_info->attr_groups; 441 441 pa_pmu->num_counters = PA_NR_COUNTERS; 442 442 pa_pmu->ops = &hisi_uncore_pa_ops; 443 - pa_pmu->check_event = 0xB0; 443 + pa_pmu->check_event = PA_EVTYPE_MASK; 444 444 pa_pmu->counter_bits = 64; 445 445 pa_pmu->dev = &pdev->dev; 446 446 pa_pmu->on_cpu = -1;

+8 -3

drivers/perf/hisilicon/hisi_uncore_pmu.c

··· 510 510 return 0; 511 511 512 512 hisi_pmu->on_cpu = cpumask_local_spread(0, dev_to_node(hisi_pmu->dev)); 513 - WARN_ON(irq_set_affinity(hisi_pmu->irq, cpumask_of(hisi_pmu->on_cpu))); 513 + if (hisi_pmu->irq > 0) 514 + WARN_ON(irq_set_affinity(hisi_pmu->irq, 515 + cpumask_of(hisi_pmu->on_cpu))); 514 516 return 0; 515 517 } 516 518 ··· 527 525 hisi_pmu->on_cpu = cpu; 528 526 529 527 /* Overflow interrupt also should use the same CPU */ 530 - WARN_ON(irq_set_affinity(hisi_pmu->irq, cpumask_of(cpu))); 528 + if (hisi_pmu->irq > 0) 529 + WARN_ON(irq_set_affinity(hisi_pmu->irq, cpumask_of(cpu))); 531 530 532 531 return 0; 533 532 } ··· 563 560 perf_pmu_migrate_context(&hisi_pmu->pmu, cpu, target); 564 561 /* Use this CPU for event counting */ 565 562 hisi_pmu->on_cpu = target; 566 - WARN_ON(irq_set_affinity(hisi_pmu->irq, cpumask_of(target))); 563 + 564 + if (hisi_pmu->irq > 0) 565 + WARN_ON(irq_set_affinity(hisi_pmu->irq, cpumask_of(target))); 567 566 568 567 return 0; 569 568 }

+2

drivers/perf/hisilicon/hisi_uncore_pmu.h

··· 72 72 struct hisi_pmu_dev_info { 73 73 const char *name; 74 74 const struct attribute_group **attr_groups; 75 + u32 counter_bits; 76 + u32 check_event; 75 77 void *private; 76 78 }; 77 79

+159 -61

drivers/perf/hisilicon/hisi_uncore_sllc_pmu.c

··· 28 28 #define SLLC_VERSION 0x1cf0 29 29 #define SLLC_EVENT_CNT0_L 0x1d00 30 30 31 + /* SLLC registers definition in v3 */ 32 + #define SLLC_V3_INT_MASK 0x6834 33 + #define SLLC_V3_INT_STATUS 0x6838 34 + #define SLLC_V3_INT_CLEAR 0x683c 35 + #define SLLC_V3_VERSION 0x6c00 36 + #define SLLC_V3_PERF_CTRL 0x6d00 37 + #define SLLC_V3_SRCID_CTRL 0x6d04 38 + #define SLLC_V3_TGTID_CTRL 0x6d08 39 + #define SLLC_V3_EVENT_CTRL 0x6d14 40 + #define SLLC_V3_EVENT_TYPE0 0x6d18 41 + #define SLLC_V3_EVENT_CNT0_L 0x6e00 42 + 31 43 #define SLLC_EVTYPE_MASK 0xff 32 44 #define SLLC_PERF_CTRL_EN BIT(0) 33 45 #define SLLC_FILT_EN BIT(1) ··· 52 40 #define SLLC_TGTID_MAX_SHIFT 12 53 41 #define SLLC_SRCID_CMD_SHIFT 1 54 42 #define SLLC_SRCID_MSK_SHIFT 12 55 - #define SLLC_NR_EVENTS 0x80 43 + 44 + #define SLLC_V3_TGTID_MIN_SHIFT 1 45 + #define SLLC_V3_TGTID_MAX_SHIFT 10 46 + #define SLLC_V3_SRCID_CMD_SHIFT 1 47 + #define SLLC_V3_SRCID_MSK_SHIFT 10 48 + 49 + #define SLLC_NR_EVENTS 0xff 50 + #define SLLC_EVENT_CNTn(cnt0, n) ((cnt0) + (n) * 8) 56 51 57 52 HISI_PMU_EVENT_ATTR_EXTRACTOR(tgtid_min, config1, 10, 0); 58 53 HISI_PMU_EVENT_ATTR_EXTRACTOR(tgtid_max, config1, 21, 11); 59 54 HISI_PMU_EVENT_ATTR_EXTRACTOR(srcid_cmd, config1, 32, 22); 60 55 HISI_PMU_EVENT_ATTR_EXTRACTOR(srcid_msk, config1, 43, 33); 61 56 HISI_PMU_EVENT_ATTR_EXTRACTOR(tracetag_en, config1, 44, 44); 57 + 58 + struct hisi_sllc_pmu_regs { 59 + u32 int_mask; 60 + u32 int_clear; 61 + u32 int_status; 62 + u32 perf_ctrl; 63 + u32 srcid_ctrl; 64 + u32 srcid_cmd_shift; 65 + u32 srcid_mask_shift; 66 + u32 tgtid_ctrl; 67 + u32 tgtid_min_shift; 68 + u32 tgtid_max_shift; 69 + u32 event_ctrl; 70 + u32 event_type0; 71 + u32 version; 72 + u32 event_cnt0; 73 + }; 62 74 63 75 static bool tgtid_is_valid(u32 max, u32 min) 64 76 { ··· 92 56 static void hisi_sllc_pmu_enable_tracetag(struct perf_event *event) 93 57 { 94 58 struct hisi_pmu *sllc_pmu = to_hisi_pmu(event->pmu); 59 + struct hisi_sllc_pmu_regs *regs = sllc_pmu->dev_info->private; 95 60 u32 tt_en = hisi_get_tracetag_en(event); 96 61 97 62 if (tt_en) { 98 63 u32 val; 99 64 100 - val = readl(sllc_pmu->base + SLLC_PERF_CTRL); 65 + val = readl(sllc_pmu->base + regs->perf_ctrl); 101 66 val |= SLLC_TRACETAG_EN | SLLC_FILT_EN; 102 - writel(val, sllc_pmu->base + SLLC_PERF_CTRL); 67 + writel(val, sllc_pmu->base + regs->perf_ctrl); 103 68 } 104 69 } 105 70 106 71 static void hisi_sllc_pmu_disable_tracetag(struct perf_event *event) 107 72 { 108 73 struct hisi_pmu *sllc_pmu = to_hisi_pmu(event->pmu); 74 + struct hisi_sllc_pmu_regs *regs = sllc_pmu->dev_info->private; 109 75 u32 tt_en = hisi_get_tracetag_en(event); 110 76 111 77 if (tt_en) { 112 78 u32 val; 113 79 114 - val = readl(sllc_pmu->base + SLLC_PERF_CTRL); 80 + val = readl(sllc_pmu->base + regs->perf_ctrl); 115 81 val &= ~(SLLC_TRACETAG_EN | SLLC_FILT_EN); 116 - writel(val, sllc_pmu->base + SLLC_PERF_CTRL); 82 + writel(val, sllc_pmu->base + regs->perf_ctrl); 117 83 } 118 84 } 119 85 120 86 static void hisi_sllc_pmu_config_tgtid(struct perf_event *event) 121 87 { 122 88 struct hisi_pmu *sllc_pmu = to_hisi_pmu(event->pmu); 89 + struct hisi_sllc_pmu_regs *regs = sllc_pmu->dev_info->private; 123 90 u32 min = hisi_get_tgtid_min(event); 124 91 u32 max = hisi_get_tgtid_max(event); 125 92 126 93 if (tgtid_is_valid(max, min)) { 127 - u32 val = (max << SLLC_TGTID_MAX_SHIFT) | (min << SLLC_TGTID_MIN_SHIFT); 94 + u32 val = (max << regs->tgtid_max_shift) | 95 + (min << regs->tgtid_min_shift); 128 96 129 - writel(val, sllc_pmu->base + SLLC_TGTID_CTRL); 97 + writel(val, sllc_pmu->base + regs->tgtid_ctrl); 130 98 /* Enable the tgtid */ 131 - val = readl(sllc_pmu->base + SLLC_PERF_CTRL); 99 + val = readl(sllc_pmu->base + regs->perf_ctrl); 132 100 val |= SLLC_TGTID_EN | SLLC_FILT_EN; 133 - writel(val, sllc_pmu->base + SLLC_PERF_CTRL); 101 + writel(val, sllc_pmu->base + regs->perf_ctrl); 134 102 } 135 103 } 136 104 137 105 static void hisi_sllc_pmu_clear_tgtid(struct perf_event *event) 138 106 { 139 107 struct hisi_pmu *sllc_pmu = to_hisi_pmu(event->pmu); 108 + struct hisi_sllc_pmu_regs *regs = sllc_pmu->dev_info->private; 140 109 u32 min = hisi_get_tgtid_min(event); 141 110 u32 max = hisi_get_tgtid_max(event); 142 111 143 112 if (tgtid_is_valid(max, min)) { 144 113 u32 val; 145 114 146 - writel(SLLC_TGTID_NONE, sllc_pmu->base + SLLC_TGTID_CTRL); 115 + writel(SLLC_TGTID_NONE, sllc_pmu->base + regs->tgtid_ctrl); 147 116 /* Disable the tgtid */ 148 - val = readl(sllc_pmu->base + SLLC_PERF_CTRL); 117 + val = readl(sllc_pmu->base + regs->perf_ctrl); 149 118 val &= ~(SLLC_TGTID_EN | SLLC_FILT_EN); 150 - writel(val, sllc_pmu->base + SLLC_PERF_CTRL); 119 + writel(val, sllc_pmu->base + regs->perf_ctrl); 151 120 } 152 121 } 153 122 154 123 static void hisi_sllc_pmu_config_srcid(struct perf_event *event) 155 124 { 156 125 struct hisi_pmu *sllc_pmu = to_hisi_pmu(event->pmu); 126 + struct hisi_sllc_pmu_regs *regs = sllc_pmu->dev_info->private; 157 127 u32 cmd = hisi_get_srcid_cmd(event); 158 128 159 129 if (cmd) { 160 130 u32 val, msk; 161 131 162 132 msk = hisi_get_srcid_msk(event); 163 - val = (cmd << SLLC_SRCID_CMD_SHIFT) | (msk << SLLC_SRCID_MSK_SHIFT); 164 - writel(val, sllc_pmu->base + SLLC_SRCID_CTRL); 133 + val = (cmd << regs->srcid_cmd_shift) | 134 + (msk << regs->srcid_mask_shift); 135 + writel(val, sllc_pmu->base + regs->srcid_ctrl); 165 136 /* Enable the srcid */ 166 - val = readl(sllc_pmu->base + SLLC_PERF_CTRL); 137 + val = readl(sllc_pmu->base + regs->perf_ctrl); 167 138 val |= SLLC_SRCID_EN | SLLC_FILT_EN; 168 - writel(val, sllc_pmu->base + SLLC_PERF_CTRL); 139 + writel(val, sllc_pmu->base + regs->perf_ctrl); 169 140 } 170 141 } 171 142 172 143 static void hisi_sllc_pmu_clear_srcid(struct perf_event *event) 173 144 { 174 145 struct hisi_pmu *sllc_pmu = to_hisi_pmu(event->pmu); 146 + struct hisi_sllc_pmu_regs *regs = sllc_pmu->dev_info->private; 175 147 u32 cmd = hisi_get_srcid_cmd(event); 176 148 177 149 if (cmd) { 178 150 u32 val; 179 151 180 - writel(SLLC_SRCID_NONE, sllc_pmu->base + SLLC_SRCID_CTRL); 152 + writel(SLLC_SRCID_NONE, sllc_pmu->base + regs->srcid_ctrl); 181 153 /* Disable the srcid */ 182 - val = readl(sllc_pmu->base + SLLC_PERF_CTRL); 154 + val = readl(sllc_pmu->base + regs->perf_ctrl); 183 155 val &= ~(SLLC_SRCID_EN | SLLC_FILT_EN); 184 - writel(val, sllc_pmu->base + SLLC_PERF_CTRL); 156 + writel(val, sllc_pmu->base + regs->perf_ctrl); 185 157 } 186 158 } 187 159 ··· 211 167 } 212 168 } 213 169 214 - static u32 hisi_sllc_pmu_get_counter_offset(int idx) 215 - { 216 - return (SLLC_EVENT_CNT0_L + idx * 8); 217 - } 218 - 219 170 static u64 hisi_sllc_pmu_read_counter(struct hisi_pmu *sllc_pmu, 220 171 struct hw_perf_event *hwc) 221 172 { 222 - return readq(sllc_pmu->base + 223 - hisi_sllc_pmu_get_counter_offset(hwc->idx)); 173 + struct hisi_sllc_pmu_regs *regs = sllc_pmu->dev_info->private; 174 + 175 + return readq(sllc_pmu->base + SLLC_EVENT_CNTn(regs->event_cnt0, hwc->idx)); 224 176 } 225 177 226 178 static void hisi_sllc_pmu_write_counter(struct hisi_pmu *sllc_pmu, 227 179 struct hw_perf_event *hwc, u64 val) 228 180 { 229 - writeq(val, sllc_pmu->base + 230 - hisi_sllc_pmu_get_counter_offset(hwc->idx)); 181 + struct hisi_sllc_pmu_regs *regs = sllc_pmu->dev_info->private; 182 + 183 + writeq(val, sllc_pmu->base + SLLC_EVENT_CNTn(regs->event_cnt0, hwc->idx)); 231 184 } 232 185 233 186 static void hisi_sllc_pmu_write_evtype(struct hisi_pmu *sllc_pmu, int idx, 234 187 u32 type) 235 188 { 236 - u32 reg, reg_idx, shift, val; 189 + struct hisi_sllc_pmu_regs *regs = sllc_pmu->dev_info->private; 190 + u32 reg, val; 237 191 238 192 /* 239 193 * Select the appropriate event select register(SLLC_EVENT_TYPE0/1). ··· 240 198 * SLLC_EVENT_TYPE0 is chosen. For the latter 4 hardware counters, 241 199 * SLLC_EVENT_TYPE1 is chosen. 242 200 */ 243 - reg = SLLC_EVENT_TYPE0 + (idx / 4) * 4; 244 - reg_idx = idx % 4; 245 - shift = 8 * reg_idx; 201 + reg = regs->event_type0 + (idx / 4) * 4; 246 202 247 203 /* Write event code to SLLC_EVENT_TYPEx Register */ 248 204 val = readl(sllc_pmu->base + reg); 249 - val &= ~(SLLC_EVTYPE_MASK << shift); 250 - val |= (type << shift); 205 + val &= ~(SLLC_EVTYPE_MASK << HISI_PMU_EVTYPE_SHIFT(idx)); 206 + val |= (type << HISI_PMU_EVTYPE_SHIFT(idx)); 251 207 writel(val, sllc_pmu->base + reg); 252 208 } 253 209 254 210 static void hisi_sllc_pmu_start_counters(struct hisi_pmu *sllc_pmu) 255 211 { 212 + struct hisi_sllc_pmu_regs *regs = sllc_pmu->dev_info->private; 256 213 u32 val; 257 214 258 - val = readl(sllc_pmu->base + SLLC_PERF_CTRL); 215 + val = readl(sllc_pmu->base + regs->perf_ctrl); 259 216 val |= SLLC_PERF_CTRL_EN; 260 - writel(val, sllc_pmu->base + SLLC_PERF_CTRL); 217 + writel(val, sllc_pmu->base + regs->perf_ctrl); 261 218 } 262 219 263 220 static void hisi_sllc_pmu_stop_counters(struct hisi_pmu *sllc_pmu) 264 221 { 222 + struct hisi_sllc_pmu_regs *regs = sllc_pmu->dev_info->private; 265 223 u32 val; 266 224 267 - val = readl(sllc_pmu->base + SLLC_PERF_CTRL); 225 + val = readl(sllc_pmu->base + regs->perf_ctrl); 268 226 val &= ~(SLLC_PERF_CTRL_EN); 269 - writel(val, sllc_pmu->base + SLLC_PERF_CTRL); 227 + writel(val, sllc_pmu->base + regs->perf_ctrl); 270 228 } 271 229 272 230 static void hisi_sllc_pmu_enable_counter(struct hisi_pmu *sllc_pmu, 273 231 struct hw_perf_event *hwc) 274 232 { 233 + struct hisi_sllc_pmu_regs *regs = sllc_pmu->dev_info->private; 275 234 u32 val; 276 235 277 - val = readl(sllc_pmu->base + SLLC_EVENT_CTRL); 278 - val |= 1 << hwc->idx; 279 - writel(val, sllc_pmu->base + SLLC_EVENT_CTRL); 236 + val = readl(sllc_pmu->base + regs->event_ctrl); 237 + val |= BIT_ULL(hwc->idx); 238 + writel(val, sllc_pmu->base + regs->event_ctrl); 280 239 } 281 240 282 241 static void hisi_sllc_pmu_disable_counter(struct hisi_pmu *sllc_pmu, 283 242 struct hw_perf_event *hwc) 284 243 { 244 + struct hisi_sllc_pmu_regs *regs = sllc_pmu->dev_info->private; 285 245 u32 val; 286 246 287 - val = readl(sllc_pmu->base + SLLC_EVENT_CTRL); 288 - val &= ~(1 << hwc->idx); 289 - writel(val, sllc_pmu->base + SLLC_EVENT_CTRL); 247 + val = readl(sllc_pmu->base + regs->event_ctrl); 248 + val &= ~BIT_ULL(hwc->idx); 249 + writel(val, sllc_pmu->base + regs->event_ctrl); 290 250 } 291 251 292 252 static void hisi_sllc_pmu_enable_counter_int(struct hisi_pmu *sllc_pmu, 293 253 struct hw_perf_event *hwc) 294 254 { 255 + struct hisi_sllc_pmu_regs *regs = sllc_pmu->dev_info->private; 295 256 u32 val; 296 257 297 - val = readl(sllc_pmu->base + SLLC_INT_MASK); 298 - /* Write 0 to enable interrupt */ 299 - val &= ~(1 << hwc->idx); 300 - writel(val, sllc_pmu->base + SLLC_INT_MASK); 258 + val = readl(sllc_pmu->base + regs->int_mask); 259 + val &= ~BIT_ULL(hwc->idx); 260 + writel(val, sllc_pmu->base + regs->int_mask); 301 261 } 302 262 303 263 static void hisi_sllc_pmu_disable_counter_int(struct hisi_pmu *sllc_pmu, 304 264 struct hw_perf_event *hwc) 305 265 { 266 + struct hisi_sllc_pmu_regs *regs = sllc_pmu->dev_info->private; 306 267 u32 val; 307 268 308 - val = readl(sllc_pmu->base + SLLC_INT_MASK); 309 - /* Write 1 to mask interrupt */ 310 - val |= 1 << hwc->idx; 311 - writel(val, sllc_pmu->base + SLLC_INT_MASK); 269 + val = readl(sllc_pmu->base + regs->int_mask); 270 + val |= BIT_ULL(hwc->idx); 271 + writel(val, sllc_pmu->base + regs->int_mask); 312 272 } 313 273 314 274 static u32 hisi_sllc_pmu_get_int_status(struct hisi_pmu *sllc_pmu) 315 275 { 316 - return readl(sllc_pmu->base + SLLC_INT_STATUS); 276 + struct hisi_sllc_pmu_regs *regs = sllc_pmu->dev_info->private; 277 + 278 + return readl(sllc_pmu->base + regs->int_status); 317 279 } 318 280 319 281 static void hisi_sllc_pmu_clear_int_status(struct hisi_pmu *sllc_pmu, int idx) 320 282 { 321 - writel(1 << idx, sllc_pmu->base + SLLC_INT_CLEAR); 322 - } 283 + struct hisi_sllc_pmu_regs *regs = sllc_pmu->dev_info->private; 323 284 324 - static const struct acpi_device_id hisi_sllc_pmu_acpi_match[] = { 325 - { "HISI0263", }, 326 - {} 327 - }; 328 - MODULE_DEVICE_TABLE(acpi, hisi_sllc_pmu_acpi_match); 285 + writel(BIT_ULL(idx), sllc_pmu->base + regs->int_clear); 286 + } 329 287 330 288 static int hisi_sllc_pmu_init_data(struct platform_device *pdev, 331 289 struct hisi_pmu *sllc_pmu) 332 290 { 291 + struct hisi_sllc_pmu_regs *regs; 292 + 333 293 hisi_uncore_pmu_init_topology(sllc_pmu, &pdev->dev); 334 294 335 295 /* ··· 348 304 return -EINVAL; 349 305 } 350 306 307 + sllc_pmu->dev_info = device_get_match_data(&pdev->dev); 308 + if (!sllc_pmu->dev_info) 309 + return -ENODEV; 310 + 351 311 sllc_pmu->base = devm_platform_ioremap_resource(pdev, 0); 352 312 if (IS_ERR(sllc_pmu->base)) { 353 313 dev_err(&pdev->dev, "ioremap failed for sllc_pmu resource.\n"); 354 314 return PTR_ERR(sllc_pmu->base); 355 315 } 356 316 357 - sllc_pmu->identifier = readl(sllc_pmu->base + SLLC_VERSION); 317 + regs = sllc_pmu->dev_info->private; 318 + sllc_pmu->identifier = readl(sllc_pmu->base + regs->version); 358 319 359 320 return 0; 360 321 } ··· 399 350 &hisi_pmu_cpumask_attr_group, 400 351 &hisi_pmu_identifier_group, 401 352 NULL 353 + }; 354 + 355 + static struct hisi_sllc_pmu_regs hisi_sllc_v2_pmu_regs = { 356 + .int_mask = SLLC_INT_MASK, 357 + .int_clear = SLLC_INT_CLEAR, 358 + .int_status = SLLC_INT_STATUS, 359 + .perf_ctrl = SLLC_PERF_CTRL, 360 + .srcid_ctrl = SLLC_SRCID_CTRL, 361 + .srcid_cmd_shift = SLLC_SRCID_CMD_SHIFT, 362 + .srcid_mask_shift = SLLC_SRCID_MSK_SHIFT, 363 + .tgtid_ctrl = SLLC_TGTID_CTRL, 364 + .tgtid_min_shift = SLLC_TGTID_MIN_SHIFT, 365 + .tgtid_max_shift = SLLC_TGTID_MAX_SHIFT, 366 + .event_ctrl = SLLC_EVENT_CTRL, 367 + .event_type0 = SLLC_EVENT_TYPE0, 368 + .version = SLLC_VERSION, 369 + .event_cnt0 = SLLC_EVENT_CNT0_L, 370 + }; 371 + 372 + static const struct hisi_pmu_dev_info hisi_sllc_v2 = { 373 + .private = &hisi_sllc_v2_pmu_regs, 374 + }; 375 + 376 + static struct hisi_sllc_pmu_regs hisi_sllc_v3_pmu_regs = { 377 + .int_mask = SLLC_V3_INT_MASK, 378 + .int_clear = SLLC_V3_INT_CLEAR, 379 + .int_status = SLLC_V3_INT_STATUS, 380 + .perf_ctrl = SLLC_V3_PERF_CTRL, 381 + .srcid_ctrl = SLLC_V3_SRCID_CTRL, 382 + .srcid_cmd_shift = SLLC_V3_SRCID_CMD_SHIFT, 383 + .srcid_mask_shift = SLLC_V3_SRCID_MSK_SHIFT, 384 + .tgtid_ctrl = SLLC_V3_TGTID_CTRL, 385 + .tgtid_min_shift = SLLC_V3_TGTID_MIN_SHIFT, 386 + .tgtid_max_shift = SLLC_V3_TGTID_MAX_SHIFT, 387 + .event_ctrl = SLLC_V3_EVENT_CTRL, 388 + .event_type0 = SLLC_V3_EVENT_TYPE0, 389 + .version = SLLC_V3_VERSION, 390 + .event_cnt0 = SLLC_V3_EVENT_CNT0_L, 391 + }; 392 + 393 + static const struct hisi_pmu_dev_info hisi_sllc_v3 = { 394 + .private = &hisi_sllc_v3_pmu_regs, 402 395 }; 403 396 404 397 static const struct hisi_uncore_ops hisi_uncore_sllc_ops = { ··· 533 442 cpuhp_state_remove_instance_nocalls(CPUHP_AP_PERF_ARM_HISI_SLLC_ONLINE, 534 443 &sllc_pmu->node); 535 444 } 445 + 446 + static const struct acpi_device_id hisi_sllc_pmu_acpi_match[] = { 447 + { "HISI0263", (kernel_ulong_t)&hisi_sllc_v2 }, 448 + { "HISI0264", (kernel_ulong_t)&hisi_sllc_v3 }, 449 + {} 450 + }; 451 + MODULE_DEVICE_TABLE(acpi, hisi_sllc_pmu_acpi_match); 536 452 537 453 static struct platform_driver hisi_sllc_pmu_driver = { 538 454 .driver = {

+1 -1

include/linux/acpi.h

··· 1503 1503 #else 1504 1504 static inline int acpi_parse_spcr(bool enable_earlycon, bool enable_console) 1505 1505 { 1506 - return 0; 1506 + return -ENODEV; 1507 1507 } 1508 1508 #endif 1509 1509

+2

include/linux/nmi.h

··· 103 103 extern void hardlockup_detector_perf_stop(void); 104 104 extern void hardlockup_detector_perf_restart(void); 105 105 extern void hardlockup_config_perf_event(const char *str); 106 + extern void hardlockup_detector_perf_adjust_period(u64 period); 106 107 #else 107 108 static inline void hardlockup_detector_perf_stop(void) { } 108 109 static inline void hardlockup_detector_perf_restart(void) { } 109 110 static inline void hardlockup_config_perf_event(const char *str) { } 111 + static inline void hardlockup_detector_perf_adjust_period(u64 period) { } 110 112 #endif 111 113 112 114 void watchdog_hardlockup_stop(void);

+8

include/linux/perf/arm_pmu.h

··· 70 70 struct arm_pmu *percpu_pmu; 71 71 72 72 int irq; 73 + 74 + struct perf_branch_stack *branch_stack; 75 + 76 + /* Active events requesting branch records */ 77 + unsigned int branch_users; 73 78 }; 74 79 75 80 enum armpmu_attr_groups { ··· 120 115 /* PMUv3 only */ 121 116 int pmuver; 122 117 u64 reg_pmmir; 118 + u64 reg_brbidr; 123 119 #define ARMV8_PMUV3_MAX_COMMON_EVENTS 0x40 124 120 DECLARE_BITMAP(pmceid_bitmap, ARMV8_PMUV3_MAX_COMMON_EVENTS); 125 121 #define ARMV8_PMUV3_EXT_COMMON_EVENT_BASE 0x4000 ··· 131 125 }; 132 126 133 127 #define to_arm_pmu(p) (container_of(p, struct arm_pmu, pmu)) 128 + 129 + DECLARE_PER_CPU(struct arm_pmu *, cpu_armpmu); 134 130 135 131 u64 armpmu_event_update(struct perf_event *event); 136 132

+2

include/uapi/linux/prctl.h

··· 244 244 # define PR_MTE_TAG_MASK (0xffffUL << PR_MTE_TAG_SHIFT) 245 245 /* Unused; kept only for source compatibility */ 246 246 # define PR_MTE_TCF_SHIFT 1 247 + /* MTE tag check store only */ 248 + # define PR_MTE_STORE_ONLY (1UL << 19) 247 249 /* RISC-V pointer masking tag length */ 248 250 # define PR_PMLEN_SHIFT 24 249 251 # define PR_PMLEN_MASK (0x7fUL << PR_PMLEN_SHIFT)

+22

kernel/watchdog_perf.c

··· 187 187 } 188 188 189 189 /** 190 + * hardlockup_detector_perf_adjust_period - Adjust the event period due 191 + * to current cpu frequency change 192 + * @period: The target period to be set 193 + */ 194 + void hardlockup_detector_perf_adjust_period(u64 period) 195 + { 196 + struct perf_event *event = this_cpu_read(watchdog_ev); 197 + 198 + if (!(watchdog_enabled & WATCHDOG_HARDLOCKUP_ENABLED)) 199 + return; 200 + 201 + if (!event) 202 + return; 203 + 204 + if (event->attr.sample_period == period) 205 + return; 206 + 207 + if (perf_event_period(event, period)) 208 + pr_err("failed to change period to %llu\n", period); 209 + } 210 + 211 + /** 190 212 * hardlockup_detector_perf_stop - Globally stop watchdog events 191 213 * 192 214 * Special interface for x86 to handle the perf HT bug.

+1 -1

tools/testing/selftests/arm64/abi/Makefile

··· 12 12 $(OUTPUT)/tpidr2: tpidr2.c 13 13 $(CC) -fno-asynchronous-unwind-tables -fno-ident -s -Os -nostdlib \ 14 14 -static -include ../../../../include/nolibc/nolibc.h \ 15 - -ffreestanding -Wall $^ -o $@ -lgcc 15 + -I../.. -ffreestanding -Wall $^ -o $@ -lgcc

+16

tools/testing/selftests/arm64/abi/hwcap.c

··· 21 21 22 22 #define TESTS_PER_HWCAP 3 23 23 24 + #ifndef AT_HWCAP3 25 + #define AT_HWCAP3 29 26 + #endif 27 + 24 28 /* 25 29 * Function expected to generate exception when the feature is not 26 30 * supported and return when it is supported. If the specific exception ··· 1101 1097 .cpuinfo = "hbc", 1102 1098 .sigill_fn = hbc_sigill, 1103 1099 .sigill_reliable = true, 1100 + }, 1101 + { 1102 + .name = "MTE_FAR", 1103 + .at_hwcap = AT_HWCAP3, 1104 + .hwcap_bit = HWCAP3_MTE_FAR, 1105 + .cpuinfo = "mtefar", 1106 + }, 1107 + { 1108 + .name = "MTE_STOREONLY", 1109 + .at_hwcap = AT_HWCAP3, 1110 + .hwcap_bit = HWCAP3_MTE_STORE_ONLY, 1111 + .cpuinfo = "mtestoreonly", 1104 1112 }, 1105 1113 }; 1106 1114

+37 -103

tools/testing/selftests/arm64/abi/tpidr2.c

··· 3 3 #include <linux/sched.h> 4 4 #include <linux/wait.h> 5 5 6 + #include "kselftest.h" 7 + 6 8 #define SYS_TPIDR2 "S3_3_C13_C0_5" 7 9 8 10 #define EXPECTED_TESTS 5 9 - 10 - static void putstr(const char *str) 11 - { 12 - write(1, str, strlen(str)); 13 - } 14 - 15 - static void putnum(unsigned int num) 16 - { 17 - char c; 18 - 19 - if (num / 10) 20 - putnum(num / 10); 21 - 22 - c = '0' + (num % 10); 23 - write(1, &c, 1); 24 - } 25 - 26 - static int tests_run; 27 - static int tests_passed; 28 - static int tests_failed; 29 - static int tests_skipped; 30 11 31 12 static void set_tpidr2(uint64_t val) 32 13 { ··· 29 48 : "cc"); 30 49 31 50 return val; 32 - } 33 - 34 - static void print_summary(void) 35 - { 36 - if (tests_passed + tests_failed + tests_skipped != EXPECTED_TESTS) 37 - putstr("# UNEXPECTED TEST COUNT: "); 38 - 39 - putstr("# Totals: pass:"); 40 - putnum(tests_passed); 41 - putstr(" fail:"); 42 - putnum(tests_failed); 43 - putstr(" xfail:0 xpass:0 skip:"); 44 - putnum(tests_skipped); 45 - putstr(" error:0\n"); 46 51 } 47 52 48 53 /* Processes should start with TPIDR2 == 0 */ ··· 72 105 if (newpid == 0) { 73 106 /* In child */ 74 107 if (get_tpidr2() != oldpid) { 75 - putstr("# TPIDR2 changed in child: "); 76 - putnum(get_tpidr2()); 77 - putstr("\n"); 108 + ksft_print_msg("TPIDR2 changed in child: %llx\n", 109 + get_tpidr2()); 78 110 exit(0); 79 111 } 80 112 ··· 81 115 if (get_tpidr2() == getpid()) { 82 116 exit(1); 83 117 } else { 84 - putstr("# Failed to set TPIDR2 in child\n"); 118 + ksft_print_msg("Failed to set TPIDR2 in child\n"); 85 119 exit(0); 86 120 } 87 121 } 88 122 if (newpid < 0) { 89 - putstr("# fork() failed: -"); 90 - putnum(-newpid); 91 - putstr("\n"); 123 + ksft_print_msg("fork() failed: %d\n", newpid); 92 124 return 0; 93 125 } 94 126 ··· 96 132 if (waiting < 0) { 97 133 if (errno == EINTR) 98 134 continue; 99 - putstr("# waitpid() failed: "); 100 - putnum(errno); 101 - putstr("\n"); 135 + ksft_print_msg("waitpid() failed: %d\n", errno); 102 136 return 0; 103 137 } 104 138 if (waiting != newpid) { 105 - putstr("# waitpid() returned wrong PID\n"); 139 + ksft_print_msg("waitpid() returned wrong PID: %d != %d\n", 140 + waiting, newpid); 106 141 return 0; 107 142 } 108 143 109 144 if (!WIFEXITED(status)) { 110 - putstr("# child did not exit\n"); 145 + ksft_print_msg("child did not exit\n"); 111 146 return 0; 112 147 } 113 148 114 149 if (getpid() != get_tpidr2()) { 115 - putstr("# TPIDR2 corrupted in parent\n"); 150 + ksft_print_msg("TPIDR2 corrupted in parent\n"); 116 151 return 0; 117 152 } 118 153 ··· 151 188 152 189 stack = malloc(__STACK_SIZE); 153 190 if (!stack) { 154 - putstr("# malloc() failed\n"); 191 + ksft_print_msg("malloc() failed\n"); 155 192 return 0; 156 193 } 157 194 158 195 ret = sys_clone(CLONE_VM, (unsigned long)stack + __STACK_SIZE, 159 196 &parent_tid, 0, &child_tid); 160 197 if (ret == -1) { 161 - putstr("# clone() failed\n"); 162 - putnum(errno); 163 - putstr("\n"); 198 + ksft_print_msg("clone() failed: %d\n", errno); 164 199 return 0; 165 200 } 166 201 167 202 if (ret == 0) { 168 203 /* In child */ 169 204 if (get_tpidr2() != 0) { 170 - putstr("# TPIDR2 non-zero in child: "); 171 - putnum(get_tpidr2()); 172 - putstr("\n"); 205 + ksft_print_msg("TPIDR2 non-zero in child: %llx\n", 206 + get_tpidr2()); 173 207 exit(0); 174 208 } 175 209 176 210 if (gettid() == 0) 177 - putstr("# Child TID==0\n"); 211 + ksft_print_msg("Child TID==0\n"); 178 212 set_tpidr2(gettid()); 179 213 if (get_tpidr2() == gettid()) { 180 214 exit(1); 181 215 } else { 182 - putstr("# Failed to set TPIDR2 in child\n"); 216 + ksft_print_msg("Failed to set TPIDR2 in child\n"); 183 217 exit(0); 184 218 } 185 219 } ··· 187 227 if (waiting < 0) { 188 228 if (errno == EINTR) 189 229 continue; 190 - putstr("# wait4() failed: "); 191 - putnum(errno); 192 - putstr("\n"); 230 + ksft_print_msg("wait4() failed: %d\n", errno); 193 231 return 0; 194 232 } 195 233 if (waiting != ret) { 196 - putstr("# wait4() returned wrong PID "); 197 - putnum(waiting); 198 - putstr("\n"); 234 + ksft_print_msg("wait4() returned wrong PID %d\n", 235 + waiting); 199 236 return 0; 200 237 } 201 238 202 239 if (!WIFEXITED(status)) { 203 - putstr("# child did not exit\n"); 240 + ksft_print_msg("child did not exit\n"); 204 241 return 0; 205 242 } 206 243 207 244 if (parent != get_tpidr2()) { 208 - putstr("# TPIDR2 corrupted in parent\n"); 245 + ksft_print_msg("TPIDR2 corrupted in parent\n"); 209 246 return 0; 210 247 } 211 248 ··· 210 253 } 211 254 } 212 255 213 - #define run_test(name) \ 214 - if (name()) { \ 215 - tests_passed++; \ 216 - } else { \ 217 - tests_failed++; \ 218 - putstr("not "); \ 219 - } \ 220 - putstr("ok "); \ 221 - putnum(++tests_run); \ 222 - putstr(" " #name "\n"); 223 - 224 - #define skip_test(name) \ 225 - tests_skipped++; \ 226 - putstr("ok "); \ 227 - putnum(++tests_run); \ 228 - putstr(" # SKIP " #name "\n"); 229 - 230 256 int main(int argc, char **argv) 231 257 { 232 258 int ret; 233 259 234 - putstr("TAP version 13\n"); 235 - putstr("1.."); 236 - putnum(EXPECTED_TESTS); 237 - putstr("\n"); 260 + ksft_print_header(); 261 + ksft_set_plan(5); 238 262 239 - putstr("# PID: "); 240 - putnum(getpid()); 241 - putstr("\n"); 263 + ksft_print_msg("PID: %d\n", getpid()); 242 264 243 265 /* 244 266 * This test is run with nolibc which doesn't support hwcap and ··· 226 290 */ 227 291 ret = open("/proc/sys/abi/sme_default_vector_length", O_RDONLY, 0); 228 292 if (ret >= 0) { 229 - run_test(default_value); 230 - run_test(write_read); 231 - run_test(write_sleep_read); 232 - run_test(write_fork_read); 233 - run_test(write_clone_read); 293 + ksft_test_result(default_value(), "default_value\n"); 294 + ksft_test_result(write_read, "write_read\n"); 295 + ksft_test_result(write_sleep_read, "write_sleep_read\n"); 296 + ksft_test_result(write_fork_read, "write_fork_read\n"); 297 + ksft_test_result(write_clone_read, "write_clone_read\n"); 234 298 235 299 } else { 236 - putstr("# SME support not present\n"); 300 + ksft_print_msg("SME support not present\n"); 237 301 238 - skip_test(default_value); 239 - skip_test(write_read); 240 - skip_test(write_sleep_read); 241 - skip_test(write_fork_read); 242 - skip_test(write_clone_read); 302 + ksft_test_result_skip("default_value\n"); 303 + ksft_test_result_skip("write_read\n"); 304 + ksft_test_result_skip("write_sleep_read\n"); 305 + ksft_test_result_skip("write_fork_read\n"); 306 + ksft_test_result_skip("write_clone_read\n"); 243 307 } 244 308 245 - print_summary(); 246 - 247 - return 0; 309 + ksft_finished(); 248 310 }

+74 -3

tools/testing/selftests/arm64/fp/fp-ptrace.c

··· 1061 1061 if (config->sme_vl_in != config->sme_vl_expected) { 1062 1062 return false; 1063 1063 } 1064 + 1065 + if (!sve_supported()) 1066 + return false; 1064 1067 } 1068 + 1069 + return true; 1070 + } 1071 + 1072 + static bool sve_write_fpsimd_supported(struct test_config *config) 1073 + { 1074 + if (!sve_supported()) 1075 + return false; 1076 + 1077 + if ((config->svcr_in & SVCR_ZA) != (config->svcr_expected & SVCR_ZA)) 1078 + return false; 1079 + 1080 + if (config->svcr_expected & SVCR_SM) 1081 + return false; 1082 + 1083 + if (config->sme_vl_in != config->sme_vl_expected) 1084 + return false; 1065 1085 1066 1086 return true; 1067 1087 } ··· 1154 1134 int vl = vl_expected(config); 1155 1135 int sme_vq = __sve_vq_from_vl(config->sme_vl_expected); 1156 1136 1137 + if (!vl) 1138 + return; 1139 + 1157 1140 fill_random(z_expected, __SVE_ZREGS_SIZE(__sve_vq_from_vl(vl))); 1158 1141 fill_random(p_expected, __SVE_PREGS_SIZE(__sve_vq_from_vl(vl))); 1159 1142 ··· 1175 1152 } 1176 1153 } 1177 1154 1178 - static void sve_write(pid_t child, struct test_config *config) 1155 + static void sve_write_sve(pid_t child, struct test_config *config) 1179 1156 { 1180 1157 struct user_sve_header *sve; 1181 1158 struct iovec iov; ··· 1183 1160 1184 1161 vl = vl_expected(config); 1185 1162 vq = __sve_vq_from_vl(vl); 1163 + 1164 + if (!vl) 1165 + return; 1186 1166 1187 1167 iov.iov_len = SVE_PT_SVE_OFFSET + SVE_PT_SVE_SIZE(vq, SVE_PT_REGS_SVE); 1188 1168 iov.iov_base = malloc(iov.iov_len); ··· 1214 1188 regset = NT_ARM_SVE; 1215 1189 1216 1190 ret = ptrace(PTRACE_SETREGSET, child, regset, &iov); 1191 + if (ret != 0) 1192 + ksft_print_msg("Failed to write SVE: %s (%d)\n", 1193 + strerror(errno), errno); 1194 + 1195 + free(iov.iov_base); 1196 + } 1197 + 1198 + static void sve_write_fpsimd(pid_t child, struct test_config *config) 1199 + { 1200 + struct user_sve_header *sve; 1201 + struct user_fpsimd_state *fpsimd; 1202 + struct iovec iov; 1203 + int ret, vl, vq; 1204 + 1205 + vl = vl_expected(config); 1206 + vq = __sve_vq_from_vl(vl); 1207 + 1208 + if (!vl) 1209 + return; 1210 + 1211 + iov.iov_len = SVE_PT_SVE_OFFSET + SVE_PT_SVE_SIZE(vq, 1212 + SVE_PT_REGS_FPSIMD); 1213 + iov.iov_base = malloc(iov.iov_len); 1214 + if (!iov.iov_base) { 1215 + ksft_print_msg("Failed allocating %lu byte SVE write buffer\n", 1216 + iov.iov_len); 1217 + return; 1218 + } 1219 + memset(iov.iov_base, 0, iov.iov_len); 1220 + 1221 + sve = iov.iov_base; 1222 + sve->size = iov.iov_len; 1223 + sve->flags = SVE_PT_REGS_FPSIMD; 1224 + sve->vl = vl; 1225 + 1226 + fpsimd = iov.iov_base + SVE_PT_REGS_OFFSET; 1227 + memcpy(&fpsimd->vregs, v_expected, sizeof(v_expected)); 1228 + 1229 + ret = ptrace(PTRACE_SETREGSET, child, NT_ARM_SVE, &iov); 1217 1230 if (ret != 0) 1218 1231 ksft_print_msg("Failed to write SVE: %s (%d)\n", 1219 1232 strerror(errno), errno); ··· 1451 1386 .name = "SVE write", 1452 1387 .supported = sve_write_supported, 1453 1388 .set_expected_values = sve_write_expected, 1454 - .modify_values = sve_write, 1389 + .modify_values = sve_write_sve, 1390 + }, 1391 + { 1392 + .name = "SVE write FPSIMD format", 1393 + .supported = sve_write_fpsimd_supported, 1394 + .set_expected_values = fpsimd_write_expected, 1395 + .modify_values = sve_write_fpsimd, 1455 1396 }, 1456 1397 }; 1457 1398 ··· 1678 1607 * Run the test set if there is no SVE or SME, with those we 1679 1608 * have to pick a VL for each run. 1680 1609 */ 1681 - if (!sve_supported()) { 1610 + if (!sve_supported() && !sme_supported()) { 1682 1611 test_config.sve_vl_in = 0; 1683 1612 test_config.sve_vl_expected = 0; 1684 1613 test_config.sme_vl_in = 0;

+6 -6

tools/testing/selftests/arm64/fp/sve-ptrace.c

··· 170 170 memset(&sve, 0, sizeof(sve)); 171 171 sve.size = sizeof(sve); 172 172 sve.vl = sve_vl_from_vq(SVE_VQ_MIN); 173 - sve.flags = SVE_PT_VL_INHERIT; 173 + sve.flags = SVE_PT_VL_INHERIT | SVE_PT_REGS_SVE; 174 174 ret = set_sve(child, type, &sve); 175 175 if (ret != 0) { 176 176 ksft_test_result_fail("Failed to set %s SVE_PT_VL_INHERIT\n", ··· 235 235 /* Set the VL by doing a set with no register payload */ 236 236 memset(&sve, 0, sizeof(sve)); 237 237 sve.size = sizeof(sve); 238 + sve.flags = SVE_PT_REGS_SVE; 238 239 sve.vl = vl; 239 240 ret = set_sve(child, type, &sve); 240 241 if (ret != 0) { ··· 254 253 return; 255 254 } 256 255 257 - ksft_test_result(new_sve->vl = prctl_vl, "Set %s VL %u\n", 256 + ksft_test_result(new_sve->vl == prctl_vl, "Set %s VL %u\n", 258 257 type->name, vl); 259 258 260 259 free(new_sve); ··· 302 301 p[j] = j; 303 302 } 304 303 304 + /* This should only succeed for SVE */ 305 305 ret = set_sve(child, type, sve); 306 - ksft_test_result(ret == 0, "%s FPSIMD set via SVE: %d\n", 306 + ksft_test_result((type->regset == NT_ARM_SVE) == (ret == 0), 307 + "%s FPSIMD set via SVE: %d\n", 307 308 type->name, ret); 308 309 if (ret) 309 310 goto out; ··· 752 749 753 750 ksft_print_header(); 754 751 ksft_set_plan(EXPECTED_TESTS); 755 - 756 - if (!(getauxval(AT_HWCAP) & HWCAP_SVE)) 757 - ksft_exit_skip("SVE not available\n"); 758 752 759 753 child = fork(); 760 754 if (!child)

+6 -6

tools/testing/selftests/arm64/mte/check_buffer_fill.c

··· 31 31 int i, j, item; 32 32 bool err; 33 33 34 - mte_switch_mode(mode, MTE_ALLOW_NON_ZERO_TAG); 34 + mte_switch_mode(mode, MTE_ALLOW_NON_ZERO_TAG, false); 35 35 item = ARRAY_SIZE(sizes); 36 36 37 37 for (i = 0; i < item; i++) { ··· 68 68 bool err; 69 69 char *und_ptr = NULL; 70 70 71 - mte_switch_mode(mode, MTE_ALLOW_NON_ZERO_TAG); 71 + mte_switch_mode(mode, MTE_ALLOW_NON_ZERO_TAG, false); 72 72 item = ARRAY_SIZE(sizes); 73 73 for (i = 0; i < item; i++) { 74 74 ptr = (char *)mte_allocate_memory_tag_range(sizes[i], mem_type, 0, ··· 164 164 size_t tagged_size, overflow_size; 165 165 char *over_ptr = NULL; 166 166 167 - mte_switch_mode(mode, MTE_ALLOW_NON_ZERO_TAG); 167 + mte_switch_mode(mode, MTE_ALLOW_NON_ZERO_TAG, false); 168 168 item = ARRAY_SIZE(sizes); 169 169 for (i = 0; i < item; i++) { 170 170 ptr = (char *)mte_allocate_memory_tag_range(sizes[i], mem_type, 0, ··· 337 337 { 338 338 int i, item, result = KSFT_PASS; 339 339 340 - mte_switch_mode(mode, MTE_ALLOW_NON_ZERO_TAG); 340 + mte_switch_mode(mode, MTE_ALLOW_NON_ZERO_TAG, false); 341 341 item = ARRAY_SIZE(sizes); 342 342 cur_mte_cxt.fault_valid = false; 343 343 for (i = 0; i < item; i++) { ··· 368 368 int run, fd; 369 369 int total = ARRAY_SIZE(sizes); 370 370 371 - mte_switch_mode(mode, MTE_ALLOW_NON_ZERO_TAG); 371 + mte_switch_mode(mode, MTE_ALLOW_NON_ZERO_TAG, false); 372 372 for (run = 0; run < total; run++) { 373 373 /* check initial tags for anonymous mmap */ 374 374 ptr = (char *)mte_allocate_memory(sizes[run], mem_type, mapping, false); ··· 415 415 return err; 416 416 417 417 /* Register SIGSEGV handler */ 418 - mte_register_signal(SIGSEGV, mte_default_handler); 418 + mte_register_signal(SIGSEGV, mte_default_handler, false); 419 419 420 420 /* Set test plan */ 421 421 ksft_set_plan(20);

+4 -4

tools/testing/selftests/arm64/mte/check_child_memory.c

··· 88 88 int item = ARRAY_SIZE(sizes); 89 89 90 90 item = ARRAY_SIZE(sizes); 91 - mte_switch_mode(mode, MTE_ALLOW_NON_ZERO_TAG); 91 + mte_switch_mode(mode, MTE_ALLOW_NON_ZERO_TAG, false); 92 92 for (run = 0; run < item; run++) { 93 93 ptr = (char *)mte_allocate_memory_tag_range(sizes[run], mem_type, mapping, 94 94 UNDERFLOW, OVERFLOW); ··· 109 109 int run, fd, map_size, result = KSFT_PASS; 110 110 int total = ARRAY_SIZE(sizes); 111 111 112 - mte_switch_mode(mode, MTE_ALLOW_NON_ZERO_TAG); 112 + mte_switch_mode(mode, MTE_ALLOW_NON_ZERO_TAG, false); 113 113 for (run = 0; run < total; run++) { 114 114 fd = create_temp_file(); 115 115 if (fd == -1) ··· 160 160 return err; 161 161 162 162 /* Register SIGSEGV handler */ 163 - mte_register_signal(SIGSEGV, mte_default_handler); 164 - mte_register_signal(SIGBUS, mte_default_handler); 163 + mte_register_signal(SIGSEGV, mte_default_handler, false); 164 + mte_register_signal(SIGBUS, mte_default_handler, false); 165 165 166 166 /* Set test plan */ 167 167 ksft_set_plan(12);

+5 -5

tools/testing/selftests/arm64/mte/check_hugetlb_options.c

··· 151 151 152 152 map_size = default_huge_page_size(); 153 153 154 - mte_switch_mode(mode, MTE_ALLOW_NON_ZERO_TAG); 154 + mte_switch_mode(mode, MTE_ALLOW_NON_ZERO_TAG, false); 155 155 map_ptr = (char *)mte_allocate_memory(map_size, mem_type, mapping, false); 156 156 if (check_allocated_memory(map_ptr, map_size, mem_type, false) != KSFT_PASS) 157 157 return KSFT_FAIL; ··· 180 180 unsigned long map_size; 181 181 182 182 prot_flag = PROT_READ | PROT_WRITE; 183 - mte_switch_mode(mode, MTE_ALLOW_NON_ZERO_TAG); 183 + mte_switch_mode(mode, MTE_ALLOW_NON_ZERO_TAG, false); 184 184 map_size = default_huge_page_size(); 185 185 map_ptr = (char *)mte_allocate_memory_tag_range(map_size, mem_type, mapping, 186 186 0, 0); ··· 210 210 211 211 map_size = default_huge_page_size(); 212 212 213 - mte_switch_mode(mode, MTE_ALLOW_NON_ZERO_TAG); 213 + mte_switch_mode(mode, MTE_ALLOW_NON_ZERO_TAG, false); 214 214 ptr = (char *)mte_allocate_memory_tag_range(map_size, mem_type, mapping, 215 215 0, 0); 216 216 if (check_allocated_memory_range(ptr, map_size, mem_type, ··· 235 235 return err; 236 236 237 237 /* Register signal handlers */ 238 - mte_register_signal(SIGBUS, mte_default_handler); 239 - mte_register_signal(SIGSEGV, mte_default_handler); 238 + mte_register_signal(SIGBUS, mte_default_handler, false); 239 + mte_register_signal(SIGSEGV, mte_default_handler, false); 240 240 241 241 allocate_hugetlb(); 242 242

+3 -3

tools/testing/selftests/arm64/mte/check_ksm_options.c

··· 106 106 return err; 107 107 } 108 108 109 - mte_switch_mode(mode, MTE_ALLOW_NON_ZERO_TAG); 109 + mte_switch_mode(mode, MTE_ALLOW_NON_ZERO_TAG, false); 110 110 ptr = mte_allocate_memory(TEST_UNIT * page_sz, mem_type, mapping, true); 111 111 if (check_allocated_memory(ptr, TEST_UNIT * page_sz, mem_type, false) != KSFT_PASS) 112 112 return KSFT_FAIL; ··· 141 141 return KSFT_FAIL; 142 142 } 143 143 /* Register signal handlers */ 144 - mte_register_signal(SIGBUS, mte_default_handler); 145 - mte_register_signal(SIGSEGV, mte_default_handler); 144 + mte_register_signal(SIGBUS, mte_default_handler, false); 145 + mte_register_signal(SIGSEGV, mte_default_handler, false); 146 146 147 147 /* Set test plan */ 148 148 ksft_set_plan(4);

+817 -73

tools/testing/selftests/arm64/mte/check_mmap_options.c

··· 3 3 4 4 #define _GNU_SOURCE 5 5 6 + #include <assert.h> 6 7 #include <errno.h> 7 8 #include <fcntl.h> 8 9 #include <signal.h> ··· 24 23 #define OVERFLOW MT_GRANULE_SIZE 25 24 #define TAG_CHECK_ON 0 26 25 #define TAG_CHECK_OFF 1 26 + #define ATAG_CHECK_ON 1 27 + #define ATAG_CHECK_OFF 0 28 + 29 + #define TEST_NAME_MAX 256 30 + 31 + enum mte_mem_check_type { 32 + CHECK_ANON_MEM = 0, 33 + CHECK_FILE_MEM = 1, 34 + CHECK_CLEAR_PROT_MTE = 2, 35 + }; 36 + 37 + enum mte_tag_op_type { 38 + TAG_OP_ALL = 0, 39 + TAG_OP_STONLY = 1, 40 + }; 41 + 42 + struct check_mmap_testcase { 43 + int check_type; 44 + int mem_type; 45 + int mte_sync; 46 + int mapping; 47 + int tag_check; 48 + int atag_check; 49 + int tag_op; 50 + bool enable_tco; 51 + }; 52 + 53 + #define TAG_OP_ALL 0 54 + #define TAG_OP_STONLY 1 27 55 28 56 static size_t page_size; 29 57 static int sizes[] = { ··· 60 30 /* page size - 1*/ 0, /* page_size */ 0, /* page size + 1 */ 0 61 31 }; 62 32 63 - static int check_mte_memory(char *ptr, int size, int mode, int tag_check) 33 + static int check_mte_memory(char *ptr, int size, int mode, 34 + int tag_check,int atag_check, int tag_op) 64 35 { 36 + char buf[MT_GRANULE_SIZE]; 37 + 38 + if (!mtefar_support && atag_check == ATAG_CHECK_ON) 39 + return KSFT_SKIP; 40 + 41 + if (atag_check == ATAG_CHECK_ON) 42 + ptr = mte_insert_atag(ptr); 43 + 65 44 mte_initialize_current_context(mode, (uintptr_t)ptr, size); 66 45 memset(ptr, '1', size); 67 46 mte_wait_after_trig(); ··· 93 54 if (cur_mte_cxt.fault_valid == true && tag_check == TAG_CHECK_OFF) 94 55 return KSFT_FAIL; 95 56 57 + if (tag_op == TAG_OP_STONLY) { 58 + mte_initialize_current_context(mode, (uintptr_t)ptr, -UNDERFLOW); 59 + memcpy(buf, ptr - UNDERFLOW, MT_GRANULE_SIZE); 60 + mte_wait_after_trig(); 61 + if (cur_mte_cxt.fault_valid == true) 62 + return KSFT_FAIL; 63 + 64 + mte_initialize_current_context(mode, (uintptr_t)ptr, size + OVERFLOW); 65 + memcpy(buf, ptr + size, MT_GRANULE_SIZE); 66 + mte_wait_after_trig(); 67 + if (cur_mte_cxt.fault_valid == true) 68 + return KSFT_FAIL; 69 + } 70 + 96 71 return KSFT_PASS; 97 72 } 98 73 99 - static int check_anonymous_memory_mapping(int mem_type, int mode, int mapping, int tag_check) 74 + static int check_anonymous_memory_mapping(int mem_type, int mode, int mapping, 75 + int tag_check, int atag_check, int tag_op) 100 76 { 101 77 char *ptr, *map_ptr; 102 78 int run, result, map_size; 103 79 int item = ARRAY_SIZE(sizes); 104 80 105 - mte_switch_mode(mode, MTE_ALLOW_NON_ZERO_TAG); 81 + if (tag_op == TAG_OP_STONLY && !mtestonly_support) 82 + return KSFT_SKIP; 83 + 84 + mte_switch_mode(mode, MTE_ALLOW_NON_ZERO_TAG, tag_op); 106 85 for (run = 0; run < item; run++) { 107 86 map_size = sizes[run] + OVERFLOW + UNDERFLOW; 108 87 map_ptr = (char *)mte_allocate_memory(map_size, mem_type, mapping, false); ··· 136 79 munmap((void *)map_ptr, map_size); 137 80 return KSFT_FAIL; 138 81 } 139 - result = check_mte_memory(ptr, sizes[run], mode, tag_check); 82 + result = check_mte_memory(ptr, sizes[run], mode, tag_check, atag_check, tag_op); 140 83 mte_clear_tags((void *)ptr, sizes[run]); 141 84 mte_free_memory((void *)map_ptr, map_size, mem_type, false); 142 - if (result == KSFT_FAIL) 143 - return KSFT_FAIL; 85 + if (result != KSFT_PASS) 86 + return result; 144 87 } 145 88 return KSFT_PASS; 146 89 } 147 90 148 - static int check_file_memory_mapping(int mem_type, int mode, int mapping, int tag_check) 91 + static int check_file_memory_mapping(int mem_type, int mode, int mapping, 92 + int tag_check, int atag_check, int tag_op) 149 93 { 150 94 char *ptr, *map_ptr; 151 95 int run, fd, map_size; 152 96 int total = ARRAY_SIZE(sizes); 153 97 int result = KSFT_PASS; 154 98 155 - mte_switch_mode(mode, MTE_ALLOW_NON_ZERO_TAG); 99 + if (tag_op == TAG_OP_STONLY && !mtestonly_support) 100 + return KSFT_SKIP; 101 + 102 + mte_switch_mode(mode, MTE_ALLOW_NON_ZERO_TAG, tag_op); 156 103 for (run = 0; run < total; run++) { 157 104 fd = create_temp_file(); 158 105 if (fd == -1) ··· 178 117 close(fd); 179 118 return KSFT_FAIL; 180 119 } 181 - result = check_mte_memory(ptr, sizes[run], mode, tag_check); 120 + result = check_mte_memory(ptr, sizes[run], mode, tag_check, atag_check, tag_op); 182 121 mte_clear_tags((void *)ptr, sizes[run]); 183 122 munmap((void *)map_ptr, map_size); 184 123 close(fd); 185 - if (result == KSFT_FAIL) 186 - break; 124 + if (result != KSFT_PASS) 125 + return result; 187 126 } 188 - return result; 127 + return KSFT_PASS; 189 128 } 190 129 191 - static int check_clear_prot_mte_flag(int mem_type, int mode, int mapping) 130 + static int check_clear_prot_mte_flag(int mem_type, int mode, int mapping, int atag_check) 192 131 { 193 132 char *ptr, *map_ptr; 194 133 int run, prot_flag, result, fd, map_size; 195 134 int total = ARRAY_SIZE(sizes); 196 135 197 136 prot_flag = PROT_READ | PROT_WRITE; 198 - mte_switch_mode(mode, MTE_ALLOW_NON_ZERO_TAG); 137 + mte_switch_mode(mode, MTE_ALLOW_NON_ZERO_TAG, false); 199 138 for (run = 0; run < total; run++) { 200 139 map_size = sizes[run] + OVERFLOW + UNDERFLOW; 201 140 ptr = (char *)mte_allocate_memory_tag_range(sizes[run], mem_type, mapping, ··· 211 150 ksft_print_msg("FAIL: mprotect not ignoring clear PROT_MTE property\n"); 212 151 return KSFT_FAIL; 213 152 } 214 - result = check_mte_memory(ptr, sizes[run], mode, TAG_CHECK_ON); 153 + result = check_mte_memory(ptr, sizes[run], mode, TAG_CHECK_ON, atag_check, TAG_OP_ALL); 215 154 mte_free_memory_tag_range((void *)ptr, sizes[run], mem_type, UNDERFLOW, OVERFLOW); 216 155 if (result != KSFT_PASS) 217 - return KSFT_FAIL; 156 + return result; 218 157 219 158 fd = create_temp_file(); 220 159 if (fd == -1) ··· 235 174 close(fd); 236 175 return KSFT_FAIL; 237 176 } 238 - result = check_mte_memory(ptr, sizes[run], mode, TAG_CHECK_ON); 177 + result = check_mte_memory(ptr, sizes[run], mode, TAG_CHECK_ON, atag_check, TAG_OP_ALL); 239 178 mte_free_memory_tag_range((void *)ptr, sizes[run], mem_type, UNDERFLOW, OVERFLOW); 240 179 close(fd); 241 180 if (result != KSFT_PASS) 242 - return KSFT_FAIL; 181 + return result; 243 182 } 244 183 return KSFT_PASS; 245 184 } 246 185 186 + const char *format_test_name(struct check_mmap_testcase *tc) 187 + { 188 + static char test_name[TEST_NAME_MAX]; 189 + const char *check_type_str; 190 + const char *mem_type_str; 191 + const char *sync_str; 192 + const char *mapping_str; 193 + const char *tag_check_str; 194 + const char *atag_check_str; 195 + const char *tag_op_str; 196 + 197 + switch (tc->check_type) { 198 + case CHECK_ANON_MEM: 199 + check_type_str = "anonymous memory"; 200 + break; 201 + case CHECK_FILE_MEM: 202 + check_type_str = "file memory"; 203 + break; 204 + case CHECK_CLEAR_PROT_MTE: 205 + check_type_str = "clear PROT_MTE flags"; 206 + break; 207 + default: 208 + assert(0); 209 + break; 210 + } 211 + 212 + switch (tc->mem_type) { 213 + case USE_MMAP: 214 + mem_type_str = "mmap"; 215 + break; 216 + case USE_MPROTECT: 217 + mem_type_str = "mmap/mprotect"; 218 + break; 219 + default: 220 + assert(0); 221 + break; 222 + } 223 + 224 + switch (tc->mte_sync) { 225 + case MTE_NONE_ERR: 226 + sync_str = "no error"; 227 + break; 228 + case MTE_SYNC_ERR: 229 + sync_str = "sync error"; 230 + break; 231 + case MTE_ASYNC_ERR: 232 + sync_str = "async error"; 233 + break; 234 + default: 235 + assert(0); 236 + break; 237 + } 238 + 239 + switch (tc->mapping) { 240 + case MAP_SHARED: 241 + mapping_str = "shared"; 242 + break; 243 + case MAP_PRIVATE: 244 + mapping_str = "private"; 245 + break; 246 + default: 247 + assert(0); 248 + break; 249 + } 250 + 251 + switch (tc->tag_check) { 252 + case TAG_CHECK_ON: 253 + tag_check_str = "tag check on"; 254 + break; 255 + case TAG_CHECK_OFF: 256 + tag_check_str = "tag check off"; 257 + break; 258 + default: 259 + assert(0); 260 + break; 261 + } 262 + 263 + switch (tc->atag_check) { 264 + case ATAG_CHECK_ON: 265 + atag_check_str = "with address tag [63:60]"; 266 + break; 267 + case ATAG_CHECK_OFF: 268 + atag_check_str = "without address tag [63:60]"; 269 + break; 270 + default: 271 + assert(0); 272 + break; 273 + } 274 + 275 + snprintf(test_name, sizeof(test_name), 276 + "Check %s with %s mapping, %s mode, %s memory and %s (%s)\n", 277 + check_type_str, mapping_str, sync_str, mem_type_str, 278 + tag_check_str, atag_check_str); 279 + 280 + switch (tc->tag_op) { 281 + case TAG_OP_ALL: 282 + tag_op_str = ""; 283 + break; 284 + case TAG_OP_STONLY: 285 + tag_op_str = " / store-only"; 286 + break; 287 + default: 288 + assert(0); 289 + break; 290 + } 291 + 292 + snprintf(test_name, TEST_NAME_MAX, 293 + "Check %s with %s mapping, %s mode, %s memory and %s (%s%s)\n", 294 + check_type_str, mapping_str, sync_str, mem_type_str, 295 + tag_check_str, atag_check_str, tag_op_str); 296 + 297 + return test_name; 298 + } 299 + 247 300 int main(int argc, char *argv[]) 248 301 { 249 - int err; 302 + int err, i; 250 303 int item = ARRAY_SIZE(sizes); 304 + struct check_mmap_testcase test_cases[]= { 305 + { 306 + .check_type = CHECK_ANON_MEM, 307 + .mem_type = USE_MMAP, 308 + .mte_sync = MTE_SYNC_ERR, 309 + .mapping = MAP_PRIVATE, 310 + .tag_check = TAG_CHECK_OFF, 311 + .atag_check = ATAG_CHECK_OFF, 312 + .tag_op = TAG_OP_ALL, 313 + .enable_tco = true, 314 + }, 315 + { 316 + .check_type = CHECK_FILE_MEM, 317 + .mem_type = USE_MPROTECT, 318 + .mte_sync = MTE_SYNC_ERR, 319 + .mapping = MAP_PRIVATE, 320 + .tag_check = TAG_CHECK_OFF, 321 + .atag_check = ATAG_CHECK_OFF, 322 + .tag_op = TAG_OP_ALL, 323 + .enable_tco = true, 324 + }, 325 + { 326 + .check_type = CHECK_ANON_MEM, 327 + .mem_type = USE_MMAP, 328 + .mte_sync = MTE_NONE_ERR, 329 + .mapping = MAP_PRIVATE, 330 + .tag_check = TAG_CHECK_OFF, 331 + .atag_check = ATAG_CHECK_OFF, 332 + .tag_op = TAG_OP_ALL, 333 + .enable_tco = false, 334 + }, 335 + { 336 + .check_type = CHECK_FILE_MEM, 337 + .mem_type = USE_MPROTECT, 338 + .mte_sync = MTE_NONE_ERR, 339 + .mapping = MAP_PRIVATE, 340 + .tag_check = TAG_CHECK_OFF, 341 + .atag_check = ATAG_CHECK_OFF, 342 + .tag_op = TAG_OP_ALL, 343 + .enable_tco = false, 344 + }, 345 + { 346 + .check_type = CHECK_ANON_MEM, 347 + .mem_type = USE_MMAP, 348 + .mte_sync = MTE_SYNC_ERR, 349 + .mapping = MAP_PRIVATE, 350 + .tag_check = TAG_CHECK_ON, 351 + .atag_check = ATAG_CHECK_OFF, 352 + .tag_op = TAG_OP_ALL, 353 + .enable_tco = false, 354 + }, 355 + { 356 + .check_type = CHECK_ANON_MEM, 357 + .mem_type = USE_MPROTECT, 358 + .mte_sync = MTE_SYNC_ERR, 359 + .mapping = MAP_PRIVATE, 360 + .tag_check = TAG_CHECK_ON, 361 + .atag_check = ATAG_CHECK_OFF, 362 + .tag_op = TAG_OP_ALL, 363 + .enable_tco = false, 364 + }, 365 + { 366 + .check_type = CHECK_ANON_MEM, 367 + .mem_type = USE_MMAP, 368 + .mte_sync = MTE_SYNC_ERR, 369 + .mapping = MAP_SHARED, 370 + .tag_check = TAG_CHECK_ON, 371 + .atag_check = ATAG_CHECK_OFF, 372 + .tag_op = TAG_OP_ALL, 373 + .enable_tco = false, 374 + }, 375 + { 376 + .check_type = CHECK_ANON_MEM, 377 + .mem_type = USE_MPROTECT, 378 + .mte_sync = MTE_SYNC_ERR, 379 + .mapping = MAP_SHARED, 380 + .tag_check = TAG_CHECK_ON, 381 + .atag_check = ATAG_CHECK_OFF, 382 + .tag_op = TAG_OP_ALL, 383 + .enable_tco = false, 384 + }, 385 + { 386 + .check_type = CHECK_ANON_MEM, 387 + .mem_type = USE_MMAP, 388 + .mte_sync = MTE_ASYNC_ERR, 389 + .mapping = MAP_PRIVATE, 390 + .tag_check = TAG_CHECK_ON, 391 + .atag_check = ATAG_CHECK_OFF, 392 + .tag_op = TAG_OP_ALL, 393 + .enable_tco = false, 394 + }, 395 + { 396 + .check_type = CHECK_ANON_MEM, 397 + .mem_type = USE_MPROTECT, 398 + .mte_sync = MTE_ASYNC_ERR, 399 + .mapping = MAP_PRIVATE, 400 + .tag_check = TAG_CHECK_ON, 401 + .atag_check = ATAG_CHECK_OFF, 402 + .tag_op = TAG_OP_ALL, 403 + .enable_tco = false, 404 + }, 405 + { 406 + .check_type = CHECK_ANON_MEM, 407 + .mem_type = USE_MMAP, 408 + .mte_sync = MTE_ASYNC_ERR, 409 + .mapping = MAP_SHARED, 410 + .tag_check = TAG_CHECK_ON, 411 + .atag_check = ATAG_CHECK_OFF, 412 + .tag_op = TAG_OP_ALL, 413 + .enable_tco = false, 414 + }, 415 + { 416 + .check_type = CHECK_ANON_MEM, 417 + .mem_type = USE_MPROTECT, 418 + .mte_sync = MTE_ASYNC_ERR, 419 + .mapping = MAP_SHARED, 420 + .tag_check = TAG_CHECK_ON, 421 + .atag_check = ATAG_CHECK_OFF, 422 + .tag_op = TAG_OP_ALL, 423 + .enable_tco = false, 424 + }, 425 + { 426 + .check_type = CHECK_FILE_MEM, 427 + .mem_type = USE_MMAP, 428 + .mte_sync = MTE_SYNC_ERR, 429 + .mapping = MAP_PRIVATE, 430 + .tag_check = TAG_CHECK_ON, 431 + .atag_check = ATAG_CHECK_OFF, 432 + .tag_op = TAG_OP_ALL, 433 + .enable_tco = false, 434 + }, 435 + { 436 + .check_type = CHECK_FILE_MEM, 437 + .mem_type = USE_MPROTECT, 438 + .mte_sync = MTE_SYNC_ERR, 439 + .mapping = MAP_PRIVATE, 440 + .tag_check = TAG_CHECK_ON, 441 + .atag_check = ATAG_CHECK_OFF, 442 + .tag_op = TAG_OP_ALL, 443 + .enable_tco = false, 444 + }, 445 + { 446 + .check_type = CHECK_FILE_MEM, 447 + .mem_type = USE_MMAP, 448 + .mte_sync = MTE_SYNC_ERR, 449 + .mapping = MAP_SHARED, 450 + .tag_check = TAG_CHECK_ON, 451 + .atag_check = ATAG_CHECK_OFF, 452 + .tag_op = TAG_OP_ALL, 453 + .enable_tco = false, 454 + }, 455 + { 456 + .check_type = CHECK_FILE_MEM, 457 + .mem_type = USE_MPROTECT, 458 + .mte_sync = MTE_SYNC_ERR, 459 + .mapping = MAP_SHARED, 460 + .tag_check = TAG_CHECK_ON, 461 + .atag_check = ATAG_CHECK_OFF, 462 + .tag_op = TAG_OP_ALL, 463 + .enable_tco = false, 464 + }, 465 + { 466 + .check_type = CHECK_FILE_MEM, 467 + .mem_type = USE_MMAP, 468 + .mte_sync = MTE_ASYNC_ERR, 469 + .mapping = MAP_PRIVATE, 470 + .tag_check = TAG_CHECK_ON, 471 + .atag_check = ATAG_CHECK_OFF, 472 + .tag_op = TAG_OP_ALL, 473 + .enable_tco = false, 474 + }, 475 + { 476 + .check_type = CHECK_FILE_MEM, 477 + .mem_type = USE_MPROTECT, 478 + .mte_sync = MTE_ASYNC_ERR, 479 + .mapping = MAP_PRIVATE, 480 + .tag_check = TAG_CHECK_ON, 481 + .atag_check = ATAG_CHECK_OFF, 482 + .tag_op = TAG_OP_ALL, 483 + .enable_tco = false, 484 + }, 485 + { 486 + .check_type = CHECK_FILE_MEM, 487 + .mem_type = USE_MMAP, 488 + .mte_sync = MTE_ASYNC_ERR, 489 + .mapping = MAP_SHARED, 490 + .tag_check = TAG_CHECK_ON, 491 + .atag_check = ATAG_CHECK_OFF, 492 + .tag_op = TAG_OP_ALL, 493 + .enable_tco = false, 494 + }, 495 + { 496 + .check_type = CHECK_FILE_MEM, 497 + .mem_type = USE_MPROTECT, 498 + .mte_sync = MTE_ASYNC_ERR, 499 + .mapping = MAP_SHARED, 500 + .tag_check = TAG_CHECK_ON, 501 + .atag_check = ATAG_CHECK_OFF, 502 + .tag_op = TAG_OP_ALL, 503 + .enable_tco = false, 504 + }, 505 + { 506 + .check_type = CHECK_CLEAR_PROT_MTE, 507 + .mem_type = USE_MMAP, 508 + .mte_sync = MTE_SYNC_ERR, 509 + .mapping = MAP_PRIVATE, 510 + .tag_check = TAG_CHECK_ON, 511 + .atag_check = ATAG_CHECK_OFF, 512 + .tag_op = TAG_OP_ALL, 513 + .enable_tco = false, 514 + }, 515 + { 516 + .check_type = CHECK_CLEAR_PROT_MTE, 517 + .mem_type = USE_MPROTECT, 518 + .mte_sync = MTE_SYNC_ERR, 519 + .mapping = MAP_PRIVATE, 520 + .tag_check = TAG_CHECK_ON, 521 + .atag_check = ATAG_CHECK_OFF, 522 + .tag_op = TAG_OP_ALL, 523 + .enable_tco = false, 524 + }, 525 + { 526 + .check_type = CHECK_ANON_MEM, 527 + .mem_type = USE_MMAP, 528 + .mte_sync = MTE_SYNC_ERR, 529 + .mapping = MAP_PRIVATE, 530 + .tag_check = TAG_CHECK_ON, 531 + .atag_check = ATAG_CHECK_ON, 532 + .tag_op = TAG_OP_ALL, 533 + .enable_tco = false, 534 + }, 535 + { 536 + .check_type = CHECK_ANON_MEM, 537 + .mem_type = USE_MPROTECT, 538 + .mte_sync = MTE_SYNC_ERR, 539 + .mapping = MAP_PRIVATE, 540 + .tag_check = TAG_CHECK_ON, 541 + .atag_check = ATAG_CHECK_ON, 542 + .tag_op = TAG_OP_ALL, 543 + .enable_tco = false, 544 + }, 545 + { 546 + .check_type = CHECK_ANON_MEM, 547 + .mem_type = USE_MMAP, 548 + .mte_sync = MTE_SYNC_ERR, 549 + .mapping = MAP_SHARED, 550 + .tag_check = TAG_CHECK_ON, 551 + .atag_check = ATAG_CHECK_ON, 552 + .tag_op = TAG_OP_ALL, 553 + .enable_tco = false, 554 + }, 555 + { 556 + .check_type = CHECK_ANON_MEM, 557 + .mem_type = USE_MPROTECT, 558 + .mte_sync = MTE_SYNC_ERR, 559 + .mapping = MAP_SHARED, 560 + .tag_check = TAG_CHECK_ON, 561 + .atag_check = ATAG_CHECK_ON, 562 + .tag_op = TAG_OP_ALL, 563 + .enable_tco = false, 564 + }, 565 + { 566 + .check_type = CHECK_FILE_MEM, 567 + .mem_type = USE_MMAP, 568 + .mte_sync = MTE_SYNC_ERR, 569 + .mapping = MAP_PRIVATE, 570 + .tag_check = TAG_CHECK_ON, 571 + .atag_check = ATAG_CHECK_ON, 572 + .tag_op = TAG_OP_ALL, 573 + .enable_tco = false, 574 + }, 575 + { 576 + .check_type = CHECK_FILE_MEM, 577 + .mem_type = USE_MPROTECT, 578 + .mte_sync = MTE_SYNC_ERR, 579 + .mapping = MAP_PRIVATE, 580 + .tag_check = TAG_CHECK_ON, 581 + .atag_check = ATAG_CHECK_ON, 582 + .tag_op = TAG_OP_ALL, 583 + .enable_tco = false, 584 + }, 585 + { 586 + .check_type = CHECK_FILE_MEM, 587 + .mem_type = USE_MMAP, 588 + .mte_sync = MTE_SYNC_ERR, 589 + .mapping = MAP_SHARED, 590 + .tag_check = TAG_CHECK_ON, 591 + .atag_check = ATAG_CHECK_ON, 592 + .tag_op = TAG_OP_ALL, 593 + .enable_tco = false, 594 + }, 595 + { 596 + .check_type = CHECK_FILE_MEM, 597 + .mem_type = USE_MPROTECT, 598 + .mte_sync = MTE_SYNC_ERR, 599 + .mapping = MAP_SHARED, 600 + .tag_check = TAG_CHECK_ON, 601 + .atag_check = ATAG_CHECK_ON, 602 + .tag_op = TAG_OP_ALL, 603 + .enable_tco = false, 604 + }, 605 + { 606 + .check_type = CHECK_FILE_MEM, 607 + .mem_type = USE_MMAP, 608 + .mte_sync = MTE_ASYNC_ERR, 609 + .mapping = MAP_PRIVATE, 610 + .tag_check = TAG_CHECK_ON, 611 + .atag_check = ATAG_CHECK_ON, 612 + .tag_op = TAG_OP_ALL, 613 + .enable_tco = false, 614 + }, 615 + { 616 + .check_type = CHECK_ANON_MEM, 617 + .mem_type = USE_MMAP, 618 + .mte_sync = MTE_SYNC_ERR, 619 + .mapping = MAP_PRIVATE, 620 + .tag_check = TAG_CHECK_ON, 621 + .atag_check = ATAG_CHECK_OFF, 622 + .tag_op = TAG_OP_STONLY, 623 + .enable_tco = false, 624 + }, 625 + { 626 + .check_type = CHECK_ANON_MEM, 627 + .mem_type = USE_MPROTECT, 628 + .mte_sync = MTE_SYNC_ERR, 629 + .mapping = MAP_PRIVATE, 630 + .tag_check = TAG_CHECK_ON, 631 + .atag_check = ATAG_CHECK_OFF, 632 + .tag_op = TAG_OP_STONLY, 633 + .enable_tco = false, 634 + }, 635 + { 636 + .check_type = CHECK_ANON_MEM, 637 + .mem_type = USE_MMAP, 638 + .mte_sync = MTE_SYNC_ERR, 639 + .mapping = MAP_SHARED, 640 + .tag_check = TAG_CHECK_ON, 641 + .atag_check = ATAG_CHECK_OFF, 642 + .tag_op = TAG_OP_STONLY, 643 + .enable_tco = false, 644 + }, 645 + { 646 + .check_type = CHECK_ANON_MEM, 647 + .mem_type = USE_MPROTECT, 648 + .mte_sync = MTE_SYNC_ERR, 649 + .mapping = MAP_SHARED, 650 + .tag_check = TAG_CHECK_ON, 651 + .atag_check = ATAG_CHECK_OFF, 652 + .tag_op = TAG_OP_STONLY, 653 + .enable_tco = false, 654 + }, 655 + { 656 + .check_type = CHECK_ANON_MEM, 657 + .mem_type = USE_MMAP, 658 + .mte_sync = MTE_ASYNC_ERR, 659 + .mapping = MAP_PRIVATE, 660 + .tag_check = TAG_CHECK_ON, 661 + .atag_check = ATAG_CHECK_OFF, 662 + .tag_op = TAG_OP_STONLY, 663 + .enable_tco = false, 664 + }, 665 + { 666 + .check_type = CHECK_ANON_MEM, 667 + .mem_type = USE_MPROTECT, 668 + .mte_sync = MTE_ASYNC_ERR, 669 + .mapping = MAP_PRIVATE, 670 + .tag_check = TAG_CHECK_ON, 671 + .atag_check = ATAG_CHECK_OFF, 672 + .tag_op = TAG_OP_STONLY, 673 + .enable_tco = false, 674 + }, 675 + { 676 + .check_type = CHECK_ANON_MEM, 677 + .mem_type = USE_MMAP, 678 + .mte_sync = MTE_ASYNC_ERR, 679 + .mapping = MAP_SHARED, 680 + .tag_check = TAG_CHECK_ON, 681 + .atag_check = ATAG_CHECK_OFF, 682 + .tag_op = TAG_OP_STONLY, 683 + .enable_tco = false, 684 + }, 685 + { 686 + .check_type = CHECK_ANON_MEM, 687 + .mem_type = USE_MPROTECT, 688 + .mte_sync = MTE_ASYNC_ERR, 689 + .mapping = MAP_SHARED, 690 + .tag_check = TAG_CHECK_ON, 691 + .atag_check = ATAG_CHECK_OFF, 692 + .tag_op = TAG_OP_STONLY, 693 + .enable_tco = false, 694 + }, 695 + { 696 + .check_type = CHECK_FILE_MEM, 697 + .mem_type = USE_MMAP, 698 + .mte_sync = MTE_SYNC_ERR, 699 + .mapping = MAP_PRIVATE, 700 + .tag_check = TAG_CHECK_ON, 701 + .atag_check = ATAG_CHECK_OFF, 702 + .tag_op = TAG_OP_STONLY, 703 + .enable_tco = false, 704 + }, 705 + { 706 + .check_type = CHECK_FILE_MEM, 707 + .mem_type = USE_MPROTECT, 708 + .mte_sync = MTE_SYNC_ERR, 709 + .mapping = MAP_PRIVATE, 710 + .tag_check = TAG_CHECK_ON, 711 + .atag_check = ATAG_CHECK_OFF, 712 + .tag_op = TAG_OP_STONLY, 713 + .enable_tco = false, 714 + }, 715 + { 716 + .check_type = CHECK_FILE_MEM, 717 + .mem_type = USE_MMAP, 718 + .mte_sync = MTE_SYNC_ERR, 719 + .mapping = MAP_SHARED, 720 + .tag_check = TAG_CHECK_ON, 721 + .atag_check = ATAG_CHECK_OFF, 722 + .tag_op = TAG_OP_STONLY, 723 + .enable_tco = false, 724 + }, 725 + { 726 + .check_type = CHECK_FILE_MEM, 727 + .mem_type = USE_MPROTECT, 728 + .mte_sync = MTE_SYNC_ERR, 729 + .mapping = MAP_SHARED, 730 + .tag_check = TAG_CHECK_ON, 731 + .atag_check = ATAG_CHECK_OFF, 732 + .tag_op = TAG_OP_STONLY, 733 + .enable_tco = false, 734 + }, 735 + { 736 + .check_type = CHECK_FILE_MEM, 737 + .mem_type = USE_MMAP, 738 + .mte_sync = MTE_ASYNC_ERR, 739 + .mapping = MAP_PRIVATE, 740 + .tag_check = TAG_CHECK_ON, 741 + .atag_check = ATAG_CHECK_OFF, 742 + .tag_op = TAG_OP_STONLY, 743 + .enable_tco = false, 744 + }, 745 + { 746 + .check_type = CHECK_FILE_MEM, 747 + .mem_type = USE_MPROTECT, 748 + .mte_sync = MTE_ASYNC_ERR, 749 + .mapping = MAP_PRIVATE, 750 + .tag_check = TAG_CHECK_ON, 751 + .atag_check = ATAG_CHECK_OFF, 752 + .tag_op = TAG_OP_STONLY, 753 + .enable_tco = false, 754 + }, 755 + { 756 + .check_type = CHECK_FILE_MEM, 757 + .mem_type = USE_MMAP, 758 + .mte_sync = MTE_ASYNC_ERR, 759 + .mapping = MAP_SHARED, 760 + .tag_check = TAG_CHECK_ON, 761 + .atag_check = ATAG_CHECK_OFF, 762 + .tag_op = TAG_OP_STONLY, 763 + .enable_tco = false, 764 + }, 765 + { 766 + .check_type = CHECK_FILE_MEM, 767 + .mem_type = USE_MPROTECT, 768 + .mte_sync = MTE_ASYNC_ERR, 769 + .mapping = MAP_SHARED, 770 + .tag_check = TAG_CHECK_ON, 771 + .atag_check = ATAG_CHECK_OFF, 772 + .tag_op = TAG_OP_STONLY, 773 + .enable_tco = false, 774 + }, 775 + { 776 + .check_type = CHECK_ANON_MEM, 777 + .mem_type = USE_MMAP, 778 + .mte_sync = MTE_SYNC_ERR, 779 + .mapping = MAP_PRIVATE, 780 + .tag_check = TAG_CHECK_ON, 781 + .atag_check = ATAG_CHECK_ON, 782 + .tag_op = TAG_OP_STONLY, 783 + .enable_tco = false, 784 + }, 785 + { 786 + .check_type = CHECK_ANON_MEM, 787 + .mem_type = USE_MPROTECT, 788 + .mte_sync = MTE_SYNC_ERR, 789 + .mapping = MAP_PRIVATE, 790 + .tag_check = TAG_CHECK_ON, 791 + .atag_check = ATAG_CHECK_ON, 792 + .tag_op = TAG_OP_STONLY, 793 + .enable_tco = false, 794 + }, 795 + { 796 + .check_type = CHECK_ANON_MEM, 797 + .mem_type = USE_MMAP, 798 + .mte_sync = MTE_SYNC_ERR, 799 + .mapping = MAP_SHARED, 800 + .tag_check = TAG_CHECK_ON, 801 + .atag_check = ATAG_CHECK_ON, 802 + .tag_op = TAG_OP_STONLY, 803 + .enable_tco = false, 804 + }, 805 + { 806 + .check_type = CHECK_ANON_MEM, 807 + .mem_type = USE_MPROTECT, 808 + .mte_sync = MTE_SYNC_ERR, 809 + .mapping = MAP_SHARED, 810 + .tag_check = TAG_CHECK_ON, 811 + .atag_check = ATAG_CHECK_ON, 812 + .tag_op = TAG_OP_STONLY, 813 + .enable_tco = false, 814 + }, 815 + { 816 + .check_type = CHECK_FILE_MEM, 817 + .mem_type = USE_MMAP, 818 + .mte_sync = MTE_SYNC_ERR, 819 + .mapping = MAP_PRIVATE, 820 + .tag_check = TAG_CHECK_ON, 821 + .atag_check = ATAG_CHECK_ON, 822 + .tag_op = TAG_OP_STONLY, 823 + .enable_tco = false, 824 + }, 825 + { 826 + .check_type = CHECK_FILE_MEM, 827 + .mem_type = USE_MPROTECT, 828 + .mte_sync = MTE_SYNC_ERR, 829 + .mapping = MAP_PRIVATE, 830 + .tag_check = TAG_CHECK_ON, 831 + .atag_check = ATAG_CHECK_ON, 832 + .tag_op = TAG_OP_STONLY, 833 + .enable_tco = false, 834 + }, 835 + { 836 + .check_type = CHECK_FILE_MEM, 837 + .mem_type = USE_MMAP, 838 + .mte_sync = MTE_SYNC_ERR, 839 + .mapping = MAP_SHARED, 840 + .tag_check = TAG_CHECK_ON, 841 + .atag_check = ATAG_CHECK_ON, 842 + .tag_op = TAG_OP_STONLY, 843 + .enable_tco = false, 844 + }, 845 + { 846 + .check_type = CHECK_FILE_MEM, 847 + .mem_type = USE_MPROTECT, 848 + .mte_sync = MTE_SYNC_ERR, 849 + .mapping = MAP_SHARED, 850 + .tag_check = TAG_CHECK_ON, 851 + .atag_check = ATAG_CHECK_ON, 852 + .tag_op = TAG_OP_STONLY, 853 + .enable_tco = false, 854 + }, 855 + { 856 + .check_type = CHECK_FILE_MEM, 857 + .mem_type = USE_MMAP, 858 + .mte_sync = MTE_ASYNC_ERR, 859 + .mapping = MAP_PRIVATE, 860 + .tag_check = TAG_CHECK_ON, 861 + .atag_check = ATAG_CHECK_ON, 862 + .tag_op = TAG_OP_STONLY, 863 + .enable_tco = false, 864 + }, 865 + { 866 + .check_type = CHECK_CLEAR_PROT_MTE, 867 + .mem_type = USE_MMAP, 868 + .mte_sync = MTE_SYNC_ERR, 869 + .mapping = MAP_PRIVATE, 870 + .tag_check = TAG_CHECK_ON, 871 + .atag_check = ATAG_CHECK_ON, 872 + .tag_op = TAG_OP_ALL, 873 + .enable_tco = false, 874 + }, 875 + { 876 + .check_type = CHECK_CLEAR_PROT_MTE, 877 + .mem_type = USE_MPROTECT, 878 + .mte_sync = MTE_SYNC_ERR, 879 + .mapping = MAP_PRIVATE, 880 + .tag_check = TAG_CHECK_ON, 881 + .atag_check = ATAG_CHECK_ON, 882 + .tag_op = TAG_OP_ALL, 883 + .enable_tco = false, 884 + }, 885 + }; 251 886 252 887 err = mte_default_setup(); 253 888 if (err) ··· 957 200 sizes[item - 2] = page_size; 958 201 sizes[item - 1] = page_size + 1; 959 202 960 - /* Register signal handlers */ 961 - mte_register_signal(SIGBUS, mte_default_handler); 962 - mte_register_signal(SIGSEGV, mte_default_handler); 963 - 964 203 /* Set test plan */ 965 - ksft_set_plan(22); 204 + ksft_set_plan(ARRAY_SIZE(test_cases)); 966 205 967 - mte_enable_pstate_tco(); 206 + for (i = 0 ; i < ARRAY_SIZE(test_cases); i++) { 207 + /* Register signal handlers */ 208 + mte_register_signal(SIGBUS, mte_default_handler, 209 + test_cases[i].atag_check == ATAG_CHECK_ON); 210 + mte_register_signal(SIGSEGV, mte_default_handler, 211 + test_cases[i].atag_check == ATAG_CHECK_ON); 968 212 969 - evaluate_test(check_anonymous_memory_mapping(USE_MMAP, MTE_SYNC_ERR, MAP_PRIVATE, TAG_CHECK_OFF), 970 - "Check anonymous memory with private mapping, sync error mode, mmap memory and tag check off\n"); 971 - evaluate_test(check_file_memory_mapping(USE_MPROTECT, MTE_SYNC_ERR, MAP_PRIVATE, TAG_CHECK_OFF), 972 - "Check file memory with private mapping, sync error mode, mmap/mprotect memory and tag check off\n"); 213 + if (test_cases[i].enable_tco) 214 + mte_enable_pstate_tco(); 215 + else 216 + mte_disable_pstate_tco(); 973 217 974 - mte_disable_pstate_tco(); 975 - evaluate_test(check_anonymous_memory_mapping(USE_MMAP, MTE_NONE_ERR, MAP_PRIVATE, TAG_CHECK_OFF), 976 - "Check anonymous memory with private mapping, no error mode, mmap memory and tag check off\n"); 977 - evaluate_test(check_file_memory_mapping(USE_MPROTECT, MTE_NONE_ERR, MAP_PRIVATE, TAG_CHECK_OFF), 978 - "Check file memory with private mapping, no error mode, mmap/mprotect memory and tag check off\n"); 979 - 980 - evaluate_test(check_anonymous_memory_mapping(USE_MMAP, MTE_SYNC_ERR, MAP_PRIVATE, TAG_CHECK_ON), 981 - "Check anonymous memory with private mapping, sync error mode, mmap memory and tag check on\n"); 982 - evaluate_test(check_anonymous_memory_mapping(USE_MPROTECT, MTE_SYNC_ERR, MAP_PRIVATE, TAG_CHECK_ON), 983 - "Check anonymous memory with private mapping, sync error mode, mmap/mprotect memory and tag check on\n"); 984 - evaluate_test(check_anonymous_memory_mapping(USE_MMAP, MTE_SYNC_ERR, MAP_SHARED, TAG_CHECK_ON), 985 - "Check anonymous memory with shared mapping, sync error mode, mmap memory and tag check on\n"); 986 - evaluate_test(check_anonymous_memory_mapping(USE_MPROTECT, MTE_SYNC_ERR, MAP_SHARED, TAG_CHECK_ON), 987 - "Check anonymous memory with shared mapping, sync error mode, mmap/mprotect memory and tag check on\n"); 988 - evaluate_test(check_anonymous_memory_mapping(USE_MMAP, MTE_ASYNC_ERR, MAP_PRIVATE, TAG_CHECK_ON), 989 - "Check anonymous memory with private mapping, async error mode, mmap memory and tag check on\n"); 990 - evaluate_test(check_anonymous_memory_mapping(USE_MPROTECT, MTE_ASYNC_ERR, MAP_PRIVATE, TAG_CHECK_ON), 991 - "Check anonymous memory with private mapping, async error mode, mmap/mprotect memory and tag check on\n"); 992 - evaluate_test(check_anonymous_memory_mapping(USE_MMAP, MTE_ASYNC_ERR, MAP_SHARED, TAG_CHECK_ON), 993 - "Check anonymous memory with shared mapping, async error mode, mmap memory and tag check on\n"); 994 - evaluate_test(check_anonymous_memory_mapping(USE_MPROTECT, MTE_ASYNC_ERR, MAP_SHARED, TAG_CHECK_ON), 995 - "Check anonymous memory with shared mapping, async error mode, mmap/mprotect memory and tag check on\n"); 996 - 997 - evaluate_test(check_file_memory_mapping(USE_MMAP, MTE_SYNC_ERR, MAP_PRIVATE, TAG_CHECK_ON), 998 - "Check file memory with private mapping, sync error mode, mmap memory and tag check on\n"); 999 - evaluate_test(check_file_memory_mapping(USE_MPROTECT, MTE_SYNC_ERR, MAP_PRIVATE, TAG_CHECK_ON), 1000 - "Check file memory with private mapping, sync error mode, mmap/mprotect memory and tag check on\n"); 1001 - evaluate_test(check_file_memory_mapping(USE_MMAP, MTE_SYNC_ERR, MAP_SHARED, TAG_CHECK_ON), 1002 - "Check file memory with shared mapping, sync error mode, mmap memory and tag check on\n"); 1003 - evaluate_test(check_file_memory_mapping(USE_MPROTECT, MTE_SYNC_ERR, MAP_SHARED, TAG_CHECK_ON), 1004 - "Check file memory with shared mapping, sync error mode, mmap/mprotect memory and tag check on\n"); 1005 - evaluate_test(check_file_memory_mapping(USE_MMAP, MTE_ASYNC_ERR, MAP_PRIVATE, TAG_CHECK_ON), 1006 - "Check file memory with private mapping, async error mode, mmap memory and tag check on\n"); 1007 - evaluate_test(check_file_memory_mapping(USE_MPROTECT, MTE_ASYNC_ERR, MAP_PRIVATE, TAG_CHECK_ON), 1008 - "Check file memory with private mapping, async error mode, mmap/mprotect memory and tag check on\n"); 1009 - evaluate_test(check_file_memory_mapping(USE_MMAP, MTE_ASYNC_ERR, MAP_SHARED, TAG_CHECK_ON), 1010 - "Check file memory with shared mapping, async error mode, mmap memory and tag check on\n"); 1011 - evaluate_test(check_file_memory_mapping(USE_MPROTECT, MTE_ASYNC_ERR, MAP_SHARED, TAG_CHECK_ON), 1012 - "Check file memory with shared mapping, async error mode, mmap/mprotect memory and tag check on\n"); 1013 - 1014 - evaluate_test(check_clear_prot_mte_flag(USE_MMAP, MTE_SYNC_ERR, MAP_PRIVATE), 1015 - "Check clear PROT_MTE flags with private mapping, sync error mode and mmap memory\n"); 1016 - evaluate_test(check_clear_prot_mte_flag(USE_MPROTECT, MTE_SYNC_ERR, MAP_PRIVATE), 1017 - "Check clear PROT_MTE flags with private mapping and sync error mode and mmap/mprotect memory\n"); 218 + switch (test_cases[i].check_type) { 219 + case CHECK_ANON_MEM: 220 + evaluate_test(check_anonymous_memory_mapping(test_cases[i].mem_type, 221 + test_cases[i].mte_sync, 222 + test_cases[i].mapping, 223 + test_cases[i].tag_check, 224 + test_cases[i].atag_check, 225 + test_cases[i].tag_op), 226 + format_test_name(&test_cases[i])); 227 + break; 228 + case CHECK_FILE_MEM: 229 + evaluate_test(check_file_memory_mapping(test_cases[i].mem_type, 230 + test_cases[i].mte_sync, 231 + test_cases[i].mapping, 232 + test_cases[i].tag_check, 233 + test_cases[i].atag_check, 234 + test_cases[i].tag_op), 235 + format_test_name(&test_cases[i])); 236 + break; 237 + case CHECK_CLEAR_PROT_MTE: 238 + evaluate_test(check_clear_prot_mte_flag(test_cases[i].mem_type, 239 + test_cases[i].mte_sync, 240 + test_cases[i].mapping, 241 + test_cases[i].atag_check), 242 + format_test_name(&test_cases[i])); 243 + break; 244 + default: 245 + exit(KSFT_FAIL); 246 + } 247 + } 1018 248 1019 249 mte_restore_setup(); 1020 250 ksft_print_cnts();

+21 -8

tools/testing/selftests/arm64/mte/check_prctl.c

··· 12 12 13 13 #include "kselftest.h" 14 14 15 + #ifndef AT_HWCAP3 16 + #define AT_HWCAP3 29 17 + #endif 18 + 15 19 static int set_tagged_addr_ctrl(int val) 16 20 { 17 21 int ret; ··· 64 60 /* 65 61 * Attempt to set a specified combination of modes. 66 62 */ 67 - void set_mode_test(const char *name, int hwcap2, int mask) 63 + void set_mode_test(const char *name, int hwcap2, int hwcap3, int mask) 68 64 { 69 65 int ret; 70 66 71 67 if ((getauxval(AT_HWCAP2) & hwcap2) != hwcap2) { 68 + ksft_test_result_skip("%s\n", name); 69 + return; 70 + } 71 + 72 + if ((getauxval(AT_HWCAP3) & hwcap3) != hwcap3) { 72 73 ksft_test_result_skip("%s\n", name); 73 74 return; 74 75 } ··· 90 81 return; 91 82 } 92 83 93 - if ((ret & PR_MTE_TCF_MASK) == mask) { 84 + if ((ret & (PR_MTE_TCF_MASK | PR_MTE_STORE_ONLY)) == mask) { 94 85 ksft_test_result_pass("%s\n", name); 95 86 } else { 96 87 ksft_print_msg("Got %x, expected %x\n", ··· 102 93 struct mte_mode { 103 94 int mask; 104 95 int hwcap2; 96 + int hwcap3; 105 97 const char *name; 106 98 } mte_modes[] = { 107 - { PR_MTE_TCF_NONE, 0, "NONE" }, 108 - { PR_MTE_TCF_SYNC, HWCAP2_MTE, "SYNC" }, 109 - { PR_MTE_TCF_ASYNC, HWCAP2_MTE, "ASYNC" }, 110 - { PR_MTE_TCF_SYNC | PR_MTE_TCF_ASYNC, HWCAP2_MTE, "SYNC+ASYNC" }, 99 + { PR_MTE_TCF_NONE, 0, 0, "NONE" }, 100 + { PR_MTE_TCF_SYNC, HWCAP2_MTE, 0, "SYNC" }, 101 + { PR_MTE_TCF_ASYNC, HWCAP2_MTE, 0, "ASYNC" }, 102 + { PR_MTE_TCF_SYNC | PR_MTE_TCF_ASYNC, HWCAP2_MTE, 0, "SYNC+ASYNC" }, 103 + { PR_MTE_TCF_SYNC | PR_MTE_STORE_ONLY, HWCAP2_MTE, HWCAP3_MTE_STORE_ONLY, "SYNC+STONLY" }, 104 + { PR_MTE_TCF_ASYNC | PR_MTE_STORE_ONLY, HWCAP2_MTE, HWCAP3_MTE_STORE_ONLY, "ASYNC+STONLY" }, 105 + { PR_MTE_TCF_SYNC | PR_MTE_TCF_ASYNC | PR_MTE_STORE_ONLY, HWCAP2_MTE, HWCAP3_MTE_STORE_ONLY, "SYNC+ASYNC+STONLY" }, 111 106 }; 112 107 113 108 int main(void) ··· 119 106 int i; 120 107 121 108 ksft_print_header(); 122 - ksft_set_plan(5); 109 + ksft_set_plan(ARRAY_SIZE(mte_modes)); 123 110 124 111 check_basic_read(); 125 112 for (i = 0; i < ARRAY_SIZE(mte_modes); i++) 126 - set_mode_test(mte_modes[i].name, mte_modes[i].hwcap2, 113 + set_mode_test(mte_modes[i].name, mte_modes[i].hwcap2, mte_modes[i].hwcap3, 127 114 mte_modes[i].mask); 128 115 129 116 ksft_print_cnts();

+5 -5

tools/testing/selftests/arm64/mte/check_tags_inclusion.c

··· 57 57 return KSFT_FAIL; 58 58 59 59 for (tag = 0; (tag < MT_TAG_COUNT) && (result == KSFT_PASS); tag++) { 60 - ret = mte_switch_mode(mode, MT_INCLUDE_VALID_TAG(tag)); 60 + ret = mte_switch_mode(mode, MT_INCLUDE_VALID_TAG(tag), false); 61 61 if (ret != 0) 62 62 result = KSFT_FAIL; 63 63 /* Try to catch a excluded tag by a number of tries. */ ··· 91 91 92 92 for (tag = 0; (tag < MT_TAG_COUNT - 1) && (result == KSFT_PASS); tag++) { 93 93 excl_mask |= 1 << tag; 94 - mte_switch_mode(mode, MT_INCLUDE_VALID_TAGS(excl_mask)); 94 + mte_switch_mode(mode, MT_INCLUDE_VALID_TAGS(excl_mask), false); 95 95 /* Try to catch a excluded tag by a number of tries. */ 96 96 for (run = 0; (run < RUNS) && (result == KSFT_PASS); run++) { 97 97 ptr = mte_insert_tags(ptr, BUFFER_SIZE); ··· 120 120 mem_type, false) != KSFT_PASS) 121 121 return KSFT_FAIL; 122 122 123 - ret = mte_switch_mode(mode, MT_INCLUDE_TAG_MASK); 123 + ret = mte_switch_mode(mode, MT_INCLUDE_TAG_MASK, false); 124 124 if (ret != 0) 125 125 return KSFT_FAIL; 126 126 /* Try to catch a excluded tag by a number of tries. */ ··· 145 145 if (check_allocated_memory(ptr, BUFFER_SIZE, mem_type, false) != KSFT_PASS) 146 146 return KSFT_FAIL; 147 147 148 - ret = mte_switch_mode(mode, MT_EXCLUDE_TAG_MASK); 148 + ret = mte_switch_mode(mode, MT_EXCLUDE_TAG_MASK, false); 149 149 if (ret != 0) 150 150 return KSFT_FAIL; 151 151 /* Try to catch a excluded tag by a number of tries. */ ··· 180 180 return err; 181 181 182 182 /* Register SIGSEGV handler */ 183 - mte_register_signal(SIGSEGV, mte_default_handler); 183 + mte_register_signal(SIGSEGV, mte_default_handler, false); 184 184 185 185 /* Set test plan */ 186 186 ksft_set_plan(4);

+2 -2

tools/testing/selftests/arm64/mte/check_user_mem.c

··· 44 44 45 45 err = KSFT_PASS; 46 46 len = 2 * page_sz; 47 - mte_switch_mode(mode, MTE_ALLOW_NON_ZERO_TAG); 47 + mte_switch_mode(mode, MTE_ALLOW_NON_ZERO_TAG, false); 48 48 fd = create_temp_file(); 49 49 if (fd == -1) 50 50 return KSFT_FAIL; ··· 211 211 return err; 212 212 213 213 /* Register signal handlers */ 214 - mte_register_signal(SIGSEGV, mte_default_handler); 214 + mte_register_signal(SIGSEGV, mte_default_handler, false); 215 215 216 216 /* Set test plan */ 217 217 ksft_set_plan(64);

+71 -13

tools/testing/selftests/arm64/mte/mte_common_util.c

··· 6 6 #include <signal.h> 7 7 #include <stdio.h> 8 8 #include <stdlib.h> 9 + #include <time.h> 9 10 #include <unistd.h> 10 11 11 12 #include <linux/auxvec.h> ··· 20 19 #include "mte_common_util.h" 21 20 #include "mte_def.h" 22 21 22 + #ifndef SA_EXPOSE_TAGBITS 23 + #define SA_EXPOSE_TAGBITS 0x00000800 24 + #endif 25 + 23 26 #define INIT_BUFFER_SIZE 256 24 27 25 28 struct mte_fault_cxt cur_mte_cxt; 29 + bool mtefar_support; 30 + bool mtestonly_support; 26 31 static unsigned int mte_cur_mode; 27 32 static unsigned int mte_cur_pstate_tco; 33 + static bool mte_cur_stonly; 28 34 29 35 void mte_default_handler(int signum, siginfo_t *si, void *uc) 30 36 { 37 + struct sigaction sa; 31 38 unsigned long addr = (unsigned long)si->si_addr; 39 + unsigned char si_tag, si_atag; 40 + 41 + sigaction(signum, NULL, &sa); 42 + 43 + if (sa.sa_flags & SA_EXPOSE_TAGBITS) { 44 + si_tag = MT_FETCH_TAG(addr); 45 + si_atag = MT_FETCH_ATAG(addr); 46 + addr = MT_CLEAR_TAGS(addr); 47 + } else { 48 + si_tag = 0; 49 + si_atag = 0; 50 + } 32 51 33 52 if (signum == SIGSEGV) { 34 53 #ifdef DEBUG 35 - ksft_print_msg("INFO: SIGSEGV signal at pc=%lx, fault addr=%lx, si_code=%lx\n", 36 - ((ucontext_t *)uc)->uc_mcontext.pc, addr, si->si_code); 54 + ksft_print_msg("INFO: SIGSEGV signal at pc=%lx, fault addr=%lx, si_code=%lx, si_tag=%x, si_atag=%x\n", 55 + ((ucontext_t *)uc)->uc_mcontext.pc, addr, si->si_code, si_tag, si_atag); 37 56 #endif 38 57 if (si->si_code == SEGV_MTEAERR) { 39 58 if (cur_mte_cxt.trig_si_code == si->si_code) ··· 66 45 } 67 46 /* Compare the context for precise error */ 68 47 else if (si->si_code == SEGV_MTESERR) { 48 + if ((!mtefar_support && si_atag) || (si_atag != MT_FETCH_ATAG(cur_mte_cxt.trig_addr))) { 49 + ksft_print_msg("Invalid MTE synchronous exception caught for address tag! si_tag=%x, si_atag: %x\n", si_tag, si_atag); 50 + exit(KSFT_FAIL); 51 + } 52 + 69 53 if (cur_mte_cxt.trig_si_code == si->si_code && 70 54 ((cur_mte_cxt.trig_range >= 0 && 71 - addr >= MT_CLEAR_TAG(cur_mte_cxt.trig_addr) && 72 - addr <= (MT_CLEAR_TAG(cur_mte_cxt.trig_addr) + cur_mte_cxt.trig_range)) || 55 + addr >= MT_CLEAR_TAGS(cur_mte_cxt.trig_addr) && 56 + addr <= (MT_CLEAR_TAGS(cur_mte_cxt.trig_addr) + cur_mte_cxt.trig_range)) || 73 57 (cur_mte_cxt.trig_range < 0 && 74 - addr <= MT_CLEAR_TAG(cur_mte_cxt.trig_addr) && 75 - addr >= (MT_CLEAR_TAG(cur_mte_cxt.trig_addr) + cur_mte_cxt.trig_range)))) { 58 + addr <= MT_CLEAR_TAGS(cur_mte_cxt.trig_addr) && 59 + addr >= (MT_CLEAR_TAGS(cur_mte_cxt.trig_addr) + cur_mte_cxt.trig_range)))) { 76 60 cur_mte_cxt.fault_valid = true; 77 61 /* Adjust the pc by 4 */ 78 62 ((ucontext_t *)uc)->uc_mcontext.pc += 4; ··· 93 67 ksft_print_msg("INFO: SIGBUS signal at pc=%llx, fault addr=%lx, si_code=%x\n", 94 68 ((ucontext_t *)uc)->uc_mcontext.pc, addr, si->si_code); 95 69 if ((cur_mte_cxt.trig_range >= 0 && 96 - addr >= MT_CLEAR_TAG(cur_mte_cxt.trig_addr) && 97 - addr <= (MT_CLEAR_TAG(cur_mte_cxt.trig_addr) + cur_mte_cxt.trig_range)) || 70 + addr >= MT_CLEAR_TAGS(cur_mte_cxt.trig_addr) && 71 + addr <= (MT_CLEAR_TAGS(cur_mte_cxt.trig_addr) + cur_mte_cxt.trig_range)) || 98 72 (cur_mte_cxt.trig_range < 0 && 99 - addr <= MT_CLEAR_TAG(cur_mte_cxt.trig_addr) && 100 - addr >= (MT_CLEAR_TAG(cur_mte_cxt.trig_addr) + cur_mte_cxt.trig_range))) { 73 + addr <= MT_CLEAR_TAGS(cur_mte_cxt.trig_addr) && 74 + addr >= (MT_CLEAR_TAGS(cur_mte_cxt.trig_addr) + cur_mte_cxt.trig_range))) { 101 75 cur_mte_cxt.fault_valid = true; 102 76 /* Adjust the pc by 4 */ 103 77 ((ucontext_t *)uc)->uc_mcontext.pc += 4; ··· 105 79 } 106 80 } 107 81 108 - void mte_register_signal(int signal, void (*handler)(int, siginfo_t *, void *)) 82 + void mte_register_signal(int signal, void (*handler)(int, siginfo_t *, void *), 83 + bool export_tags) 109 84 { 110 85 struct sigaction sa; 111 86 112 87 sa.sa_sigaction = handler; 113 88 sa.sa_flags = SA_SIGINFO; 89 + 90 + if (export_tags && signal == SIGSEGV) 91 + sa.sa_flags |= SA_EXPOSE_TAGBITS; 92 + 114 93 sigemptyset(&sa.sa_mask); 115 94 sigaction(signal, &sa, NULL); 116 95 } ··· 149 118 size = MT_ALIGN_UP(size); 150 119 ptr = (void *)MT_CLEAR_TAG((unsigned long)ptr); 151 120 mte_clear_tag_address_range(ptr, size); 121 + } 122 + 123 + void *mte_insert_atag(void *ptr) 124 + { 125 + unsigned char atag; 126 + 127 + atag = mtefar_support ? (random() % MT_ATAG_MASK) + 1 : 0; 128 + return (void *)MT_SET_ATAG((unsigned long)ptr, atag); 129 + } 130 + 131 + void *mte_clear_atag(void *ptr) 132 + { 133 + return (void *)MT_CLEAR_ATAG((unsigned long)ptr); 152 134 } 153 135 154 136 static void *__mte_allocate_memory_range(size_t size, int mem_type, int mapping, ··· 316 272 cur_mte_cxt.trig_si_code = 0; 317 273 } 318 274 319 - int mte_switch_mode(int mte_option, unsigned long incl_mask) 275 + int mte_switch_mode(int mte_option, unsigned long incl_mask, bool stonly) 320 276 { 321 277 unsigned long en = 0; 322 278 ··· 348 304 break; 349 305 } 350 306 307 + if (mtestonly_support && stonly) 308 + en |= PR_MTE_STORE_ONLY; 309 + 351 310 en |= (incl_mask << PR_MTE_TAG_SHIFT); 352 311 /* Enable address tagging ABI, mte error reporting mode and tag inclusion mask. */ 353 312 if (prctl(PR_SET_TAGGED_ADDR_CTRL, en, 0, 0, 0) != 0) { ··· 363 316 int mte_default_setup(void) 364 317 { 365 318 unsigned long hwcaps2 = getauxval(AT_HWCAP2); 319 + unsigned long hwcaps3 = getauxval(AT_HWCAP3); 366 320 unsigned long en = 0; 367 321 int ret; 368 322 323 + /* To generate random address tag */ 324 + srandom(time(NULL)); 325 + 369 326 if (!(hwcaps2 & HWCAP2_MTE)) 370 327 ksft_exit_skip("MTE features unavailable\n"); 328 + 329 + mtefar_support = !!(hwcaps3 & HWCAP3_MTE_FAR); 330 + 331 + if (hwcaps3 & HWCAP3_MTE_STORE_ONLY) 332 + mtestonly_support = true; 371 333 372 334 /* Get current mte mode */ 373 335 ret = prctl(PR_GET_TAGGED_ADDR_CTRL, en, 0, 0, 0); ··· 391 335 else if (ret & PR_MTE_TCF_NONE) 392 336 mte_cur_mode = MTE_NONE_ERR; 393 337 338 + mte_cur_stonly = (ret & PR_MTE_STORE_ONLY) ? true : false; 339 + 394 340 mte_cur_pstate_tco = mte_get_pstate_tco(); 395 341 /* Disable PSTATE.TCO */ 396 342 mte_disable_pstate_tco(); ··· 401 343 402 344 void mte_restore_setup(void) 403 345 { 404 - mte_switch_mode(mte_cur_mode, MTE_ALLOW_NON_ZERO_TAG); 346 + mte_switch_mode(mte_cur_mode, MTE_ALLOW_NON_ZERO_TAG, mte_cur_stonly); 405 347 if (mte_cur_pstate_tco == MT_PSTATE_TCO_EN) 406 348 mte_enable_pstate_tco(); 407 349 else if (mte_cur_pstate_tco == MT_PSTATE_TCO_DIS)

+7 -2

tools/testing/selftests/arm64/mte/mte_common_util.h

··· 37 37 }; 38 38 39 39 extern struct mte_fault_cxt cur_mte_cxt; 40 + extern bool mtefar_support; 41 + extern bool mtestonly_support; 40 42 41 43 /* MTE utility functions */ 42 44 void mte_default_handler(int signum, siginfo_t *si, void *uc); 43 - void mte_register_signal(int signal, void (*handler)(int, siginfo_t *, void *)); 45 + void mte_register_signal(int signal, void (*handler)(int, siginfo_t *, void *), 46 + bool export_tags); 44 47 void mte_wait_after_trig(void); 45 48 void *mte_allocate_memory(size_t size, int mem_type, int mapping, bool tags); 46 49 void *mte_allocate_memory_tag_range(size_t size, int mem_type, int mapping, ··· 57 54 size_t range_before, size_t range_after); 58 55 void *mte_insert_tags(void *ptr, size_t size); 59 56 void mte_clear_tags(void *ptr, size_t size); 57 + void *mte_insert_atag(void *ptr); 58 + void *mte_clear_atag(void *ptr); 60 59 int mte_default_setup(void); 61 60 void mte_restore_setup(void); 62 - int mte_switch_mode(int mte_option, unsigned long incl_mask); 61 + int mte_switch_mode(int mte_option, unsigned long incl_mask, bool stonly); 63 62 void mte_initialize_current_context(int mode, uintptr_t ptr, ssize_t range); 64 63 65 64 /* Common utility functions */

+8

tools/testing/selftests/arm64/mte/mte_def.h

··· 42 42 #define MT_TAG_COUNT 16 43 43 #define MT_INCLUDE_TAG_MASK 0xFFFF 44 44 #define MT_EXCLUDE_TAG_MASK 0x0 45 + #define MT_ATAG_SHIFT 60 46 + #define MT_ATAG_MASK 0xFUL 45 47 46 48 #define MT_ALIGN_GRANULE (MT_GRANULE_SIZE - 1) 47 49 #define MT_CLEAR_TAG(x) ((x) & ~(MT_TAG_MASK << MT_TAG_SHIFT)) 48 50 #define MT_SET_TAG(x, y) ((x) | (y << MT_TAG_SHIFT)) 49 51 #define MT_FETCH_TAG(x) ((x >> MT_TAG_SHIFT) & (MT_TAG_MASK)) 50 52 #define MT_ALIGN_UP(x) ((x + MT_ALIGN_GRANULE) & ~(MT_ALIGN_GRANULE)) 53 + 54 + #define MT_CLEAR_ATAG(x) ((x) & ~(MT_TAG_MASK << MT_ATAG_SHIFT)) 55 + #define MT_SET_ATAG(x, y) ((x) | (((y) & MT_ATAG_MASK) << MT_ATAG_SHIFT)) 56 + #define MT_FETCH_ATAG(x) ((x >> MT_ATAG_SHIFT) & (MT_ATAG_MASK)) 57 + 58 + #define MT_CLEAR_TAGS(x) (MT_CLEAR_ATAG(MT_CLEAR_TAG(x))) 51 59 52 60 #define MT_PSTATE_TCO_SHIFT 25 53 61 #define MT_PSTATE_TCO_MASK ~(0x1 << MT_PSTATE_TCO_SHIFT)

+2 -2

tools/testing/selftests/kvm/arm64/debug-exceptions.c

··· 140 140 141 141 static void enable_monitor_debug_exceptions(void) 142 142 { 143 - uint32_t mdscr; 143 + uint64_t mdscr; 144 144 145 145 asm volatile("msr daifclr, #8"); 146 146 ··· 223 223 224 224 static void install_ss(void) 225 225 { 226 - uint32_t mdscr; 226 + uint64_t mdscr; 227 227 228 228 asm volatile("msr daifclr, #8"); 229 229

Configure Feed

Configure Feed