Merge tag 'powerpc-5.13-1' of git://git.kernel.org/pub/scm/linux/kernel/git/powerpc/linux

+1 -1

Documentation/admin-guide/sysctl/net.rst

··· 64 64 - arm64 65 65 - arm32 66 66 - ppc64 67 + - ppc32 67 68 - sparc64 68 69 - mips64 69 70 - s390x ··· 74 73 And the older cBPF JIT supported on the following archs: 75 74 76 75 - mips 77 - - ppc 78 76 - sparc 79 77 80 78 eBPF JITs are a superset of cBPF JITs, meaning the kernel will

+1 -1

Documentation/features/debug/debug-vm-pgtable/arch-support.txt

··· 21 21 | nios2: | TODO | 22 22 | openrisc: | TODO | 23 23 | parisc: | TODO | 24 - | powerpc: | TODO | 24 + | powerpc: | ok | 25 25 | riscv: | ok | 26 26 | s390: | ok | 27 27 | sh: | TODO |

+14

Documentation/powerpc/papr_hcalls.rst

··· 275 275 Given a DRC Index collect the performance statistics for NVDIMM and copy them 276 276 to the resultBuffer. 277 277 278 + **H_SCM_FLUSH** 279 + 280 + | Input: *drcIndex, continue-token* 281 + | Out: *continue-token* 282 + | Return Value: *H_SUCCESS, H_Parameter, H_P2, H_BUSY* 283 + 284 + Given a DRC Index Flush the data to backend NVDIMM device. 285 + 286 + The hcall returns H_BUSY when the flush takes longer time and the hcall needs 287 + to be issued multiple times in order to be completely serviced. The 288 + *continue-token* from the output to be passed in the argument list of 289 + subsequent hcalls to the hypervisor until the hcall is completely serviced 290 + at which point H_SUCCESS or other error is returned by the hypervisor. 291 + 278 292 References 279 293 ========== 280 294 .. [1] "Power Architecture Platform Reference"

+2 -2

Documentation/powerpc/vas-api.rst

··· 254 254 signals. 255 255 256 256 NX-GZIP User's Manual: 257 - https://github.com/libnxz/power-gzip/blob/master/power_nx_gzip_um.pdf 257 + https://github.com/libnxz/power-gzip/blob/master/doc/power_nx_gzip_um.pdf 258 258 259 259 Simple example 260 260 ============== ··· 301 301 close(fd) or window can be closed upon process exit 302 302 } 303 303 304 - Refer https://github.com/abalib/power-gzip for tests or more 304 + Refer https://github.com/libnxz/power-gzip for tests or more 305 305 use cases.

+2 -1

arch/arm64/include/asm/vdso/compat_gettimeofday.h

··· 155 155 } 156 156 157 157 #ifdef CONFIG_TIME_NS 158 - static __always_inline const struct vdso_data *__arch_get_timens_vdso_data(void) 158 + static __always_inline 159 + const struct vdso_data *__arch_get_timens_vdso_data(const struct vdso_data *vd) 159 160 { 160 161 const struct vdso_data *ret; 161 162

+1 -1

arch/arm64/include/asm/vdso/gettimeofday.h

··· 96 96 97 97 #ifdef CONFIG_TIME_NS 98 98 static __always_inline 99 - const struct vdso_data *__arch_get_timens_vdso_data(void) 99 + const struct vdso_data *__arch_get_timens_vdso_data(const struct vdso_data *vd) 100 100 { 101 101 return _timens_data; 102 102 }

+18 -14

arch/powerpc/Kconfig

··· 119 119 # 120 120 select ARCH_32BIT_OFF_T if PPC32 121 121 select ARCH_HAS_DEBUG_VIRTUAL 122 + select ARCH_HAS_DEBUG_VM_PGTABLE 122 123 select ARCH_HAS_DEVMEM_IS_ALLOWED 123 124 select ARCH_HAS_ELF_RANDOMIZE 124 125 select ARCH_HAS_FORTIFY_SOURCE ··· 136 135 select ARCH_HAS_MEMBARRIER_CALLBACKS 137 136 select ARCH_HAS_MEMBARRIER_SYNC_CORE 138 137 select ARCH_HAS_SCALED_CPUTIME if VIRT_CPU_ACCOUNTING_NATIVE && PPC_BOOK3S_64 139 - select ARCH_HAS_STRICT_KERNEL_RWX if (PPC32 && !HIBERNATION) 138 + select ARCH_HAS_STRICT_KERNEL_RWX if ((PPC_BOOK3S_64 || PPC32) && !HIBERNATION) 140 139 select ARCH_HAS_TICK_BROADCAST if GENERIC_CLOCKEVENTS_BROADCAST 141 140 select ARCH_HAS_UACCESS_FLUSHCACHE 142 141 select ARCH_HAS_COPY_MC if PPC64 ··· 146 145 select ARCH_MIGHT_HAVE_PC_PARPORT 147 146 select ARCH_MIGHT_HAVE_PC_SERIO 148 147 select ARCH_OPTIONAL_KERNEL_RWX if ARCH_HAS_STRICT_KERNEL_RWX 148 + select ARCH_STACKWALK 149 149 select ARCH_SUPPORTS_ATOMIC_RMW 150 150 select ARCH_SUPPORTS_DEBUG_PAGEALLOC if PPC32 || PPC_BOOK3S_64 151 151 select ARCH_USE_BUILTIN_BSWAP ··· 173 171 select GENERIC_CPU_AUTOPROBE 174 172 select GENERIC_CPU_VULNERABILITIES if PPC_BARRIER_NOSPEC 175 173 select GENERIC_EARLY_IOREMAP 174 + select GENERIC_GETTIMEOFDAY 176 175 select GENERIC_IRQ_SHOW 177 176 select GENERIC_IRQ_SHOW_LEVEL 178 177 select GENERIC_PCI_IOMAP if PCI ··· 181 178 select GENERIC_STRNCPY_FROM_USER 182 179 select GENERIC_STRNLEN_USER 183 180 select GENERIC_TIME_VSYSCALL 184 - select GENERIC_GETTIMEOFDAY 181 + select GENERIC_VDSO_TIME_NS 185 182 select HAVE_ARCH_AUDITSYSCALL 186 183 select HAVE_ARCH_HUGE_VMAP if PPC_BOOK3S_64 && PPC_RADIX_MMU 187 184 select HAVE_ARCH_JUMP_LABEL 185 + select HAVE_ARCH_JUMP_LABEL_RELATIVE 188 186 select HAVE_ARCH_KASAN if PPC32 && PPC_PAGE_SHIFT <= 14 189 187 select HAVE_ARCH_KASAN_VMALLOC if PPC32 && PPC_PAGE_SHIFT <= 14 190 188 select HAVE_ARCH_KGDB 189 + select HAVE_ARCH_KFENCE if PPC32 191 190 select HAVE_ARCH_MMAP_RND_BITS 192 191 select HAVE_ARCH_MMAP_RND_COMPAT_BITS if COMPAT 193 192 select HAVE_ARCH_NVRAM_OPS ··· 197 192 select HAVE_ARCH_TRACEHOOK 198 193 select HAVE_ASM_MODVERSIONS 199 194 select HAVE_C_RECORDMCOUNT 200 - select HAVE_CBPF_JIT if !PPC64 201 195 select HAVE_STACKPROTECTOR if PPC64 && $(cc-option,-mstack-protector-guard=tls -mstack-protector-guard-reg=r13) 202 196 select HAVE_STACKPROTECTOR if PPC32 && $(cc-option,-mstack-protector-guard=tls -mstack-protector-guard-reg=r2) 203 197 select HAVE_CONTEXT_TRACKING if PPC64 ··· 204 200 select HAVE_DEBUG_STACKOVERFLOW 205 201 select HAVE_DYNAMIC_FTRACE 206 202 select HAVE_DYNAMIC_FTRACE_WITH_REGS if MPROFILE_KERNEL 207 - select HAVE_EBPF_JIT if PPC64 203 + select HAVE_EBPF_JIT 208 204 select HAVE_EFFICIENT_UNALIGNED_ACCESS if !(CPU_LITTLE_ENDIAN && POWER7_CPU) 209 205 select HAVE_FAST_GUP 210 206 select HAVE_FTRACE_MCOUNT_RECORD ··· 228 224 select HAVE_LIVEPATCH if HAVE_DYNAMIC_FTRACE_WITH_REGS 229 225 select HAVE_MOD_ARCH_SPECIFIC 230 226 select HAVE_NMI if PERF_EVENTS || (PPC64 && PPC_BOOK3S) 231 - select HAVE_HARDLOCKUP_DETECTOR_ARCH if (PPC64 && PPC_BOOK3S) 232 - select HAVE_OPTPROBES if PPC64 227 + select HAVE_HARDLOCKUP_DETECTOR_ARCH if PPC64 && PPC_BOOK3S && SMP 228 + select HAVE_OPTPROBES 233 229 select HAVE_PERF_EVENTS 234 230 select HAVE_PERF_EVENTS_NMI if PPC64 235 231 select HAVE_HARDLOCKUP_DETECTOR_PERF if PERF_EVENTS && HAVE_PERF_EVENTS_NMI && !HAVE_HARDLOCKUP_DETECTOR_ARCH ··· 238 234 select MMU_GATHER_RCU_TABLE_FREE 239 235 select MMU_GATHER_PAGE_SIZE 240 236 select HAVE_REGS_AND_STACK_ACCESS_API 241 - select HAVE_RELIABLE_STACKTRACE if PPC_BOOK3S_64 && CPU_LITTLE_ENDIAN 237 + select HAVE_RELIABLE_STACKTRACE 242 238 select HAVE_SOFTIRQ_ON_OWN_STACK 243 239 select HAVE_SYSCALL_TRACEPOINTS 244 240 select HAVE_VIRT_CPU_ACCOUNTING ··· 790 786 config DATA_SHIFT_BOOL 791 787 bool "Set custom data alignment" 792 788 depends on ADVANCED_OPTIONS 793 - depends on STRICT_KERNEL_RWX || DEBUG_PAGEALLOC 789 + depends on STRICT_KERNEL_RWX || DEBUG_PAGEALLOC || KFENCE 794 790 depends on PPC_BOOK3S_32 || (PPC_8xx && !PIN_TLB_DATA && !STRICT_KERNEL_RWX) 795 791 help 796 792 This option allows you to set the kernel data alignment. When ··· 802 798 config DATA_SHIFT 803 799 int "Data shift" if DATA_SHIFT_BOOL 804 800 default 24 if STRICT_KERNEL_RWX && PPC64 805 - range 17 28 if (STRICT_KERNEL_RWX || DEBUG_PAGEALLOC) && PPC_BOOK3S_32 806 - range 19 23 if (STRICT_KERNEL_RWX || DEBUG_PAGEALLOC) && PPC_8xx 801 + range 17 28 if (STRICT_KERNEL_RWX || DEBUG_PAGEALLOC || KFENCE) && PPC_BOOK3S_32 802 + range 19 23 if (STRICT_KERNEL_RWX || DEBUG_PAGEALLOC || KFENCE) && PPC_8xx 807 803 default 22 if STRICT_KERNEL_RWX && PPC_BOOK3S_32 808 - default 18 if DEBUG_PAGEALLOC && PPC_BOOK3S_32 804 + default 18 if (DEBUG_PAGEALLOC || KFENCE) && PPC_BOOK3S_32 809 805 default 23 if STRICT_KERNEL_RWX && PPC_8xx 810 - default 23 if DEBUG_PAGEALLOC && PPC_8xx && PIN_TLB_DATA 811 - default 19 if DEBUG_PAGEALLOC && PPC_8xx 806 + default 23 if (DEBUG_PAGEALLOC || KFENCE) && PPC_8xx && PIN_TLB_DATA 807 + default 19 if (DEBUG_PAGEALLOC || KFENCE) && PPC_8xx 812 808 default PPC_PAGE_SHIFT 813 809 help 814 810 On Book3S 32 (603+), DBATs are used to map kernel text and rodata RO. ··· 1221 1217 config TASK_SIZE 1222 1218 hex "Size of user task space" if TASK_SIZE_BOOL 1223 1219 default "0x80000000" if PPC_8xx 1224 - default "0xb0000000" if PPC_BOOK3S_32 && STRICT_KERNEL_RWX 1220 + default "0xb0000000" if PPC_BOOK3S_32 1225 1221 default "0xc0000000" 1226 1222 endmenu 1227 1223

+1

arch/powerpc/Kconfig.debug

··· 353 353 config FAIL_IOMMU 354 354 bool "Fault-injection capability for IOMMU" 355 355 depends on FAULT_INJECTION 356 + depends on PCI || IBMVIO 356 357 help 357 358 Provide fault-injection capability for IOMMU. Each device can 358 359 be selectively enabled via the fail_iommu property.

+4 -7

arch/powerpc/Makefile

··· 181 181 ifdef CONFIG_MPROFILE_KERNEL 182 182 CC_FLAGS_FTRACE += -mprofile-kernel 183 183 endif 184 - # Work around gcc code-gen bugs with -pg / -fno-omit-frame-pointer in gcc <= 4.8 185 - # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=44199 186 - # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=52828 187 - ifndef CONFIG_CC_IS_CLANG 188 - CC_FLAGS_FTRACE += $(call cc-ifversion, -lt, 0409, -mno-sched-epilog) 189 - endif 190 184 endif 191 185 192 186 CFLAGS-$(CONFIG_TARGET_CPU_BOOL) += $(call cc-option,-mcpu=$(CONFIG_TARGET_CPU)) ··· 438 444 endif 439 445 440 446 ifdef CONFIG_SMP 447 + ifdef CONFIG_PPC32 441 448 prepare: task_cpu_prepare 442 449 443 450 PHONY += task_cpu_prepare 444 451 task_cpu_prepare: prepare0 445 452 $(eval KBUILD_CFLAGS += -D_TASK_CPU=$(shell awk '{if ($$2 == "TASK_CPU") print $$3;}' include/generated/asm-offsets.h)) 446 - endif 453 + 454 + endif # CONFIG_PPC32 455 + endif # CONFIG_SMP 447 456 448 457 PHONY += checkbin 449 458 # Check toolchain versions:

+2

arch/powerpc/configs/ppc64_defconfig

··· 50 50 CONFIG_KEXEC=y 51 51 CONFIG_KEXEC_FILE=y 52 52 CONFIG_CRASH_DUMP=y 53 + CONFIG_FA_DUMP=y 53 54 CONFIG_IRQ_ALL_CPUS=y 54 55 CONFIG_PPC_64K_PAGES=y 55 56 CONFIG_SCHED_SMT=y ··· 178 177 CONFIG_BE2NET=m 179 178 CONFIG_IBMVETH=m 180 179 CONFIG_EHEA=m 180 + CONFIG_IBMVNIC=m 181 181 CONFIG_E100=y 182 182 CONFIG_E1000=y 183 183 CONFIG_E1000E=y

+2

arch/powerpc/configs/pseries_defconfig

··· 41 41 CONFIG_SCANLOG=m 42 42 CONFIG_PPC_SMLPAR=y 43 43 CONFIG_IBMEBUS=y 44 + CONFIG_PAPR_SCM=m 44 45 CONFIG_PPC_SVM=y 45 46 # CONFIG_PPC_PMAC is not set 46 47 CONFIG_RTAS_FLASH=m ··· 160 159 CONFIG_S2IO=m 161 160 CONFIG_IBMVETH=y 162 161 CONFIG_EHEA=y 162 + CONFIG_IBMVNIC=y 163 163 CONFIG_E100=y 164 164 CONFIG_E1000=y 165 165 CONFIG_E1000E=y

-1

arch/powerpc/include/asm/Kbuild

··· 1 1 # SPDX-License-Identifier: GPL-2.0 2 2 generated-y += syscall_table_32.h 3 3 generated-y += syscall_table_64.h 4 - generated-y += syscall_table_c32.h 5 4 generated-y += syscall_table_spu.h 6 5 generic-y += export.h 7 6 generic-y += kvm_types.h

-2

arch/powerpc/include/asm/asm-prototypes.h

··· 77 77 long ppc_fadvise64_64(int fd, int advice, u32 offset_high, u32 offset_low, 78 78 u32 len_high, u32 len_low); 79 79 long sys_switch_endian(void); 80 - notrace unsigned int __check_irq_replay(void); 81 - void notrace restore_interrupts(void); 82 80 83 81 /* prom_init (OpenFirmware) */ 84 82 unsigned long __init prom_init(unsigned long r3, unsigned long r4,

-16

arch/powerpc/include/asm/barrier.h

··· 80 80 ___p1; \ 81 81 }) 82 82 83 - #ifdef CONFIG_PPC64 84 - #define smp_cond_load_relaxed(ptr, cond_expr) ({ \ 85 - typeof(ptr) __PTR = (ptr); \ 86 - __unqual_scalar_typeof(*ptr) VAL; \ 87 - VAL = READ_ONCE(*__PTR); \ 88 - if (unlikely(!(cond_expr))) { \ 89 - spin_begin(); \ 90 - do { \ 91 - VAL = READ_ONCE(*__PTR); \ 92 - } while (!(cond_expr)); \ 93 - spin_end(); \ 94 - } \ 95 - (typeof(*ptr))VAL; \ 96 - }) 97 - #endif 98 - 99 83 #ifdef CONFIG_PPC_BOOK3S_64 100 84 #define NOSPEC_BARRIER_SLOT nop 101 85 #elif defined(CONFIG_PPC_FSL_BOOK3E)

+46 -80

arch/powerpc/include/asm/book3s/32/kup.h

··· 5 5 #include <asm/bug.h> 6 6 #include <asm/book3s/32/mmu-hash.h> 7 7 8 - #ifdef __ASSEMBLY__ 9 - 10 - .macro kuep_update_sr gpr1, gpr2 /* NEVER use r0 as gpr2 due to addis */ 11 - 101: mtsrin \gpr1, \gpr2 12 - addi \gpr1, \gpr1, 0x111 /* next VSID */ 13 - rlwinm \gpr1, \gpr1, 0, 0xf0ffffff /* clear VSID overflow */ 14 - addis \gpr2, \gpr2, 0x1000 /* address of next segment */ 15 - bdnz 101b 16 - isync 17 - .endm 18 - 19 - .macro kuep_lock gpr1, gpr2 20 - #ifdef CONFIG_PPC_KUEP 21 - li \gpr1, NUM_USER_SEGMENTS 22 - li \gpr2, 0 23 - mtctr \gpr1 24 - mfsrin \gpr1, \gpr2 25 - oris \gpr1, \gpr1, SR_NX@h /* set Nx */ 26 - kuep_update_sr \gpr1, \gpr2 27 - #endif 28 - .endm 29 - 30 - .macro kuep_unlock gpr1, gpr2 31 - #ifdef CONFIG_PPC_KUEP 32 - li \gpr1, NUM_USER_SEGMENTS 33 - li \gpr2, 0 34 - mtctr \gpr1 35 - mfsrin \gpr1, \gpr2 36 - rlwinm \gpr1, \gpr1, 0, ~SR_NX /* Clear Nx */ 37 - kuep_update_sr \gpr1, \gpr2 38 - #endif 39 - .endm 40 - 41 - #ifdef CONFIG_PPC_KUAP 42 - 43 - .macro kuap_update_sr gpr1, gpr2, gpr3 /* NEVER use r0 as gpr2 due to addis */ 44 - 101: mtsrin \gpr1, \gpr2 45 - addi \gpr1, \gpr1, 0x111 /* next VSID */ 46 - rlwinm \gpr1, \gpr1, 0, 0xf0ffffff /* clear VSID overflow */ 47 - addis \gpr2, \gpr2, 0x1000 /* address of next segment */ 48 - cmplw \gpr2, \gpr3 49 - blt- 101b 50 - isync 51 - .endm 52 - 53 - .macro kuap_save_and_lock sp, thread, gpr1, gpr2, gpr3 54 - lwz \gpr2, KUAP(\thread) 55 - rlwinm. \gpr3, \gpr2, 28, 0xf0000000 56 - stw \gpr2, STACK_REGS_KUAP(\sp) 57 - beq+ 102f 58 - li \gpr1, 0 59 - stw \gpr1, KUAP(\thread) 60 - mfsrin \gpr1, \gpr2 61 - oris \gpr1, \gpr1, SR_KS@h /* set Ks */ 62 - kuap_update_sr \gpr1, \gpr2, \gpr3 63 - 102: 64 - .endm 65 - 66 - .macro kuap_restore sp, current, gpr1, gpr2, gpr3 67 - lwz \gpr2, STACK_REGS_KUAP(\sp) 68 - rlwinm. \gpr3, \gpr2, 28, 0xf0000000 69 - stw \gpr2, THREAD + KUAP(\current) 70 - beq+ 102f 71 - mfsrin \gpr1, \gpr2 72 - rlwinm \gpr1, \gpr1, 0, ~SR_KS /* Clear Ks */ 73 - kuap_update_sr \gpr1, \gpr2, \gpr3 74 - 102: 75 - .endm 76 - 77 - .macro kuap_check current, gpr 78 - #ifdef CONFIG_PPC_KUAP_DEBUG 79 - lwz \gpr, THREAD + KUAP(\current) 80 - 999: twnei \gpr, 0 81 - EMIT_BUG_ENTRY 999b, __FILE__, __LINE__, (BUGFLAG_WARNING | BUGFLAG_ONCE) 82 - #endif 83 - .endm 84 - 85 - #endif /* CONFIG_PPC_KUAP */ 86 - 87 - #else /* !__ASSEMBLY__ */ 8 + #ifndef __ASSEMBLY__ 88 9 89 10 #ifdef CONFIG_PPC_KUAP 90 11 ··· 22 101 addr += 0x10000000; /* address of next segment */ 23 102 } 24 103 isync(); /* Context sync required after mtsr() */ 104 + } 105 + 106 + static inline void kuap_save_and_lock(struct pt_regs *regs) 107 + { 108 + unsigned long kuap = current->thread.kuap; 109 + u32 addr = kuap & 0xf0000000; 110 + u32 end = kuap << 28; 111 + 112 + regs->kuap = kuap; 113 + if (unlikely(!kuap)) 114 + return; 115 + 116 + current->thread.kuap = 0; 117 + kuap_update_sr(mfsr(addr) | SR_KS, addr, end); /* Set Ks */ 118 + } 119 + 120 + static inline void kuap_user_restore(struct pt_regs *regs) 121 + { 122 + } 123 + 124 + static inline void kuap_kernel_restore(struct pt_regs *regs, unsigned long kuap) 125 + { 126 + u32 addr = regs->kuap & 0xf0000000; 127 + u32 end = regs->kuap << 28; 128 + 129 + current->thread.kuap = regs->kuap; 130 + 131 + if (unlikely(regs->kuap == kuap)) 132 + return; 133 + 134 + kuap_update_sr(mfsr(addr) & ~SR_KS, addr, end); /* Clear Ks */ 135 + } 136 + 137 + static inline unsigned long kuap_get_and_assert_locked(void) 138 + { 139 + unsigned long kuap = current->thread.kuap; 140 + 141 + WARN_ON_ONCE(IS_ENABLED(CONFIG_PPC_KUAP_DEBUG) && kuap != 0); 142 + 143 + return kuap; 144 + } 145 + 146 + static inline void kuap_assert_locked(void) 147 + { 148 + kuap_get_and_assert_locked(); 25 149 } 26 150 27 151 static __always_inline void allow_user_access(void __user *to, const void __user *from,

-2

arch/powerpc/include/asm/book3s/32/pgtable.h

··· 194 194 #define VMALLOC_END ioremap_bot 195 195 #endif 196 196 197 - #ifdef CONFIG_STRICT_KERNEL_RWX 198 197 #define MODULES_END ALIGN_DOWN(PAGE_OFFSET, SZ_256M) 199 198 #define MODULES_VADDR (MODULES_END - SZ_256M) 200 - #endif 201 199 202 200 #ifndef __ASSEMBLY__ 203 201 #include <linux/sched.h>

+1 -1

arch/powerpc/include/asm/book3s/32/tlbflush.h

··· 79 79 flush_tlb_mm(mm); 80 80 } 81 81 82 - #endif /* _ASM_POWERPC_TLBFLUSH_H */ 82 + #endif /* _ASM_POWERPC_BOOK3S_32_TLBFLUSH_H */

+2 -22

arch/powerpc/include/asm/book3s/64/kup.h

··· 287 287 */ 288 288 } 289 289 290 - static inline unsigned long kuap_get_and_check_amr(void) 290 + static inline unsigned long kuap_get_and_assert_locked(void) 291 291 { 292 292 if (mmu_has_feature(MMU_FTR_BOOK3S_KUAP)) { 293 293 unsigned long amr = mfspr(SPRN_AMR); ··· 298 298 return 0; 299 299 } 300 300 301 - #else /* CONFIG_PPC_PKEY */ 302 - 303 - static inline void kuap_user_restore(struct pt_regs *regs) 304 - { 305 - } 306 - 307 - static inline void kuap_kernel_restore(struct pt_regs *regs, unsigned long amr) 308 - { 309 - } 310 - 311 - static inline unsigned long kuap_get_and_check_amr(void) 312 - { 313 - return 0; 314 - } 315 - 316 - #endif /* CONFIG_PPC_PKEY */ 317 - 318 - 319 - #ifdef CONFIG_PPC_KUAP 320 - 321 - static inline void kuap_check_amr(void) 301 + static inline void kuap_assert_locked(void) 322 302 { 323 303 if (IS_ENABLED(CONFIG_PPC_KUAP_DEBUG) && mmu_has_feature(MMU_FTR_BOOK3S_KUAP)) 324 304 WARN_ON_ONCE(mfspr(SPRN_AMR) != AMR_KUAP_BLOCKED);

-1

arch/powerpc/include/asm/book3s/64/mmu-hash.h

··· 18 18 * complete pgtable.h but only a portion of it. 19 19 */ 20 20 #include <asm/book3s/64/pgtable.h> 21 - #include <asm/bug.h> 22 21 #include <asm/task_size_64.h> 23 22 #include <asm/cpu_has_feature.h> 24 23

+4 -1

arch/powerpc/include/asm/book3s/64/pgtable.h

··· 7 7 #ifndef __ASSEMBLY__ 8 8 #include <linux/mmdebug.h> 9 9 #include <linux/bug.h> 10 + #include <linux/sizes.h> 10 11 #endif 11 12 12 13 /* ··· 117 116 */ 118 117 #define _PAGE_KERNEL_RW (_PAGE_PRIVILEGED | _PAGE_RW | _PAGE_DIRTY) 119 118 #define _PAGE_KERNEL_RO (_PAGE_PRIVILEGED | _PAGE_READ) 119 + #define _PAGE_KERNEL_ROX (_PAGE_PRIVILEGED | _PAGE_READ | _PAGE_EXEC) 120 120 #define _PAGE_KERNEL_RWX (_PAGE_PRIVILEGED | _PAGE_DIRTY | \ 121 121 _PAGE_RW | _PAGE_EXEC) 122 122 /* ··· 325 323 #define PHB_IO_END (KERN_IO_START + FULL_IO_SIZE) 326 324 #define IOREMAP_BASE (PHB_IO_END) 327 325 #define IOREMAP_START (ioremap_bot) 328 - #define IOREMAP_END (KERN_IO_END) 326 + #define IOREMAP_END (KERN_IO_END - FIXADDR_SIZE) 327 + #define FIXADDR_SIZE SZ_32M 329 328 330 329 /* Advertise special mapping type for AGP */ 331 330 #define HAVE_PAGE_AGP

+4 -2

arch/powerpc/include/asm/book3s/64/radix.h

··· 222 222 * from ptesync, it should probably go into update_mmu_cache, rather 223 223 * than set_pte_at (which is used to set ptes unrelated to faults). 224 224 * 225 - * Spurious faults to vmalloc region are not tolerated, so there is 226 - * a ptesync in flush_cache_vmap. 225 + * Spurious faults from the kernel memory are not tolerated, so there 226 + * is a ptesync in flush_cache_vmap, and __map_kernel_page() follows 227 + * the pte update sequence from ISA Book III 6.10 Translation Table 228 + * Update Synchronization Requirements. 227 229 */ 228 230 } 229 231

+1 -4

arch/powerpc/include/asm/bug.h

··· 111 111 #ifndef __ASSEMBLY__ 112 112 113 113 struct pt_regs; 114 - long do_page_fault(struct pt_regs *); 115 - long hash__do_page_fault(struct pt_regs *); 114 + void hash__do_page_fault(struct pt_regs *); 116 115 void bad_page_fault(struct pt_regs *, int); 117 - void __bad_page_fault(struct pt_regs *regs, int sig); 118 - void do_bad_page_fault_segv(struct pt_regs *regs); 119 116 extern void _exception(int, struct pt_regs *, int, unsigned long); 120 117 extern void _exception_pkey(struct pt_regs *, unsigned long, int); 121 118 extern void die(const char *, struct pt_regs *, long);

+13 -2

arch/powerpc/include/asm/cacheflush.h

··· 30 30 #endif /* CONFIG_PPC_BOOK3S_64 */ 31 31 32 32 #define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE 1 33 - extern void flush_dcache_page(struct page *page); 33 + /* 34 + * This is called when a page has been modified by the kernel. 35 + * It just marks the page as not i-cache clean. We do the i-cache 36 + * flush later when the page is given to a user process, if necessary. 37 + */ 38 + static inline void flush_dcache_page(struct page *page) 39 + { 40 + if (cpu_has_feature(CPU_FTR_COHERENT_ICACHE)) 41 + return; 42 + /* avoid an atomic op if possible */ 43 + if (test_bit(PG_dcache_clean, &page->flags)) 44 + clear_bit(PG_dcache_clean, &page->flags); 45 + } 34 46 35 47 void flush_icache_range(unsigned long start, unsigned long stop); 36 48 #define flush_icache_range flush_icache_range ··· 52 40 #define flush_icache_user_page flush_icache_user_page 53 41 54 42 void flush_dcache_icache_page(struct page *page); 55 - void __flush_dcache_icache(void *page); 56 43 57 44 /** 58 45 * flush_dcache_range(): Write any modified data cache blocks out to memory and

+1 -1

arch/powerpc/include/asm/cpm2.h

··· 594 594 uint fen_p256c; /* Total packets 256 < bytes <= 511 */ 595 595 uint fen_p512c; /* Total packets 512 < bytes <= 1023 */ 596 596 uint fen_p1024c; /* Total packets 1024 < bytes <= 1518 */ 597 - uint fen_cambuf; /* Internal CAM buffer poiner */ 597 + uint fen_cambuf; /* Internal CAM buffer pointer */ 598 598 ushort fen_rfthr; /* Received frames threshold */ 599 599 ushort fen_rfcnt; /* Received frames count */ 600 600 } fcc_enet_t;

+9

arch/powerpc/include/asm/fixmap.h

··· 23 23 #include <asm/kmap_size.h> 24 24 #endif 25 25 26 + #ifdef CONFIG_PPC64 27 + #define FIXADDR_TOP (IOREMAP_END + FIXADDR_SIZE) 28 + #else 29 + #define FIXADDR_SIZE 0 26 30 #ifdef CONFIG_KASAN 27 31 #include <asm/kasan.h> 28 32 #define FIXADDR_TOP (KASAN_SHADOW_START - PAGE_SIZE) 29 33 #else 30 34 #define FIXADDR_TOP ((unsigned long)(-PAGE_SIZE)) 35 + #endif 31 36 #endif 32 37 33 38 /* ··· 55 50 */ 56 51 enum fixed_addresses { 57 52 FIX_HOLE, 53 + #ifdef CONFIG_PPC32 58 54 /* reserve the top 128K for early debugging purposes */ 59 55 FIX_EARLY_DEBUG_TOP = FIX_HOLE, 60 56 FIX_EARLY_DEBUG_BASE = FIX_EARLY_DEBUG_TOP+(ALIGN(SZ_128K, PAGE_SIZE)/PAGE_SIZE)-1, ··· 78 72 FIX_IMMR_SIZE, 79 73 #endif 80 74 /* FIX_PCIE_MCFG, */ 75 + #endif /* CONFIG_PPC32 */ 81 76 __end_of_permanent_fixed_addresses, 82 77 83 78 #define NR_FIX_BTMAPS (SZ_256K / PAGE_SIZE) ··· 105 98 static inline void __set_fixmap(enum fixed_addresses idx, 106 99 phys_addr_t phys, pgprot_t flags) 107 100 { 101 + BUILD_BUG_ON(IS_ENABLED(CONFIG_PPC64) && __FIXADDR_SIZE > FIXADDR_SIZE); 102 + 108 103 if (__builtin_constant_p(idx)) 109 104 BUILD_BUG_ON(idx >= __end_of_fixed_addresses); 110 105 else if (WARN_ON(idx >= __end_of_fixed_addresses))

+5 -7

arch/powerpc/include/asm/futex.h

··· 33 33 { 34 34 int oldval = 0, ret; 35 35 36 - if (!access_ok(uaddr, sizeof(u32))) 36 + if (!user_access_begin(uaddr, sizeof(u32))) 37 37 return -EFAULT; 38 - allow_read_write_user(uaddr, uaddr, sizeof(*uaddr)); 39 38 40 39 switch (op) { 41 40 case FUTEX_OP_SET: ··· 55 56 default: 56 57 ret = -ENOSYS; 57 58 } 59 + user_access_end(); 58 60 59 61 *oval = oldval; 60 62 61 - prevent_read_write_user(uaddr, uaddr, sizeof(*uaddr)); 62 63 return ret; 63 64 } 64 65 ··· 69 70 int ret = 0; 70 71 u32 prev; 71 72 72 - if (!access_ok(uaddr, sizeof(u32))) 73 + if (!user_access_begin(uaddr, sizeof(u32))) 73 74 return -EFAULT; 74 - 75 - allow_read_write_user(uaddr, uaddr, sizeof(*uaddr)); 76 75 77 76 __asm__ __volatile__ ( 78 77 PPC_ATOMIC_ENTRY_BARRIER ··· 90 93 : "r" (uaddr), "r" (oldval), "r" (newval), "i" (-EFAULT) 91 94 : "cc", "memory"); 92 95 96 + user_access_end(); 97 + 93 98 *uval = prev; 94 - prevent_read_write_user(uaddr, uaddr, sizeof(*uaddr)); 95 99 96 100 return ret; 97 101 }

+3 -1

arch/powerpc/include/asm/hvcall.h

··· 315 315 #define H_SCM_HEALTH 0x400 316 316 #define H_SCM_PERFORMANCE_STATS 0x418 317 317 #define H_RPT_INVALIDATE 0x448 318 - #define MAX_HCALL_OPCODE H_RPT_INVALIDATE 318 + #define H_SCM_FLUSH 0x44C 319 + #define MAX_HCALL_OPCODE H_SCM_FLUSH 319 320 320 321 /* Scope args for H_SCM_UNBIND_ALL */ 321 322 #define H_UNBIND_SCOPE_ALL (0x1) ··· 390 389 #define H_CPU_BEHAV_FAVOUR_SECURITY (1ull << 63) // IBM bit 0 391 390 #define H_CPU_BEHAV_L1D_FLUSH_PR (1ull << 62) // IBM bit 1 392 391 #define H_CPU_BEHAV_BNDS_CHK_SPEC_BAR (1ull << 61) // IBM bit 2 392 + #define H_CPU_BEHAV_FAVOUR_SECURITY_H (1ull << 60) // IBM bit 3 393 393 #define H_CPU_BEHAV_FLUSH_COUNT_CACHE (1ull << 58) // IBM bit 5 394 394 #define H_CPU_BEHAV_FLUSH_LINK_STACK (1ull << 57) // IBM bit 6 395 395

+3

arch/powerpc/include/asm/hvconsole.h

··· 24 24 extern int hvc_get_chars(uint32_t vtermno, char *buf, int count); 25 25 extern int hvc_put_chars(uint32_t vtermno, const char *buf, int count); 26 26 27 + /* Provided by HVC VIO */ 28 + void hvc_vio_init_early(void); 29 + 27 30 #endif /* __KERNEL__ */ 28 31 #endif /* _PPC64_HVCONSOLE_H */

-2

arch/powerpc/include/asm/hydra.h

··· 94 94 #define HYDRA_INT_EXT7 18 /* Power Off Request */ 95 95 #define HYDRA_INT_SPARE 19 96 96 97 - extern int hydra_init(void); 98 - 99 97 #endif /* __KERNEL__ */ 100 98 101 99 #endif /* _ASMPPC_HYDRA_H */

+44 -11

arch/powerpc/include/asm/inst.h

··· 4 4 5 5 #include <asm/ppc-opcode.h> 6 6 7 + #ifdef CONFIG_PPC64 8 + 9 + #define ___get_user_instr(gu_op, dest, ptr) \ 10 + ({ \ 11 + long __gui_ret = 0; \ 12 + unsigned long __gui_ptr = (unsigned long)ptr; \ 13 + struct ppc_inst __gui_inst; \ 14 + unsigned int __prefix, __suffix; \ 15 + __gui_ret = gu_op(__prefix, (unsigned int __user *)__gui_ptr); \ 16 + if (__gui_ret == 0) { \ 17 + if ((__prefix >> 26) == OP_PREFIX) { \ 18 + __gui_ret = gu_op(__suffix, \ 19 + (unsigned int __user *)__gui_ptr + 1); \ 20 + __gui_inst = ppc_inst_prefix(__prefix, \ 21 + __suffix); \ 22 + } else { \ 23 + __gui_inst = ppc_inst(__prefix); \ 24 + } \ 25 + if (__gui_ret == 0) \ 26 + (dest) = __gui_inst; \ 27 + } \ 28 + __gui_ret; \ 29 + }) 30 + #else /* !CONFIG_PPC64 */ 31 + #define ___get_user_instr(gu_op, dest, ptr) \ 32 + gu_op((dest).val, (u32 __user *)(ptr)) 33 + #endif /* CONFIG_PPC64 */ 34 + 35 + #define get_user_instr(x, ptr) \ 36 + ___get_user_instr(get_user, x, ptr) 37 + 38 + #define __get_user_instr(x, ptr) \ 39 + ___get_user_instr(__get_user, x, ptr) 40 + 7 41 /* 8 42 * Instruction data type for POWER 9 43 */ ··· 102 68 103 69 #define ppc_inst(x) ((struct ppc_inst){ .val = x }) 104 70 71 + #define ppc_inst_prefix(x, y) ppc_inst(x) 72 + 105 73 static inline bool ppc_inst_prefixed(struct ppc_inst x) 106 74 { 107 75 return false; ··· 149 113 return location + ppc_inst_len(tmp); 150 114 } 151 115 152 - static inline u64 ppc_inst_as_u64(struct ppc_inst x) 116 + static inline unsigned long ppc_inst_as_ulong(struct ppc_inst x) 153 117 { 154 - #ifdef CONFIG_CPU_LITTLE_ENDIAN 155 - return (u64)ppc_inst_suffix(x) << 32 | ppc_inst_val(x); 156 - #else 157 - return (u64)ppc_inst_val(x) << 32 | ppc_inst_suffix(x); 158 - #endif 118 + if (IS_ENABLED(CONFIG_PPC32)) 119 + return ppc_inst_val(x); 120 + else if (IS_ENABLED(CONFIG_CPU_LITTLE_ENDIAN)) 121 + return (u64)ppc_inst_suffix(x) << 32 | ppc_inst_val(x); 122 + else 123 + return (u64)ppc_inst_val(x) << 32 | ppc_inst_suffix(x); 159 124 } 160 125 161 126 #define PPC_INST_STR_LEN sizeof("00000000 00000000") ··· 178 141 __str; \ 179 142 }) 180 143 181 - int probe_user_read_inst(struct ppc_inst *inst, 182 - struct ppc_inst __user *nip); 183 - 184 - int probe_kernel_read_inst(struct ppc_inst *inst, 185 - struct ppc_inst *src); 144 + int copy_inst_from_kernel_nofault(struct ppc_inst *inst, struct ppc_inst *src); 186 145 187 146 #endif /* _ASM_POWERPC_INST_H */

+131 -42

arch/powerpc/include/asm/interrupt.h

··· 2 2 #ifndef _ASM_POWERPC_INTERRUPT_H 3 3 #define _ASM_POWERPC_INTERRUPT_H 4 4 5 + /* BookE/4xx */ 6 + #define INTERRUPT_CRITICAL_INPUT 0x100 7 + 8 + /* BookE */ 9 + #define INTERRUPT_DEBUG 0xd00 10 + #ifdef CONFIG_BOOKE 11 + #define INTERRUPT_PERFMON 0x260 12 + #define INTERRUPT_DOORBELL 0x280 13 + #endif 14 + 15 + /* BookS/4xx/8xx */ 16 + #define INTERRUPT_MACHINE_CHECK 0x200 17 + 18 + /* BookS/8xx */ 19 + #define INTERRUPT_SYSTEM_RESET 0x100 20 + 21 + /* BookS */ 22 + #define INTERRUPT_DATA_SEGMENT 0x380 23 + #define INTERRUPT_INST_SEGMENT 0x480 24 + #define INTERRUPT_TRACE 0xd00 25 + #define INTERRUPT_H_DATA_STORAGE 0xe00 26 + #define INTERRUPT_HMI 0xe60 27 + #define INTERRUPT_H_FAC_UNAVAIL 0xf80 28 + #ifdef CONFIG_PPC_BOOK3S 29 + #define INTERRUPT_DOORBELL 0xa00 30 + #define INTERRUPT_PERFMON 0xf00 31 + #define INTERRUPT_ALTIVEC_UNAVAIL 0xf20 32 + #endif 33 + 34 + /* BookE/BookS/4xx/8xx */ 35 + #define INTERRUPT_DATA_STORAGE 0x300 36 + #define INTERRUPT_INST_STORAGE 0x400 37 + #define INTERRUPT_EXTERNAL 0x500 38 + #define INTERRUPT_ALIGNMENT 0x600 39 + #define INTERRUPT_PROGRAM 0x700 40 + #define INTERRUPT_SYSCALL 0xc00 41 + #define INTERRUPT_TRACE 0xd00 42 + 43 + /* BookE/BookS/44x */ 44 + #define INTERRUPT_FP_UNAVAIL 0x800 45 + 46 + /* BookE/BookS/44x/8xx */ 47 + #define INTERRUPT_DECREMENTER 0x900 48 + 49 + #ifndef INTERRUPT_PERFMON 50 + #define INTERRUPT_PERFMON 0x0 51 + #endif 52 + 53 + /* 8xx */ 54 + #define INTERRUPT_SOFT_EMU_8xx 0x1000 55 + #define INTERRUPT_INST_TLB_MISS_8xx 0x1100 56 + #define INTERRUPT_DATA_TLB_MISS_8xx 0x1200 57 + #define INTERRUPT_INST_TLB_ERROR_8xx 0x1300 58 + #define INTERRUPT_DATA_TLB_ERROR_8xx 0x1400 59 + #define INTERRUPT_DATA_BREAKPOINT_8xx 0x1c00 60 + #define INTERRUPT_INST_BREAKPOINT_8xx 0x1d00 61 + 62 + /* 603 */ 63 + #define INTERRUPT_INST_TLB_MISS_603 0x1000 64 + #define INTERRUPT_DATA_LOAD_TLB_MISS_603 0x1100 65 + #define INTERRUPT_DATA_STORE_TLB_MISS_603 0x1200 66 + 67 + #ifndef __ASSEMBLY__ 68 + 5 69 #include <linux/context_tracking.h> 6 70 #include <linux/hardirq.h> 7 71 #include <asm/cputime.h> ··· 73 9 #include <asm/kprobes.h> 74 10 #include <asm/runlatch.h> 75 11 76 - struct interrupt_state { 77 - #ifdef CONFIG_PPC_BOOK3E_64 78 - enum ctx_state ctx_state; 12 + static inline void nap_adjust_return(struct pt_regs *regs) 13 + { 14 + #ifdef CONFIG_PPC_970_NAP 15 + if (unlikely(test_thread_local_flags(_TLF_NAPPING))) { 16 + /* Can avoid a test-and-clear because NMIs do not call this */ 17 + clear_thread_local_flags(_TLF_NAPPING); 18 + regs->nip = (unsigned long)power4_idle_nap_return; 19 + } 79 20 #endif 21 + } 22 + 23 + struct interrupt_state { 80 24 }; 81 25 82 26 static inline void booke_restore_dbcr0(void) ··· 101 29 102 30 static inline void interrupt_enter_prepare(struct pt_regs *regs, struct interrupt_state *state) 103 31 { 104 - /* 105 - * Book3E reconciles irq soft mask in asm 106 - */ 107 - #ifdef CONFIG_PPC_BOOK3S_64 32 + #ifdef CONFIG_PPC32 33 + if (!arch_irq_disabled_regs(regs)) 34 + trace_hardirqs_off(); 35 + 36 + if (user_mode(regs)) { 37 + kuep_lock(); 38 + account_cpu_user_entry(); 39 + } else { 40 + kuap_save_and_lock(regs); 41 + } 42 + #endif 43 + 44 + #ifdef CONFIG_PPC64 108 45 if (irq_soft_mask_set_return(IRQS_ALL_DISABLED) == IRQS_ENABLED) 109 46 trace_hardirqs_off(); 110 47 local_paca->irq_happened |= PACA_IRQ_HARD_DIS; ··· 129 48 * CT_WARN_ON comes here via program_check_exception, 130 49 * so avoid recursion. 131 50 */ 132 - if (TRAP(regs) != 0x700) 51 + if (TRAP(regs) != INTERRUPT_PROGRAM) 133 52 CT_WARN_ON(ct_state() != CONTEXT_KERNEL); 134 53 } 135 54 #endif 136 55 137 - #ifdef CONFIG_PPC_BOOK3E_64 138 - state->ctx_state = exception_enter(); 139 - if (user_mode(regs)) 140 - account_cpu_user_entry(); 141 - #endif 56 + booke_restore_dbcr0(); 142 57 } 143 58 144 59 /* ··· 153 76 */ 154 77 static inline void interrupt_exit_prepare(struct pt_regs *regs, struct interrupt_state *state) 155 78 { 156 - #ifdef CONFIG_PPC_BOOK3E_64 157 - exception_exit(state->ctx_state); 158 - #endif 159 - 160 - /* 161 - * Book3S exits to user via interrupt_exit_user_prepare(), which does 162 - * context tracking, which is a cleaner way to handle PREEMPT=y 163 - * and avoid context entry/exit in e.g., preempt_schedule_irq()), 164 - * which is likely to be where the core code wants to end up. 165 - * 166 - * The above comment explains why we can't do the 167 - * 168 - * if (user_mode(regs)) 169 - * user_exit_irqoff(); 170 - * 171 - * sequence here. 172 - */ 79 + if (user_mode(regs)) 80 + kuep_unlock(); 173 81 } 174 82 175 83 static inline void interrupt_async_enter_prepare(struct pt_regs *regs, struct interrupt_state *state) ··· 171 109 172 110 static inline void interrupt_async_exit_prepare(struct pt_regs *regs, struct interrupt_state *state) 173 111 { 112 + /* 113 + * Adjust at exit so the main handler sees the true NIA. This must 114 + * come before irq_exit() because irq_exit can enable interrupts, and 115 + * if another interrupt is taken before nap_adjust_return has run 116 + * here, then that interrupt would return directly to idle nap return. 117 + */ 118 + nap_adjust_return(regs); 119 + 174 120 irq_exit(); 175 121 interrupt_exit_prepare(regs, state); 176 122 } 177 123 178 124 struct interrupt_nmi_state { 179 125 #ifdef CONFIG_PPC64 180 - #ifdef CONFIG_PPC_BOOK3S_64 181 126 u8 irq_soft_mask; 182 127 u8 irq_happened; 183 - #endif 184 128 u8 ftrace_enabled; 185 129 #endif 186 130 }; 187 131 132 + static inline bool nmi_disables_ftrace(struct pt_regs *regs) 133 + { 134 + /* Allow DEC and PMI to be traced when they are soft-NMI */ 135 + if (IS_ENABLED(CONFIG_PPC_BOOK3S_64)) { 136 + if (TRAP(regs) == INTERRUPT_DECREMENTER) 137 + return false; 138 + if (TRAP(regs) == INTERRUPT_PERFMON) 139 + return false; 140 + } 141 + if (IS_ENABLED(CONFIG_PPC_BOOK3E)) { 142 + if (TRAP(regs) == INTERRUPT_PERFMON) 143 + return false; 144 + } 145 + 146 + return true; 147 + } 148 + 188 149 static inline void interrupt_nmi_enter_prepare(struct pt_regs *regs, struct interrupt_nmi_state *state) 189 150 { 190 151 #ifdef CONFIG_PPC64 191 - #ifdef CONFIG_PPC_BOOK3S_64 192 152 state->irq_soft_mask = local_paca->irq_soft_mask; 193 153 state->irq_happened = local_paca->irq_happened; 194 154 ··· 223 139 local_paca->irq_happened |= PACA_IRQ_HARD_DIS; 224 140 225 141 /* Don't do any per-CPU operations until interrupt state is fixed */ 226 - #endif 227 - /* Allow DEC and PMI to be traced when they are soft-NMI */ 228 - if (TRAP(regs) != 0x900 && TRAP(regs) != 0xf00 && TRAP(regs) != 0x260) { 142 + 143 + if (nmi_disables_ftrace(regs)) { 229 144 state->ftrace_enabled = this_cpu_get_ftrace_enabled(); 230 145 this_cpu_set_ftrace_enabled(0); 231 146 } ··· 247 164 radix_enabled() || (mfmsr() & MSR_DR)) 248 165 nmi_exit(); 249 166 167 + /* 168 + * nmi does not call nap_adjust_return because nmi should not create 169 + * new work to do (must use irq_work for that). 170 + */ 171 + 250 172 #ifdef CONFIG_PPC64 251 - if (TRAP(regs) != 0x900 && TRAP(regs) != 0xf00 && TRAP(regs) != 0x260) 173 + if (nmi_disables_ftrace(regs)) 252 174 this_cpu_set_ftrace_enabled(state->ftrace_enabled); 253 175 254 - #ifdef CONFIG_PPC_BOOK3S_64 255 176 /* Check we didn't change the pending interrupt mask. */ 256 177 WARN_ON_ONCE((state->irq_happened | PACA_IRQ_HARD_DIS) != local_paca->irq_happened); 257 178 local_paca->irq_happened = state->irq_happened; 258 179 local_paca->irq_soft_mask = state->irq_soft_mask; 259 - #endif 260 180 #endif 261 181 } 262 182 ··· 473 387 DECLARE_INTERRUPT_HANDLER(handle_hmi_exception); 474 388 DECLARE_INTERRUPT_HANDLER(unknown_exception); 475 389 DECLARE_INTERRUPT_HANDLER_ASYNC(unknown_async_exception); 390 + DECLARE_INTERRUPT_HANDLER_NMI(unknown_nmi_exception); 476 391 DECLARE_INTERRUPT_HANDLER(instruction_breakpoint_exception); 477 392 DECLARE_INTERRUPT_HANDLER(RunModeException); 478 393 DECLARE_INTERRUPT_HANDLER(single_step_exception); ··· 497 410 DECLARE_INTERRUPT_HANDLER(CacheLockingException); 498 411 DECLARE_INTERRUPT_HANDLER(SPEFloatingPointException); 499 412 DECLARE_INTERRUPT_HANDLER(SPEFloatingPointRoundException); 500 - DECLARE_INTERRUPT_HANDLER(WatchdogException); 413 + DECLARE_INTERRUPT_HANDLER_NMI(WatchdogException); 501 414 DECLARE_INTERRUPT_HANDLER(kernel_bad_stack); 502 415 503 416 /* slb.c */ ··· 508 421 DECLARE_INTERRUPT_HANDLER_RAW(do_hash_fault); 509 422 510 423 /* fault.c */ 511 - DECLARE_INTERRUPT_HANDLER_RET(do_page_fault); 424 + DECLARE_INTERRUPT_HANDLER(do_page_fault); 512 425 DECLARE_INTERRUPT_HANDLER(do_bad_page_fault_segv); 513 426 514 427 /* process.c */ ··· 523 436 524 437 DECLARE_INTERRUPT_HANDLER_ASYNC(TAUException); 525 438 526 - void unrecoverable_exception(struct pt_regs *regs); 439 + void __noreturn unrecoverable_exception(struct pt_regs *regs); 527 440 528 441 void replay_system_reset(void); 529 442 void replay_soft_interrupts(void); ··· 533 446 if (!arch_irq_disabled_regs(regs)) 534 447 local_irq_enable(); 535 448 } 449 + 450 + #endif /* __ASSEMBLY__ */ 536 451 537 452 #endif /* _ASM_POWERPC_INTERRUPT_H */

-2

arch/powerpc/include/asm/irq.h

··· 53 53 extern void *hardirq_ctx[NR_CPUS]; 54 54 extern void *softirq_ctx[NR_CPUS]; 55 55 56 - void call_do_softirq(void *sp); 57 - void call_do_irq(struct pt_regs *regs, void *sp); 58 56 extern void do_IRQ(struct pt_regs *regs); 59 57 extern void __init init_IRQ(void); 60 58 extern void __do_irq(struct pt_regs *regs);

+6 -15

arch/powerpc/include/asm/jump_label.h

··· 20 20 asm_volatile_goto("1:\n\t" 21 21 "nop # arch_static_branch\n\t" 22 22 ".pushsection __jump_table, \"aw\"\n\t" 23 - JUMP_ENTRY_TYPE "1b, %l[l_yes], %c0\n\t" 23 + ".long 1b - ., %l[l_yes] - .\n\t" 24 + JUMP_ENTRY_TYPE "%c0 - .\n\t" 24 25 ".popsection \n\t" 25 26 : : "i" (&((char *)key)[branch]) : : l_yes); 26 27 ··· 35 34 asm_volatile_goto("1:\n\t" 36 35 "b %l[l_yes] # arch_static_branch_jump\n\t" 37 36 ".pushsection __jump_table, \"aw\"\n\t" 38 - JUMP_ENTRY_TYPE "1b, %l[l_yes], %c0\n\t" 37 + ".long 1b - ., %l[l_yes] - .\n\t" 38 + JUMP_ENTRY_TYPE "%c0 - .\n\t" 39 39 ".popsection \n\t" 40 40 : : "i" (&((char *)key)[branch]) : : l_yes); 41 41 ··· 45 43 return true; 46 44 } 47 45 48 - #ifdef CONFIG_PPC64 49 - typedef u64 jump_label_t; 50 - #else 51 - typedef u32 jump_label_t; 52 - #endif 53 - 54 - struct jump_entry { 55 - jump_label_t code; 56 - jump_label_t target; 57 - jump_label_t key; 58 - }; 59 - 60 46 #else 61 47 #define ARCH_STATIC_BRANCH(LABEL, KEY) \ 62 48 1098: nop; \ 63 49 .pushsection __jump_table, "aw"; \ 64 - FTR_ENTRY_LONG 1098b, LABEL, KEY; \ 50 + .long 1098b - ., LABEL - .; \ 51 + FTR_ENTRY_LONG KEY; \ 65 52 .popsection 66 53 #endif 67 54

+1 -1

arch/powerpc/include/asm/kasan.h

··· 19 19 20 20 #define KASAN_SHADOW_SCALE_SHIFT 3 21 21 22 - #if defined(CONFIG_PPC_BOOK3S_32) && defined(CONFIG_MODULES) && defined(CONFIG_STRICT_KERNEL_RWX) 22 + #ifdef CONFIG_MODULES 23 23 #define KASAN_KERN_START ALIGN_DOWN(PAGE_OFFSET - SZ_256M, SZ_256M) 24 24 #else 25 25 #define KASAN_KERN_START PAGE_OFFSET

+33

arch/powerpc/include/asm/kfence.h

··· 1 + /* SPDX-License-Identifier: GPL-2.0 */ 2 + /* 3 + * powerpc KFENCE support. 4 + * 5 + * Copyright (C) 2020 CS GROUP France 6 + */ 7 + 8 + #ifndef __ASM_POWERPC_KFENCE_H 9 + #define __ASM_POWERPC_KFENCE_H 10 + 11 + #include <linux/mm.h> 12 + #include <asm/pgtable.h> 13 + 14 + static inline bool arch_kfence_init_pool(void) 15 + { 16 + return true; 17 + } 18 + 19 + static inline bool kfence_protect_page(unsigned long addr, bool protect) 20 + { 21 + pte_t *kpte = virt_to_kpte(addr); 22 + 23 + if (protect) { 24 + pte_update(&init_mm, addr, kpte, _PAGE_PRESENT, 0, 0); 25 + flush_tlb_kernel_range(addr, addr + PAGE_SIZE); 26 + } else { 27 + pte_update(&init_mm, addr, kpte, 0, _PAGE_PRESENT, 0); 28 + } 29 + 30 + return true; 31 + } 32 + 33 + #endif /* __ASM_POWERPC_KFENCE_H */

+17 -10

arch/powerpc/include/asm/kup.h

··· 28 28 29 29 #ifdef __ASSEMBLY__ 30 30 #ifndef CONFIG_PPC_KUAP 31 - .macro kuap_save_and_lock sp, thread, gpr1, gpr2, gpr3 32 - .endm 33 - 34 - .macro kuap_restore sp, current, gpr1, gpr2, gpr3 35 - .endm 36 - 37 - .macro kuap_check current, gpr 38 - .endm 39 - 40 31 .macro kuap_check_amr gpr1, gpr2 41 32 .endm 42 33 ··· 46 55 static inline void setup_kuep(bool disabled) { } 47 56 #endif /* CONFIG_PPC_KUEP */ 48 57 58 + #if defined(CONFIG_PPC_KUEP) && defined(CONFIG_PPC_BOOK3S_32) 59 + void kuep_lock(void); 60 + void kuep_unlock(void); 61 + #else 62 + static inline void kuep_lock(void) { } 63 + static inline void kuep_unlock(void) { } 64 + #endif 65 + 49 66 #ifdef CONFIG_PPC_KUAP 50 67 void setup_kuap(bool disabled); 51 68 #else ··· 65 66 return false; 66 67 } 67 68 68 - static inline void kuap_check_amr(void) { } 69 + static inline void kuap_assert_locked(void) { } 70 + static inline void kuap_save_and_lock(struct pt_regs *regs) { } 71 + static inline void kuap_user_restore(struct pt_regs *regs) { } 72 + static inline void kuap_kernel_restore(struct pt_regs *regs, unsigned long amr) { } 73 + 74 + static inline unsigned long kuap_get_and_assert_locked(void) 75 + { 76 + return 0; 77 + } 69 78 70 79 /* 71 80 * book3s/64/kup-radix.h defines these functions for the !KUAP case to flush

+2

arch/powerpc/include/asm/kvm_book3s.h

··· 258 258 extern void kvmppc_harvest_vpa_dirty(struct kvmppc_vpa *vpa, 259 259 struct kvm_memory_slot *memslot, 260 260 unsigned long *map); 261 + extern unsigned long kvmppc_filter_lpcr_hv(struct kvm *kvm, 262 + unsigned long lpcr); 261 263 extern void kvmppc_update_lpcr(struct kvm *kvm, unsigned long lpcr, 262 264 unsigned long mask); 263 265 extern void kvmppc_set_fscr(struct kvm_vcpu *vcpu, u64 fscr);

+1 -2

arch/powerpc/include/asm/kvm_ppc.h

··· 767 767 unsigned long pte_index, unsigned long avpn); 768 768 long kvmppc_h_bulk_remove(struct kvm_vcpu *vcpu); 769 769 long kvmppc_h_protect(struct kvm_vcpu *vcpu, unsigned long flags, 770 - unsigned long pte_index, unsigned long avpn, 771 - unsigned long va); 770 + unsigned long pte_index, unsigned long avpn); 772 771 long kvmppc_h_read(struct kvm_vcpu *vcpu, unsigned long flags, 773 772 unsigned long pte_index); 774 773 long kvmppc_h_clear_ref(struct kvm_vcpu *vcpu, unsigned long flags,

+1 -1

arch/powerpc/include/asm/mmu_context.h

··· 263 263 static inline void arch_unmap(struct mm_struct *mm, 264 264 unsigned long start, unsigned long end) 265 265 { 266 - unsigned long vdso_base = (unsigned long)mm->context.vdso - PAGE_SIZE; 266 + unsigned long vdso_base = (unsigned long)mm->context.vdso; 267 267 268 268 if (start <= vdso_base && vdso_base < end) 269 269 mm->context.vdso = NULL;

+32 -24

arch/powerpc/include/asm/nohash/32/kup-8xx.h

··· 7 7 8 8 #ifdef CONFIG_PPC_KUAP 9 9 10 - #ifdef __ASSEMBLY__ 11 - 12 - .macro kuap_save_and_lock sp, thread, gpr1, gpr2, gpr3 13 - lis \gpr2, MD_APG_KUAP@h /* only APG0 and APG1 are used */ 14 - mfspr \gpr1, SPRN_MD_AP 15 - mtspr SPRN_MD_AP, \gpr2 16 - stw \gpr1, STACK_REGS_KUAP(\sp) 17 - .endm 18 - 19 - .macro kuap_restore sp, current, gpr1, gpr2, gpr3 20 - lwz \gpr1, STACK_REGS_KUAP(\sp) 21 - mtspr SPRN_MD_AP, \gpr1 22 - .endm 23 - 24 - .macro kuap_check current, gpr 25 - #ifdef CONFIG_PPC_KUAP_DEBUG 26 - mfspr \gpr, SPRN_MD_AP 27 - rlwinm \gpr, \gpr, 16, 0xffff 28 - 999: twnei \gpr, MD_APG_KUAP@h 29 - EMIT_BUG_ENTRY 999b, __FILE__, __LINE__, (BUGFLAG_WARNING | BUGFLAG_ONCE) 30 - #endif 31 - .endm 32 - 33 - #else /* !__ASSEMBLY__ */ 10 + #ifndef __ASSEMBLY__ 34 11 35 12 #include <asm/reg.h> 13 + 14 + static inline void kuap_save_and_lock(struct pt_regs *regs) 15 + { 16 + regs->kuap = mfspr(SPRN_MD_AP); 17 + mtspr(SPRN_MD_AP, MD_APG_KUAP); 18 + } 19 + 20 + static inline void kuap_user_restore(struct pt_regs *regs) 21 + { 22 + } 23 + 24 + static inline void kuap_kernel_restore(struct pt_regs *regs, unsigned long kuap) 25 + { 26 + mtspr(SPRN_MD_AP, regs->kuap); 27 + } 28 + 29 + static inline unsigned long kuap_get_and_assert_locked(void) 30 + { 31 + unsigned long kuap = mfspr(SPRN_MD_AP); 32 + 33 + if (IS_ENABLED(CONFIG_PPC_KUAP_DEBUG)) 34 + WARN_ON_ONCE(kuap >> 16 != MD_APG_KUAP >> 16); 35 + 36 + return kuap; 37 + } 38 + 39 + static inline void kuap_assert_locked(void) 40 + { 41 + if (IS_ENABLED(CONFIG_PPC_KUAP_DEBUG)) 42 + kuap_get_and_assert_locked(); 43 + } 36 44 37 45 static inline void allow_user_access(void __user *to, const void __user *from, 38 46 unsigned long size, unsigned long dir)

+3

arch/powerpc/include/asm/nohash/32/mmu-8xx.h

··· 172 172 173 173 #define mmu_linear_psize MMU_PAGE_8M 174 174 175 + #define MODULES_VADDR (PAGE_OFFSET - SZ_256M) 176 + #define MODULES_END PAGE_OFFSET 177 + 175 178 #ifndef __ASSEMBLY__ 176 179 177 180 #include <linux/mmdebug.h>

+4 -1

arch/powerpc/include/asm/nohash/64/pgtable.h

··· 6 6 * the ppc64 non-hashed page table. 7 7 */ 8 8 9 + #include <linux/sizes.h> 10 + 9 11 #include <asm/nohash/64/pgtable-4k.h> 10 12 #include <asm/barrier.h> 11 13 #include <asm/asm-const.h> ··· 56 54 #define PHB_IO_END (KERN_IO_START + FULL_IO_SIZE) 57 55 #define IOREMAP_BASE (PHB_IO_END) 58 56 #define IOREMAP_START (ioremap_bot) 59 - #define IOREMAP_END (KERN_VIRT_START + KERN_VIRT_SIZE) 57 + #define IOREMAP_END (KERN_VIRT_START + KERN_VIRT_SIZE - FIXADDR_SIZE) 58 + #define FIXADDR_SIZE SZ_32M 60 59 61 60 62 61 /*

+1 -1

arch/powerpc/include/asm/opal.h

··· 307 307 308 308 s64 opal_mpipl_update(enum opal_mpipl_ops op, u64 src, u64 dest, u64 size); 309 309 s64 opal_mpipl_register_tag(enum opal_mpipl_tags tag, u64 addr); 310 - s64 opal_mpipl_query_tag(enum opal_mpipl_tags tag, u64 *addr); 310 + s64 opal_mpipl_query_tag(enum opal_mpipl_tags tag, __be64 *addr); 311 311 312 312 s64 opal_signal_system_reset(s32 cpu); 313 313 s64 opal_quiesce(u64 shutdown_type, s32 cpu);

+7 -1

arch/powerpc/include/asm/perf_event_server.h

··· 43 43 u64 alt[]); 44 44 void (*get_mem_data_src)(union perf_mem_data_src *dsrc, 45 45 u32 flags, struct pt_regs *regs); 46 - void (*get_mem_weight)(u64 *weight); 46 + void (*get_mem_weight)(u64 *weight, u64 type); 47 47 unsigned long group_constraint_mask; 48 48 unsigned long group_constraint_val; 49 49 u64 (*bhrb_filter_map)(u64 branch_sample_type); ··· 67 67 * the pmu supports extended perf regs capability 68 68 */ 69 69 int capabilities; 70 + /* 71 + * Function to check event code for values which are 72 + * reserved. Function takes struct perf_event as input, 73 + * since event code could be spread in attr.config* 74 + */ 75 + int (*check_attr_config)(struct perf_event *ev); 70 76 }; 71 77 72 78 /*

-2

arch/powerpc/include/asm/pgtable.h

··· 41 41 42 42 #ifndef __ASSEMBLY__ 43 43 44 - #include <asm/tlbflush.h> 45 - 46 44 /* Keep these as a macros to avoid include dependency mess */ 47 45 #define pte_page(x) pfn_to_page(pte_pfn(x)) 48 46 #define mk_pte(page, pgprot) pfn_pte(page_to_pfn(page), (pgprot))

+13

arch/powerpc/include/asm/ppc-opcode.h

··· 265 265 #define PPC_INST_ORI 0x60000000 266 266 #define PPC_INST_ORIS 0x64000000 267 267 #define PPC_INST_BRANCH 0x48000000 268 + #define PPC_INST_BL 0x48000001 268 269 #define PPC_INST_BRANCH_COND 0x40800000 269 270 270 271 /* Prefixes */ ··· 438 437 #define PPC_RAW_STFDX(s, a, b) (0x7c0005ae | ___PPC_RS(s) | ___PPC_RA(a) | ___PPC_RB(b)) 439 438 #define PPC_RAW_LVX(t, a, b) (0x7c0000ce | ___PPC_RT(t) | ___PPC_RA(a) | ___PPC_RB(b)) 440 439 #define PPC_RAW_STVX(s, a, b) (0x7c0001ce | ___PPC_RS(s) | ___PPC_RA(a) | ___PPC_RB(b)) 440 + #define PPC_RAW_ADDE(t, a, b) (0x7c000114 | ___PPC_RT(t) | ___PPC_RA(a) | ___PPC_RB(b)) 441 + #define PPC_RAW_ADDZE(t, a) (0x7c000194 | ___PPC_RT(t) | ___PPC_RA(a)) 442 + #define PPC_RAW_ADDME(t, a) (0x7c0001d4 | ___PPC_RT(t) | ___PPC_RA(a)) 441 443 #define PPC_RAW_ADD(t, a, b) (PPC_INST_ADD | ___PPC_RT(t) | ___PPC_RA(a) | ___PPC_RB(b)) 442 444 #define PPC_RAW_ADD_DOT(t, a, b) (PPC_INST_ADD | ___PPC_RT(t) | ___PPC_RA(a) | ___PPC_RB(b) | 0x1) 443 445 #define PPC_RAW_ADDC(t, a, b) (0x7c000014 | ___PPC_RT(t) | ___PPC_RA(a) | ___PPC_RB(b)) ··· 449 445 #define PPC_RAW_BLR() (PPC_INST_BLR) 450 446 #define PPC_RAW_BLRL() (0x4e800021) 451 447 #define PPC_RAW_MTLR(r) (0x7c0803a6 | ___PPC_RT(r)) 448 + #define PPC_RAW_MFLR(t) (PPC_INST_MFLR | ___PPC_RT(t)) 452 449 #define PPC_RAW_BCTR() (PPC_INST_BCTR) 453 450 #define PPC_RAW_MTCTR(r) (PPC_INST_MTCTR | ___PPC_RT(r)) 454 451 #define PPC_RAW_ADDI(d, a, i) (PPC_INST_ADDI | ___PPC_RT(d) | ___PPC_RA(a) | IMM_L(i)) 455 452 #define PPC_RAW_LI(r, i) PPC_RAW_ADDI(r, 0, i) 456 453 #define PPC_RAW_ADDIS(d, a, i) (PPC_INST_ADDIS | ___PPC_RT(d) | ___PPC_RA(a) | IMM_L(i)) 454 + #define PPC_RAW_ADDIC(d, a, i) (0x30000000 | ___PPC_RT(d) | ___PPC_RA(a) | IMM_L(i)) 455 + #define PPC_RAW_ADDIC_DOT(d, a, i) (0x34000000 | ___PPC_RT(d) | ___PPC_RA(a) | IMM_L(i)) 457 456 #define PPC_RAW_LIS(r, i) PPC_RAW_ADDIS(r, 0, i) 458 457 #define PPC_RAW_STDX(r, base, b) (0x7c00012a | ___PPC_RS(r) | ___PPC_RA(base) | ___PPC_RB(b)) 459 458 #define PPC_RAW_STDU(r, base, i) (0xf8000001 | ___PPC_RS(r) | ___PPC_RA(base) | ((i) & 0xfffc)) ··· 479 472 #define PPC_RAW_CMPLW(a, b) (0x7c000040 | ___PPC_RA(a) | ___PPC_RB(b)) 480 473 #define PPC_RAW_CMPLD(a, b) (0x7c200040 | ___PPC_RA(a) | ___PPC_RB(b)) 481 474 #define PPC_RAW_SUB(d, a, b) (0x7c000050 | ___PPC_RT(d) | ___PPC_RB(a) | ___PPC_RA(b)) 475 + #define PPC_RAW_SUBFC(d, a, b) (0x7c000010 | ___PPC_RT(d) | ___PPC_RA(a) | ___PPC_RB(b)) 476 + #define PPC_RAW_SUBFE(d, a, b) (0x7c000110 | ___PPC_RT(d) | ___PPC_RA(a) | ___PPC_RB(b)) 477 + #define PPC_RAW_SUBFIC(d, a, i) (0x20000000 | ___PPC_RT(d) | ___PPC_RA(a) | IMM_L(i)) 478 + #define PPC_RAW_SUBFZE(d, a) (0x7c000190 | ___PPC_RT(d) | ___PPC_RA(a)) 482 479 #define PPC_RAW_MULD(d, a, b) (0x7c0001d2 | ___PPC_RT(d) | ___PPC_RA(a) | ___PPC_RB(b)) 483 480 #define PPC_RAW_MULW(d, a, b) (0x7c0001d6 | ___PPC_RT(d) | ___PPC_RA(a) | ___PPC_RB(b)) 484 481 #define PPC_RAW_MULHWU(d, a, b) (0x7c000016 | ___PPC_RT(d) | ___PPC_RA(a) | ___PPC_RB(b)) ··· 495 484 #define PPC_RAW_DIVDEU_DOT(t, a, b) (0x7c000312 | ___PPC_RT(t) | ___PPC_RA(a) | ___PPC_RB(b) | 0x1) 496 485 #define PPC_RAW_AND(d, a, b) (0x7c000038 | ___PPC_RA(d) | ___PPC_RS(a) | ___PPC_RB(b)) 497 486 #define PPC_RAW_ANDI(d, a, i) (0x70000000 | ___PPC_RA(d) | ___PPC_RS(a) | IMM_L(i)) 487 + #define PPC_RAW_ANDIS(d, a, i) (0x74000000 | ___PPC_RA(d) | ___PPC_RS(a) | IMM_L(i)) 498 488 #define PPC_RAW_AND_DOT(d, a, b) (0x7c000039 | ___PPC_RA(d) | ___PPC_RS(a) | ___PPC_RB(b)) 499 489 #define PPC_RAW_OR(d, a, b) (0x7c000378 | ___PPC_RA(d) | ___PPC_RS(a) | ___PPC_RB(b)) 500 490 #define PPC_RAW_MR(d, a) PPC_RAW_OR(d, a, a) 501 491 #define PPC_RAW_ORI(d, a, i) (PPC_INST_ORI | ___PPC_RA(d) | ___PPC_RS(a) | IMM_L(i)) 502 492 #define PPC_RAW_ORIS(d, a, i) (PPC_INST_ORIS | ___PPC_RA(d) | ___PPC_RS(a) | IMM_L(i)) 493 + #define PPC_RAW_NOR(d, a, b) (0x7c0000f8 | ___PPC_RA(d) | ___PPC_RS(a) | ___PPC_RB(b)) 503 494 #define PPC_RAW_XOR(d, a, b) (0x7c000278 | ___PPC_RA(d) | ___PPC_RS(a) | ___PPC_RB(b)) 504 495 #define PPC_RAW_XORI(d, a, i) (0x68000000 | ___PPC_RA(d) | ___PPC_RS(a) | IMM_L(i)) 505 496 #define PPC_RAW_XORIS(d, a, i) (0x6c000000 | ___PPC_RA(d) | ___PPC_RS(a) | IMM_L(i))

-30

arch/powerpc/include/asm/ppc_asm.h

··· 16 16 #define SZL (BITS_PER_LONG/8) 17 17 18 18 /* 19 - * Stuff for accurate CPU time accounting. 20 - * These macros handle transitions between user and system state 21 - * in exception entry and exit and accumulate time to the 22 - * user_time and system_time fields in the paca. 23 - */ 24 - 25 - #ifndef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE 26 - #define ACCOUNT_CPU_USER_ENTRY(ptr, ra, rb) 27 - #define ACCOUNT_CPU_USER_EXIT(ptr, ra, rb) 28 - #else 29 - #define ACCOUNT_CPU_USER_ENTRY(ptr, ra, rb) \ 30 - MFTB(ra); /* get timebase */ \ 31 - PPC_LL rb, ACCOUNT_STARTTIME_USER(ptr); \ 32 - PPC_STL ra, ACCOUNT_STARTTIME(ptr); \ 33 - subf rb,rb,ra; /* subtract start value */ \ 34 - PPC_LL ra, ACCOUNT_USER_TIME(ptr); \ 35 - add ra,ra,rb; /* add on to user time */ \ 36 - PPC_STL ra, ACCOUNT_USER_TIME(ptr); \ 37 - 38 - #define ACCOUNT_CPU_USER_EXIT(ptr, ra, rb) \ 39 - MFTB(ra); /* get timebase */ \ 40 - PPC_LL rb, ACCOUNT_STARTTIME(ptr); \ 41 - PPC_STL ra, ACCOUNT_STARTTIME_USER(ptr); \ 42 - subf rb,rb,ra; /* subtract start value */ \ 43 - PPC_LL ra, ACCOUNT_SYSTEM_TIME(ptr); \ 44 - add ra,ra,rb; /* add on to system time */ \ 45 - PPC_STL ra, ACCOUNT_SYSTEM_TIME(ptr) 46 - #endif /* CONFIG_VIRT_CPU_ACCOUNTING_NATIVE */ 47 - 48 - /* 49 19 * Macros for storing registers into and loading registers from 50 20 * exception frames. 51 21 */

+4 -5

arch/powerpc/include/asm/processor.h

··· 144 144 #endif 145 145 #ifdef CONFIG_PPC32 146 146 void *pgdir; /* root of page-table tree */ 147 - unsigned long ksp_limit; /* if ksp <= ksp_limit stack overflow */ 148 147 #ifdef CONFIG_PPC_RTAS 149 148 unsigned long rtas_sp; /* stack pointer for when in RTAS */ 150 - #endif 151 149 #endif 152 150 #if defined(CONFIG_PPC_BOOK3S_32) && defined(CONFIG_PPC_KUAP) 153 151 unsigned long kuap; /* opened segments for user access */ 154 152 #endif 155 - #ifdef CONFIG_VMAP_STACK 156 153 unsigned long srr0; 157 154 unsigned long srr1; 158 155 unsigned long dar; ··· 158 161 unsigned long r0, r3, r4, r5, r6, r8, r9, r11; 159 162 unsigned long lr, ctr; 160 163 #endif 161 - #endif 164 + #endif /* CONFIG_PPC32 */ 162 165 /* Debug Registers */ 163 166 struct debug_reg debug; 164 167 #ifdef CONFIG_PPC_FPU_REGS ··· 279 282 #ifdef CONFIG_PPC32 280 283 #define INIT_THREAD { \ 281 284 .ksp = INIT_SP, \ 282 - .ksp_limit = INIT_SP_LIMIT, \ 283 285 .pgdir = swapper_pg_dir, \ 284 286 .fpexc_mode = MSR_FE0 | MSR_FE1, \ 285 287 SPEFSCR_INIT \ ··· 389 393 extern unsigned long isa206_idle_insn_mayloss(unsigned long type); 390 394 #ifdef CONFIG_PPC_970_NAP 391 395 extern void power4_idle_nap(void); 396 + void power4_idle_nap_return(void); 392 397 #endif 393 398 394 399 extern unsigned long cpuidle_disable; ··· 413 416 */ 414 417 #define NET_IP_ALIGN 0 415 418 #endif 419 + 420 + int do_mathemu(struct pt_regs *regs); 416 421 417 422 #endif /* __KERNEL__ */ 418 423 #endif /* __ASSEMBLY__ */

+14 -31

arch/powerpc/include/asm/ptrace.h

··· 185 185 #define current_pt_regs() \ 186 186 ((struct pt_regs *)((unsigned long)task_stack_page(current) + THREAD_SIZE) - 1) 187 187 188 + /* 189 + * The 4 low bits (0xf) are available as flags to overload the trap word, 190 + * because interrupt vectors have minimum alignment of 0x10. TRAP_FLAGS_MASK 191 + * must cover the bits used as flags, including bit 0 which is used as the 192 + * "norestart" bit. 193 + */ 188 194 #ifdef __powerpc64__ 189 - #ifdef CONFIG_PPC_BOOK3S 190 - #define TRAP_FLAGS_MASK 0x10 191 - #define TRAP(regs) ((regs)->trap & ~TRAP_FLAGS_MASK) 192 - #define FULL_REGS(regs) true 193 - #define SET_FULL_REGS(regs) do { } while (0) 194 - #else 195 - #define TRAP_FLAGS_MASK 0x11 196 - #define TRAP(regs) ((regs)->trap & ~TRAP_FLAGS_MASK) 197 - #define FULL_REGS(regs) (((regs)->trap & 1) == 0) 198 - #define SET_FULL_REGS(regs) ((regs)->trap &= ~1) 199 - #endif 200 - #define CHECK_FULL_REGS(regs) BUG_ON(!FULL_REGS(regs)) 201 - #define NV_REG_POISON 0xdeadbeefdeadbeefUL 195 + #define TRAP_FLAGS_MASK 0x1 202 196 #else 203 197 /* 204 - * We use the least-significant bit of the trap field to indicate 205 - * whether we have saved the full set of registers, or only a 206 - * partial set. A 1 there means the partial set. 207 - * On 4xx we use the next bit to indicate whether the exception 198 + * On 4xx we use bit 1 in the trap word to indicate whether the exception 208 199 * is a critical exception (1 means it is). 209 200 */ 210 - #define TRAP_FLAGS_MASK 0x1F 211 - #define TRAP(regs) ((regs)->trap & ~TRAP_FLAGS_MASK) 212 - #define FULL_REGS(regs) (((regs)->trap & 1) == 0) 213 - #define SET_FULL_REGS(regs) ((regs)->trap &= ~1) 201 + #define TRAP_FLAGS_MASK 0xf 214 202 #define IS_CRITICAL_EXC(regs) (((regs)->trap & 2) != 0) 215 203 #define IS_MCHECK_EXC(regs) (((regs)->trap & 4) != 0) 216 204 #define IS_DEBUG_EXC(regs) (((regs)->trap & 8) != 0) 217 - #define NV_REG_POISON 0xdeadbeef 218 - #define CHECK_FULL_REGS(regs) \ 219 - do { \ 220 - if ((regs)->trap & 1) \ 221 - printk(KERN_CRIT "%s: partial register set\n", __func__); \ 222 - } while (0) 223 205 #endif /* __powerpc64__ */ 206 + #define TRAP(regs) ((regs)->trap & ~TRAP_FLAGS_MASK) 224 207 225 - static inline void set_trap(struct pt_regs *regs, unsigned long val) 208 + static __always_inline void set_trap(struct pt_regs *regs, unsigned long val) 226 209 { 227 210 regs->trap = (regs->trap & TRAP_FLAGS_MASK) | (val & ~TRAP_FLAGS_MASK); 228 211 } ··· 227 244 228 245 static inline bool trap_norestart(struct pt_regs *regs) 229 246 { 230 - return regs->trap & 0x10; 247 + return regs->trap & 0x1; 231 248 } 232 249 233 - static inline void set_trap_norestart(struct pt_regs *regs) 250 + static __always_inline void set_trap_norestart(struct pt_regs *regs) 234 251 { 235 - regs->trap |= 0x10; 252 + regs->trap |= 0x1; 236 253 } 237 254 238 255 #define arch_has_single_step() (1)

+7 -14

arch/powerpc/include/asm/qspinlock.h

··· 44 44 } 45 45 #define queued_spin_lock queued_spin_lock 46 46 47 - #define smp_mb__after_spinlock() smp_mb() 48 - 49 - static __always_inline int queued_spin_is_locked(struct qspinlock *lock) 50 - { 51 - /* 52 - * This barrier was added to simple spinlocks by commit 51d7d5205d338, 53 - * but it should now be possible to remove it, asm arm64 has done with 54 - * commit c6f5d02b6a0f. 55 - */ 56 - smp_mb(); 57 - return atomic_read(&lock->val); 58 - } 59 - #define queued_spin_is_locked queued_spin_is_locked 60 - 61 47 #ifdef CONFIG_PARAVIRT_SPINLOCKS 62 48 #define SPIN_THRESHOLD (1<<15) /* not tuned */ 63 49 ··· 71 85 } 72 86 73 87 #endif 88 + 89 + /* 90 + * Queued spinlocks rely heavily on smp_cond_load_relaxed() to busy-wait, 91 + * which was found to have performance problems if implemented with 92 + * the preferred spin_begin()/spin_end() SMT priority pattern. Use the 93 + * generic version instead. 94 + */ 74 95 75 96 #include <asm-generic/qspinlock.h> 76 97

+3 -3

arch/powerpc/include/asm/reg.h

··· 124 124 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM 125 125 #define MSR_TM_ACTIVE(x) (((x) & MSR_TS_MASK) != 0) /* Transaction active? */ 126 126 #else 127 - #define MSR_TM_ACTIVE(x) 0 127 + #define MSR_TM_ACTIVE(x) ((void)(x), 0) 128 128 #endif 129 129 130 130 #if defined(CONFIG_PPC_BOOK3S_64) ··· 441 441 #define LPCR_VRMA_LP1 ASM_CONST(0x0000800000000000) 442 442 #define LPCR_RMLS 0x1C000000 /* Implementation dependent RMO limit sel */ 443 443 #define LPCR_RMLS_SH 26 444 + #define LPCR_HAIL ASM_CONST(0x0000000004000000) /* HV AIL (ISAv3.1) */ 444 445 #define LPCR_ILE ASM_CONST(0x0000000002000000) /* !HV irqs set MSR:LE */ 445 446 #define LPCR_AIL ASM_CONST(0x0000000001800000) /* Alternate interrupt location */ 446 447 #define LPCR_AIL_0 ASM_CONST(0x0000000000000000) /* MMU off exception offset 0x0 */ ··· 1394 1393 : "r" ((unsigned long)(v)) \ 1395 1394 : "memory") 1396 1395 #endif 1397 - #define wrtspr(rn) asm volatile("mtspr " __stringify(rn) ",0" : \ 1398 - : : "memory") 1396 + #define wrtspr(rn) asm volatile("mtspr " __stringify(rn) ",2" : : : "memory") 1399 1397 1400 1398 static inline void wrtee(unsigned long val) 1401 1399 {

+3 -3

arch/powerpc/include/asm/rtas.h

··· 19 19 #define RTAS_UNKNOWN_SERVICE (-1) 20 20 #define RTAS_INSTANTIATE_MAX (1ULL<<30) /* Don't instantiate rtas at/above this value */ 21 21 22 - /* Buffer size for ppc_rtas system call. */ 23 - #define RTAS_RMOBUF_MAX (64 * 1024) 22 + /* Memory set aside for sys_rtas to use with calls that need a work area. */ 23 + #define RTAS_USER_REGION_SIZE (64 * 1024) 24 24 25 25 /* RTAS return status codes */ 26 26 #define RTAS_BUSY -2 /* RTAS Busy */ ··· 357 357 static inline int page_is_rtas_user_buf(unsigned long pfn) 358 358 { 359 359 unsigned long paddr = (pfn << PAGE_SHIFT); 360 - if (paddr >= rtas_rmo_buf && paddr < (rtas_rmo_buf + RTAS_RMOBUF_MAX)) 360 + if (paddr >= rtas_rmo_buf && paddr < (rtas_rmo_buf + RTAS_USER_REGION_SIZE)) 361 361 return 1; 362 362 return 0; 363 363 }

+1 -5

arch/powerpc/include/asm/simple_spinlock.h

··· 38 38 39 39 static inline int arch_spin_is_locked(arch_spinlock_t *lock) 40 40 { 41 - smp_mb(); 42 - return !arch_spin_value_unlocked(*lock); 41 + return !arch_spin_value_unlocked(READ_ONCE(*lock)); 43 42 } 44 43 45 44 /* ··· 280 281 #define arch_spin_relax(lock) spin_yield(lock) 281 282 #define arch_read_relax(lock) rw_yield(lock) 282 283 #define arch_write_relax(lock) rw_yield(lock) 283 - 284 - /* See include/linux/spinlock.h */ 285 - #define smp_mb__after_spinlock() smp_mb() 286 284 287 285 #endif /* _ASM_POWERPC_SIMPLE_SPINLOCK_H */

+6

arch/powerpc/include/asm/smp.h

··· 31 31 extern bool coregroup_enabled; 32 32 33 33 extern int cpu_to_chip_id(int cpu); 34 + extern int *chip_id_lookup_table; 34 35 35 36 #ifdef CONFIG_SMP 36 37 ··· 120 119 static inline struct cpumask *cpu_sibling_mask(int cpu) 121 120 { 122 121 return per_cpu(cpu_sibling_map, cpu); 122 + } 123 + 124 + static inline struct cpumask *cpu_core_mask(int cpu) 125 + { 126 + return per_cpu(cpu_core_map, cpu); 123 127 } 124 128 125 129 static inline struct cpumask *cpu_l2_cache_mask(int cpu)

+3

arch/powerpc/include/asm/spinlock.h

··· 10 10 #include <asm/simple_spinlock.h> 11 11 #endif 12 12 13 + /* See include/linux/spinlock.h */ 14 + #define smp_mb__after_spinlock() smp_mb() 15 + 13 16 #ifndef CONFIG_PARAVIRT_SPINLOCKS 14 17 static inline void pv_spinlocks_init(void) { } 15 18 #endif

+6 -1

arch/powerpc/include/asm/thread_info.h

··· 38 38 #ifndef __ASSEMBLY__ 39 39 #include <linux/cache.h> 40 40 #include <asm/processor.h> 41 - #include <asm/page.h> 42 41 #include <asm/accounting.h> 43 42 44 43 #define SLB_PRELOAD_NR 16U ··· 150 151 #define _TLF_RUNLATCH (1 << TLF_RUNLATCH) 151 152 152 153 #ifndef __ASSEMBLY__ 154 + 155 + static inline void clear_thread_local_flags(unsigned int flags) 156 + { 157 + struct thread_info *ti = current_thread_info(); 158 + ti->local_flags &= ~flags; 159 + } 153 160 154 161 static inline bool test_thread_local_flags(unsigned int flags) 155 162 {

+1 -1

arch/powerpc/include/asm/topology.h

··· 126 126 #define topology_physical_package_id(cpu) (cpu_to_chip_id(cpu)) 127 127 128 128 #define topology_sibling_cpumask(cpu) (per_cpu(cpu_sibling_map, cpu)) 129 - #define topology_core_cpumask(cpu) (cpu_cpu_mask(cpu)) 129 + #define topology_core_cpumask(cpu) (per_cpu(cpu_core_map, cpu)) 130 130 #define topology_core_id(cpu) (cpu_to_core_id(cpu)) 131 131 132 132 #endif

+172 -219

arch/powerpc/include/asm/uaccess.h

··· 43 43 * exception handling means that it's no longer "just"...) 44 44 * 45 45 */ 46 - #define get_user(x, ptr) \ 47 - __get_user_check((x), (ptr), sizeof(*(ptr))) 48 - #define put_user(x, ptr) \ 49 - __put_user_check((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr))) 50 - 51 - #define __get_user(x, ptr) \ 52 - __get_user_nocheck((x), (ptr), sizeof(*(ptr)), true) 53 - #define __put_user(x, ptr) \ 54 - __put_user_nocheck((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr))) 55 - 56 - #define __get_user_allowed(x, ptr) \ 57 - __get_user_nocheck((x), (ptr), sizeof(*(ptr)), false) 58 - 59 - #define __get_user_inatomic(x, ptr) \ 60 - __get_user_nosleep((x), (ptr), sizeof(*(ptr))) 61 - #define __put_user_inatomic(x, ptr) \ 62 - __put_user_nosleep((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr))) 63 - 64 - #ifdef CONFIG_PPC64 65 - 66 - #define ___get_user_instr(gu_op, dest, ptr) \ 67 - ({ \ 68 - long __gui_ret = 0; \ 69 - unsigned long __gui_ptr = (unsigned long)ptr; \ 70 - struct ppc_inst __gui_inst; \ 71 - unsigned int __prefix, __suffix; \ 72 - __gui_ret = gu_op(__prefix, (unsigned int __user *)__gui_ptr); \ 73 - if (__gui_ret == 0) { \ 74 - if ((__prefix >> 26) == OP_PREFIX) { \ 75 - __gui_ret = gu_op(__suffix, \ 76 - (unsigned int __user *)__gui_ptr + 1); \ 77 - __gui_inst = ppc_inst_prefix(__prefix, \ 78 - __suffix); \ 79 - } else { \ 80 - __gui_inst = ppc_inst(__prefix); \ 81 - } \ 82 - if (__gui_ret == 0) \ 83 - (dest) = __gui_inst; \ 84 - } \ 85 - __gui_ret; \ 86 - }) 87 - 88 - #define get_user_instr(x, ptr) \ 89 - ___get_user_instr(get_user, x, ptr) 90 - 91 - #define __get_user_instr(x, ptr) \ 92 - ___get_user_instr(__get_user, x, ptr) 93 - 94 - #define __get_user_instr_inatomic(x, ptr) \ 95 - ___get_user_instr(__get_user_inatomic, x, ptr) 96 - 97 - #else /* !CONFIG_PPC64 */ 98 - #define get_user_instr(x, ptr) \ 99 - get_user((x).val, (u32 __user *)(ptr)) 100 - 101 - #define __get_user_instr(x, ptr) \ 102 - __get_user_nocheck((x).val, (u32 __user *)(ptr), sizeof(u32), true) 103 - 104 - #define __get_user_instr_inatomic(x, ptr) \ 105 - __get_user_nosleep((x).val, (u32 __user *)(ptr), sizeof(u32)) 106 - 107 - #endif /* CONFIG_PPC64 */ 108 - 109 - extern long __put_user_bad(void); 110 - 111 - #define __put_user_size(x, ptr, size, retval) \ 112 - do { \ 113 - __label__ __pu_failed; \ 46 + #define __put_user(x, ptr) \ 47 + ({ \ 48 + long __pu_err; \ 49 + __typeof__(*(ptr)) __user *__pu_addr = (ptr); \ 50 + __typeof__(*(ptr)) __pu_val = (__typeof__(*(ptr)))(x); \ 51 + __typeof__(sizeof(*(ptr))) __pu_size = sizeof(*(ptr)); \ 114 52 \ 115 - retval = 0; \ 116 - allow_write_to_user(ptr, size); \ 117 - __put_user_size_goto(x, ptr, size, __pu_failed); \ 118 - prevent_write_to_user(ptr, size); \ 119 - break; \ 53 + might_fault(); \ 54 + do { \ 55 + __label__ __pu_failed; \ 56 + \ 57 + allow_write_to_user(__pu_addr, __pu_size); \ 58 + __put_user_size_goto(__pu_val, __pu_addr, __pu_size, __pu_failed); \ 59 + prevent_write_to_user(__pu_addr, __pu_size); \ 60 + __pu_err = 0; \ 61 + break; \ 120 62 \ 121 63 __pu_failed: \ 122 - retval = -EFAULT; \ 123 - prevent_write_to_user(ptr, size); \ 124 - } while (0) 125 - 126 - #define __put_user_nocheck(x, ptr, size) \ 127 - ({ \ 128 - long __pu_err; \ 129 - __typeof__(*(ptr)) __user *__pu_addr = (ptr); \ 130 - __typeof__(*(ptr)) __pu_val = (x); \ 131 - __typeof__(size) __pu_size = (size); \ 132 - \ 133 - if (!is_kernel_addr((unsigned long)__pu_addr)) \ 134 - might_fault(); \ 135 - __chk_user_ptr(__pu_addr); \ 136 - __put_user_size(__pu_val, __pu_addr, __pu_size, __pu_err); \ 64 + prevent_write_to_user(__pu_addr, __pu_size); \ 65 + __pu_err = -EFAULT; \ 66 + } while (0); \ 137 67 \ 138 68 __pu_err; \ 139 69 }) 140 70 141 - #define __put_user_check(x, ptr, size) \ 71 + #define put_user(x, ptr) \ 142 72 ({ \ 143 - long __pu_err = -EFAULT; \ 144 - __typeof__(*(ptr)) __user *__pu_addr = (ptr); \ 145 - __typeof__(*(ptr)) __pu_val = (x); \ 146 - __typeof__(size) __pu_size = (size); \ 73 + __typeof__(*(ptr)) __user *_pu_addr = (ptr); \ 147 74 \ 148 - might_fault(); \ 149 - if (access_ok(__pu_addr, __pu_size)) \ 150 - __put_user_size(__pu_val, __pu_addr, __pu_size, __pu_err); \ 151 - \ 152 - __pu_err; \ 75 + access_ok(_pu_addr, sizeof(*(ptr))) ? \ 76 + __put_user(x, _pu_addr) : -EFAULT; \ 153 77 }) 154 - 155 - #define __put_user_nosleep(x, ptr, size) \ 156 - ({ \ 157 - long __pu_err; \ 158 - __typeof__(*(ptr)) __user *__pu_addr = (ptr); \ 159 - __typeof__(*(ptr)) __pu_val = (x); \ 160 - __typeof__(size) __pu_size = (size); \ 161 - \ 162 - __chk_user_ptr(__pu_addr); \ 163 - __put_user_size(__pu_val, __pu_addr, __pu_size, __pu_err); \ 164 - \ 165 - __pu_err; \ 166 - }) 167 - 168 78 169 79 /* 170 80 * We don't tell gcc that we are accessing memory, but this is OK ··· 108 198 109 199 #define __put_user_size_goto(x, ptr, size, label) \ 110 200 do { \ 201 + __typeof__(*(ptr)) __user *__pus_addr = (ptr); \ 202 + \ 111 203 switch (size) { \ 112 - case 1: __put_user_asm_goto(x, ptr, label, "stb"); break; \ 113 - case 2: __put_user_asm_goto(x, ptr, label, "sth"); break; \ 114 - case 4: __put_user_asm_goto(x, ptr, label, "stw"); break; \ 115 - case 8: __put_user_asm2_goto(x, ptr, label); break; \ 116 - default: __put_user_bad(); \ 204 + case 1: __put_user_asm_goto(x, __pus_addr, label, "stb"); break; \ 205 + case 2: __put_user_asm_goto(x, __pus_addr, label, "sth"); break; \ 206 + case 4: __put_user_asm_goto(x, __pus_addr, label, "stw"); break; \ 207 + case 8: __put_user_asm2_goto(x, __pus_addr, label); break; \ 208 + default: BUILD_BUG(); \ 117 209 } \ 118 210 } while (0) 119 - 120 - #define __unsafe_put_user_goto(x, ptr, size, label) \ 121 - do { \ 122 - __typeof__(*(ptr)) __user *__pu_addr = (ptr); \ 123 - __chk_user_ptr(ptr); \ 124 - __put_user_size_goto((x), __pu_addr, (size), label); \ 125 - } while (0) 126 - 127 - 128 - extern long __get_user_bad(void); 129 211 130 212 /* 131 213 * This does an atomic 128 byte aligned load from userspace. ··· 135 233 EX_TABLE(1b, 3b) \ 136 234 : "=r" (err) \ 137 235 : "b" (uaddr), "b" (kaddr), "i" (-EFAULT), "0" (err)) 236 + 237 + #ifdef CONFIG_CC_HAS_ASM_GOTO_OUTPUT 238 + 239 + #define __get_user_asm_goto(x, addr, label, op) \ 240 + asm_volatile_goto( \ 241 + "1: "op"%U1%X1 %0, %1 # get_user\n" \ 242 + EX_TABLE(1b, %l2) \ 243 + : "=r" (x) \ 244 + : "m"UPD_CONSTR (*addr) \ 245 + : \ 246 + : label) 247 + 248 + #ifdef __powerpc64__ 249 + #define __get_user_asm2_goto(x, addr, label) \ 250 + __get_user_asm_goto(x, addr, label, "ld") 251 + #else /* __powerpc64__ */ 252 + #define __get_user_asm2_goto(x, addr, label) \ 253 + asm_volatile_goto( \ 254 + "1: lwz%X1 %0, %1\n" \ 255 + "2: lwz%X1 %L0, %L1\n" \ 256 + EX_TABLE(1b, %l2) \ 257 + EX_TABLE(2b, %l2) \ 258 + : "=r" (x) \ 259 + : "m" (*addr) \ 260 + : \ 261 + : label) 262 + #endif /* __powerpc64__ */ 263 + 264 + #define __get_user_size_goto(x, ptr, size, label) \ 265 + do { \ 266 + BUILD_BUG_ON(size > sizeof(x)); \ 267 + switch (size) { \ 268 + case 1: __get_user_asm_goto(x, (u8 __user *)ptr, label, "lbz"); break; \ 269 + case 2: __get_user_asm_goto(x, (u16 __user *)ptr, label, "lhz"); break; \ 270 + case 4: __get_user_asm_goto(x, (u32 __user *)ptr, label, "lwz"); break; \ 271 + case 8: __get_user_asm2_goto(x, (u64 __user *)ptr, label); break; \ 272 + default: x = 0; BUILD_BUG(); \ 273 + } \ 274 + } while (0) 275 + 276 + #define __get_user_size_allowed(x, ptr, size, retval) \ 277 + do { \ 278 + __label__ __gus_failed; \ 279 + \ 280 + __get_user_size_goto(x, ptr, size, __gus_failed); \ 281 + retval = 0; \ 282 + break; \ 283 + __gus_failed: \ 284 + x = 0; \ 285 + retval = -EFAULT; \ 286 + } while (0) 287 + 288 + #else /* CONFIG_CC_HAS_ASM_GOTO_OUTPUT */ 138 289 139 290 #define __get_user_asm(x, addr, err, op) \ 140 291 __asm__ __volatile__( \ ··· 226 271 #define __get_user_size_allowed(x, ptr, size, retval) \ 227 272 do { \ 228 273 retval = 0; \ 229 - __chk_user_ptr(ptr); \ 230 - if (size > sizeof(x)) \ 231 - (x) = __get_user_bad(); \ 274 + BUILD_BUG_ON(size > sizeof(x)); \ 232 275 switch (size) { \ 233 276 case 1: __get_user_asm(x, (u8 __user *)ptr, retval, "lbz"); break; \ 234 277 case 2: __get_user_asm(x, (u16 __user *)ptr, retval, "lhz"); break; \ 235 278 case 4: __get_user_asm(x, (u32 __user *)ptr, retval, "lwz"); break; \ 236 279 case 8: __get_user_asm2(x, (u64 __user *)ptr, retval); break; \ 237 - default: (x) = __get_user_bad(); \ 280 + default: x = 0; BUILD_BUG(); \ 238 281 } \ 239 282 } while (0) 240 283 241 - #define __get_user_size(x, ptr, size, retval) \ 284 + #define __get_user_size_goto(x, ptr, size, label) \ 242 285 do { \ 243 - allow_read_from_user(ptr, size); \ 244 - __get_user_size_allowed(x, ptr, size, retval); \ 245 - prevent_read_from_user(ptr, size); \ 286 + long __gus_retval; \ 287 + \ 288 + __get_user_size_allowed(x, ptr, size, __gus_retval); \ 289 + if (__gus_retval) \ 290 + goto label; \ 246 291 } while (0) 292 + 293 + #endif /* CONFIG_CC_HAS_ASM_GOTO_OUTPUT */ 247 294 248 295 /* 249 296 * This is a type: either unsigned long, if the argument fits into ··· 254 297 #define __long_type(x) \ 255 298 __typeof__(__builtin_choose_expr(sizeof(x) > sizeof(0UL), 0ULL, 0UL)) 256 299 257 - #define __get_user_nocheck(x, ptr, size, do_allow) \ 300 + #define __get_user(x, ptr) \ 258 301 ({ \ 259 302 long __gu_err; \ 260 303 __long_type(*(ptr)) __gu_val; \ 261 304 __typeof__(*(ptr)) __user *__gu_addr = (ptr); \ 262 - __typeof__(size) __gu_size = (size); \ 305 + __typeof__(sizeof(*(ptr))) __gu_size = sizeof(*(ptr)); \ 263 306 \ 264 - __chk_user_ptr(__gu_addr); \ 265 - if (do_allow && !is_kernel_addr((unsigned long)__gu_addr)) \ 266 - might_fault(); \ 267 - if (do_allow) \ 268 - __get_user_size(__gu_val, __gu_addr, __gu_size, __gu_err); \ 269 - else \ 270 - __get_user_size_allowed(__gu_val, __gu_addr, __gu_size, __gu_err); \ 307 + might_fault(); \ 308 + allow_read_from_user(__gu_addr, __gu_size); \ 309 + __get_user_size_allowed(__gu_val, __gu_addr, __gu_size, __gu_err); \ 310 + prevent_read_from_user(__gu_addr, __gu_size); \ 271 311 (x) = (__typeof__(*(ptr)))__gu_val; \ 272 312 \ 273 313 __gu_err; \ 274 314 }) 275 315 276 - #define __get_user_check(x, ptr, size) \ 316 + #define get_user(x, ptr) \ 277 317 ({ \ 278 - long __gu_err = -EFAULT; \ 279 - __long_type(*(ptr)) __gu_val = 0; \ 280 - __typeof__(*(ptr)) __user *__gu_addr = (ptr); \ 281 - __typeof__(size) __gu_size = (size); \ 318 + __typeof__(*(ptr)) __user *_gu_addr = (ptr); \ 282 319 \ 283 - might_fault(); \ 284 - if (access_ok(__gu_addr, __gu_size)) \ 285 - __get_user_size(__gu_val, __gu_addr, __gu_size, __gu_err); \ 286 - (x) = (__force __typeof__(*(ptr)))__gu_val; \ 287 - \ 288 - __gu_err; \ 320 + access_ok(_gu_addr, sizeof(*(ptr))) ? \ 321 + __get_user(x, _gu_addr) : \ 322 + ((x) = (__force __typeof__(*(ptr)))0, -EFAULT); \ 289 323 }) 290 - 291 - #define __get_user_nosleep(x, ptr, size) \ 292 - ({ \ 293 - long __gu_err; \ 294 - __long_type(*(ptr)) __gu_val; \ 295 - __typeof__(*(ptr)) __user *__gu_addr = (ptr); \ 296 - __typeof__(size) __gu_size = (size); \ 297 - \ 298 - __chk_user_ptr(__gu_addr); \ 299 - __get_user_size(__gu_val, __gu_addr, __gu_size, __gu_err); \ 300 - (x) = (__force __typeof__(*(ptr)))__gu_val; \ 301 - \ 302 - __gu_err; \ 303 - }) 304 - 305 324 306 325 /* more complex routines */ 307 326 308 327 extern unsigned long __copy_tofrom_user(void __user *to, 309 328 const void __user *from, unsigned long size); 310 - 311 - #ifdef CONFIG_ARCH_HAS_COPY_MC 312 - unsigned long __must_check 313 - copy_mc_generic(void *to, const void *from, unsigned long size); 314 - 315 - static inline unsigned long __must_check 316 - copy_mc_to_kernel(void *to, const void *from, unsigned long size) 317 - { 318 - return copy_mc_generic(to, from, size); 319 - } 320 - #define copy_mc_to_kernel copy_mc_to_kernel 321 - 322 - static inline unsigned long __must_check 323 - copy_mc_to_user(void __user *to, const void *from, unsigned long n) 324 - { 325 - if (likely(check_copy_size(from, n, true))) { 326 - if (access_ok(to, n)) { 327 - allow_write_to_user(to, n); 328 - n = copy_mc_generic((void *)to, from, n); 329 - prevent_write_to_user(to, n); 330 - } 331 - } 332 - 333 - return n; 334 - } 335 - #endif 336 329 337 330 #ifdef __powerpc64__ 338 331 static inline unsigned long ··· 321 414 322 415 unsigned long __arch_clear_user(void __user *addr, unsigned long size); 323 416 324 - static inline unsigned long clear_user(void __user *addr, unsigned long size) 417 + static inline unsigned long __clear_user(void __user *addr, unsigned long size) 325 418 { 326 - unsigned long ret = size; 419 + unsigned long ret; 420 + 327 421 might_fault(); 328 - if (likely(access_ok(addr, size))) { 329 - allow_write_to_user(addr, size); 330 - ret = __arch_clear_user(addr, size); 331 - prevent_write_to_user(addr, size); 332 - } 422 + allow_write_to_user(addr, size); 423 + ret = __arch_clear_user(addr, size); 424 + prevent_write_to_user(addr, size); 333 425 return ret; 334 426 } 335 427 336 - static inline unsigned long __clear_user(void __user *addr, unsigned long size) 428 + static inline unsigned long clear_user(void __user *addr, unsigned long size) 337 429 { 338 - return clear_user(addr, size); 430 + return likely(access_ok(addr, size)) ? __clear_user(addr, size) : size; 339 431 } 340 432 341 433 extern long strncpy_from_user(char *dst, const char __user *src, long count); 342 434 extern __must_check long strnlen_user(const char __user *str, long n); 435 + 436 + #ifdef CONFIG_ARCH_HAS_COPY_MC 437 + unsigned long __must_check 438 + copy_mc_generic(void *to, const void *from, unsigned long size); 439 + 440 + static inline unsigned long __must_check 441 + copy_mc_to_kernel(void *to, const void *from, unsigned long size) 442 + { 443 + return copy_mc_generic(to, from, size); 444 + } 445 + #define copy_mc_to_kernel copy_mc_to_kernel 446 + 447 + static inline unsigned long __must_check 448 + copy_mc_to_user(void __user *to, const void *from, unsigned long n) 449 + { 450 + if (likely(check_copy_size(from, n, true))) { 451 + if (access_ok(to, n)) { 452 + allow_write_to_user(to, n); 453 + n = copy_mc_generic((void *)to, from, n); 454 + prevent_write_to_user(to, n); 455 + } 456 + } 457 + 458 + return n; 459 + } 460 + #endif 343 461 344 462 extern long __copy_from_user_flushcache(void *dst, const void __user *src, 345 463 unsigned size); ··· 414 482 #define user_write_access_begin user_write_access_begin 415 483 #define user_write_access_end prevent_current_write_to_user 416 484 417 - #define unsafe_op_wrap(op, err) do { if (unlikely(op)) goto err; } while (0) 418 - #define unsafe_get_user(x, p, e) unsafe_op_wrap(__get_user_allowed(x, p), e) 485 + #define unsafe_get_user(x, p, e) do { \ 486 + __long_type(*(p)) __gu_val; \ 487 + __typeof__(*(p)) __user *__gu_addr = (p); \ 488 + \ 489 + __get_user_size_goto(__gu_val, __gu_addr, sizeof(*(p)), e); \ 490 + (x) = (__typeof__(*(p)))__gu_val; \ 491 + } while (0) 492 + 419 493 #define unsafe_put_user(x, p, e) \ 420 - __unsafe_put_user_goto((__typeof__(*(p)))(x), (p), sizeof(*(p)), e) 494 + __put_user_size_goto((__typeof__(*(p)))(x), (p), sizeof(*(p)), e) 495 + 496 + #define unsafe_copy_from_user(d, s, l, e) \ 497 + do { \ 498 + u8 *_dst = (u8 *)(d); \ 499 + const u8 __user *_src = (const u8 __user *)(s); \ 500 + size_t _len = (l); \ 501 + int _i; \ 502 + \ 503 + for (_i = 0; _i < (_len & ~(sizeof(u64) - 1)); _i += sizeof(u64)) \ 504 + unsafe_get_user(*(u64 *)(_dst + _i), (u64 __user *)(_src + _i), e); \ 505 + if (_len & 4) { \ 506 + unsafe_get_user(*(u32 *)(_dst + _i), (u32 __user *)(_src + _i), e); \ 507 + _i += 4; \ 508 + } \ 509 + if (_len & 2) { \ 510 + unsafe_get_user(*(u16 *)(_dst + _i), (u16 __user *)(_src + _i), e); \ 511 + _i += 2; \ 512 + } \ 513 + if (_len & 1) \ 514 + unsafe_get_user(*(u8 *)(_dst + _i), (u8 __user *)(_src + _i), e); \ 515 + } while (0) 421 516 422 517 #define unsafe_copy_to_user(d, s, l, e) \ 423 518 do { \ ··· 453 494 size_t _len = (l); \ 454 495 int _i; \ 455 496 \ 456 - for (_i = 0; _i < (_len & ~(sizeof(long) - 1)); _i += sizeof(long)) \ 457 - unsafe_put_user(*(long*)(_src + _i), (long __user *)(_dst + _i), e); \ 458 - if (IS_ENABLED(CONFIG_PPC64) && (_len & 4)) { \ 497 + for (_i = 0; _i < (_len & ~(sizeof(u64) - 1)); _i += sizeof(u64)) \ 498 + unsafe_put_user(*(u64 *)(_src + _i), (u64 __user *)(_dst + _i), e); \ 499 + if (_len & 4) { \ 459 500 unsafe_put_user(*(u32*)(_src + _i), (u32 __user *)(_dst + _i), e); \ 460 501 _i += 4; \ 461 502 } \ ··· 470 511 #define HAVE_GET_KERNEL_NOFAULT 471 512 472 513 #define __get_kernel_nofault(dst, src, type, err_label) \ 473 - do { \ 474 - int __kr_err; \ 475 - \ 476 - __get_user_size_allowed(*((type *)(dst)), (__force type __user *)(src),\ 477 - sizeof(type), __kr_err); \ 478 - if (unlikely(__kr_err)) \ 479 - goto err_label; \ 480 - } while (0) 514 + __get_user_size_goto(*((type *)(dst)), \ 515 + (__force type __user *)(src), sizeof(type), err_label) 481 516 482 517 #define __put_kernel_nofault(dst, src, type, err_label) \ 483 518 __put_user_size_goto(*((type *)(src)), \

+1

arch/powerpc/include/asm/unistd.h

··· 40 40 #define __ARCH_WANT_SYS_SIGPROCMASK 41 41 #ifdef CONFIG_PPC32 42 42 #define __ARCH_WANT_OLD_STAT 43 + #define __ARCH_WANT_SYS_OLD_SELECT 43 44 #endif 44 45 #ifdef CONFIG_PPC64 45 46 #define __ARCH_WANT_SYS_TIME

+10

arch/powerpc/include/asm/vdso/gettimeofday.h

··· 2 2 #ifndef _ASM_POWERPC_VDSO_GETTIMEOFDAY_H 3 3 #define _ASM_POWERPC_VDSO_GETTIMEOFDAY_H 4 4 5 + #include <asm/page.h> 6 + 5 7 #ifdef __ASSEMBLY__ 6 8 7 9 #include <asm/ppc_asm.h> ··· 155 153 } 156 154 157 155 const struct vdso_data *__arch_get_vdso_data(void); 156 + 157 + #ifdef CONFIG_TIME_NS 158 + static __always_inline 159 + const struct vdso_data *__arch_get_timens_vdso_data(const struct vdso_data *vd) 160 + { 161 + return (void *)vd + PAGE_SIZE; 162 + } 163 + #endif 158 164 159 165 static inline bool vdso_clocksource_ok(const struct vdso_data *vd) 160 166 {

-2

arch/powerpc/include/asm/vdso_datapage.h

··· 107 107 bcl 20, 31, .+4 108 108 999: 109 109 mflr \ptr 110 - #if CONFIG_PPC_PAGE_SHIFT > 14 111 110 addis \ptr, \ptr, (_vdso_datapage - 999b)@ha 112 - #endif 113 111 addi \ptr, \ptr, (_vdso_datapage - 999b)@l 114 112 .endm 115 113

+1

arch/powerpc/include/asm/vio.h

··· 114 114 const struct vio_device_id *id_table; 115 115 int (*probe)(struct vio_dev *dev, const struct vio_device_id *id); 116 116 void (*remove)(struct vio_dev *dev); 117 + void (*shutdown)(struct vio_dev *dev); 117 118 /* A driver must have a get_desired_dma() function to 118 119 * be loaded in a CMO environment if it uses DMA. 119 120 */

+1

arch/powerpc/include/asm/xive.h

··· 102 102 /* xmon hook */ 103 103 void xmon_xive_do_dump(int cpu); 104 104 int xmon_xive_get_irq_config(u32 hw_irq, struct irq_data *d); 105 + void xmon_xive_get_irq_all(void); 105 106 106 107 /* APIs used by KVM */ 107 108 u32 xive_native_default_eq_shift(void);

+1

arch/powerpc/include/uapi/asm/errno.h

··· 2 2 #ifndef _ASM_POWERPC_ERRNO_H 3 3 #define _ASM_POWERPC_ERRNO_H 4 4 5 + #undef EDEADLOCK 5 6 #include <asm-generic/errno.h> 6 7 7 8 #undef EDEADLOCK

-5

arch/powerpc/include/uapi/asm/posix_types.h

··· 12 12 typedef unsigned long __kernel_old_dev_t; 13 13 #define __kernel_old_dev_t __kernel_old_dev_t 14 14 #else 15 - typedef unsigned int __kernel_size_t; 16 - typedef int __kernel_ssize_t; 17 - typedef long __kernel_ptrdiff_t; 18 - #define __kernel_size_t __kernel_size_t 19 - 20 15 typedef short __kernel_ipc_pid_t; 21 16 #define __kernel_ipc_pid_t __kernel_ipc_pid_t 22 17 #endif

+38 -34

arch/powerpc/kernel/align.c

··· 107 107 static int emulate_spe(struct pt_regs *regs, unsigned int reg, 108 108 struct ppc_inst ppc_instr) 109 109 { 110 - int ret; 111 110 union { 112 111 u64 ll; 113 112 u32 w[2]; ··· 125 126 126 127 nb = spe_aligninfo[instr].len; 127 128 flags = spe_aligninfo[instr].flags; 128 - 129 - /* Verify the address of the operand */ 130 - if (unlikely(user_mode(regs) && 131 - !access_ok(addr, nb))) 132 - return -EFAULT; 133 129 134 130 /* userland only */ 135 131 if (unlikely(!user_mode(regs))) ··· 163 169 } 164 170 } else { 165 171 temp.ll = data.ll = 0; 166 - ret = 0; 167 172 p = addr; 173 + 174 + if (!user_read_access_begin(addr, nb)) 175 + return -EFAULT; 168 176 169 177 switch (nb) { 170 178 case 8: 171 - ret |= __get_user_inatomic(temp.v[0], p++); 172 - ret |= __get_user_inatomic(temp.v[1], p++); 173 - ret |= __get_user_inatomic(temp.v[2], p++); 174 - ret |= __get_user_inatomic(temp.v[3], p++); 179 + unsafe_get_user(temp.v[0], p++, Efault_read); 180 + unsafe_get_user(temp.v[1], p++, Efault_read); 181 + unsafe_get_user(temp.v[2], p++, Efault_read); 182 + unsafe_get_user(temp.v[3], p++, Efault_read); 175 183 fallthrough; 176 184 case 4: 177 - ret |= __get_user_inatomic(temp.v[4], p++); 178 - ret |= __get_user_inatomic(temp.v[5], p++); 185 + unsafe_get_user(temp.v[4], p++, Efault_read); 186 + unsafe_get_user(temp.v[5], p++, Efault_read); 179 187 fallthrough; 180 188 case 2: 181 - ret |= __get_user_inatomic(temp.v[6], p++); 182 - ret |= __get_user_inatomic(temp.v[7], p++); 183 - if (unlikely(ret)) 184 - return -EFAULT; 189 + unsafe_get_user(temp.v[6], p++, Efault_read); 190 + unsafe_get_user(temp.v[7], p++, Efault_read); 185 191 } 192 + user_read_access_end(); 186 193 187 194 switch (instr) { 188 195 case EVLDD: ··· 250 255 251 256 /* Store result to memory or update registers */ 252 257 if (flags & ST) { 253 - ret = 0; 254 258 p = addr; 259 + 260 + if (!user_write_access_begin(addr, nb)) 261 + return -EFAULT; 262 + 255 263 switch (nb) { 256 264 case 8: 257 - ret |= __put_user_inatomic(data.v[0], p++); 258 - ret |= __put_user_inatomic(data.v[1], p++); 259 - ret |= __put_user_inatomic(data.v[2], p++); 260 - ret |= __put_user_inatomic(data.v[3], p++); 265 + unsafe_put_user(data.v[0], p++, Efault_write); 266 + unsafe_put_user(data.v[1], p++, Efault_write); 267 + unsafe_put_user(data.v[2], p++, Efault_write); 268 + unsafe_put_user(data.v[3], p++, Efault_write); 261 269 fallthrough; 262 270 case 4: 263 - ret |= __put_user_inatomic(data.v[4], p++); 264 - ret |= __put_user_inatomic(data.v[5], p++); 271 + unsafe_put_user(data.v[4], p++, Efault_write); 272 + unsafe_put_user(data.v[5], p++, Efault_write); 265 273 fallthrough; 266 274 case 2: 267 - ret |= __put_user_inatomic(data.v[6], p++); 268 - ret |= __put_user_inatomic(data.v[7], p++); 275 + unsafe_put_user(data.v[6], p++, Efault_write); 276 + unsafe_put_user(data.v[7], p++, Efault_write); 269 277 } 270 - if (unlikely(ret)) 271 - return -EFAULT; 278 + user_write_access_end(); 272 279 } else { 273 280 *evr = data.w[0]; 274 281 regs->gpr[reg] = data.w[1]; 275 282 } 276 283 277 284 return 1; 285 + 286 + Efault_read: 287 + user_read_access_end(); 288 + return -EFAULT; 289 + 290 + Efault_write: 291 + user_write_access_end(); 292 + return -EFAULT; 278 293 } 279 294 #endif /* CONFIG_SPE */ 280 295 ··· 304 299 struct instruction_op op; 305 300 int r, type; 306 301 307 - /* 308 - * We require a complete register set, if not, then our assembly 309 - * is broken 310 - */ 311 - CHECK_FULL_REGS(regs); 302 + if (is_kernel_addr(regs->nip)) 303 + r = copy_inst_from_kernel_nofault(&instr, (void *)regs->nip); 304 + else 305 + r = __get_user_instr(instr, (void __user *)regs->nip); 312 306 313 - if (unlikely(__get_user_instr(instr, (void __user *)regs->nip))) 307 + if (unlikely(r)) 314 308 return -EFAULT; 315 309 if ((regs->msr & MSR_LE) != (MSR_KERNEL & MSR_LE)) { 316 310 /* We don't handle PPC little-endian any more... */

-17

arch/powerpc/kernel/asm-offsets.c

··· 91 91 DEFINE(SIGSEGV, SIGSEGV); 92 92 DEFINE(NMI_MASK, NMI_MASK); 93 93 #else 94 - OFFSET(KSP_LIMIT, thread_struct, ksp_limit); 95 94 #ifdef CONFIG_PPC_RTAS 96 95 OFFSET(RTAS_SP, thread_struct, rtas_sp); 97 96 #endif ··· 131 132 OFFSET(KSP_VSID, thread_struct, ksp_vsid); 132 133 #else /* CONFIG_PPC64 */ 133 134 OFFSET(PGDIR, thread_struct, pgdir); 134 - #ifdef CONFIG_VMAP_STACK 135 135 OFFSET(SRR0, thread_struct, srr0); 136 136 OFFSET(SRR1, thread_struct, srr1); 137 137 OFFSET(DAR, thread_struct, dar); ··· 146 148 OFFSET(THR11, thread_struct, r11); 147 149 OFFSET(THLR, thread_struct, lr); 148 150 OFFSET(THCTR, thread_struct, ctr); 149 - #endif 150 151 #endif 151 152 #ifdef CONFIG_SPE 152 153 OFFSET(THREAD_EVR0, thread_struct, evr[0]); ··· 282 285 OFFSET(PACAHWCPUID, paca_struct, hw_cpu_id); 283 286 OFFSET(PACAKEXECSTATE, paca_struct, kexec_state); 284 287 OFFSET(PACA_DSCR_DEFAULT, paca_struct, dscr_default); 285 - OFFSET(ACCOUNT_STARTTIME, paca_struct, accounting.starttime); 286 - OFFSET(ACCOUNT_STARTTIME_USER, paca_struct, accounting.starttime_user); 287 - OFFSET(ACCOUNT_USER_TIME, paca_struct, accounting.utime); 288 - OFFSET(ACCOUNT_SYSTEM_TIME, paca_struct, accounting.stime); 289 288 #ifdef CONFIG_PPC_BOOK3E 290 289 OFFSET(PACA_TRAP_SAVE, paca_struct, trap_save); 291 290 #endif 292 291 OFFSET(PACA_SPRG_VDSO, paca_struct, sprg_vdso); 293 292 #else /* CONFIG_PPC64 */ 294 - #ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE 295 - OFFSET(ACCOUNT_STARTTIME, thread_info, accounting.starttime); 296 - OFFSET(ACCOUNT_STARTTIME_USER, thread_info, accounting.starttime_user); 297 - OFFSET(ACCOUNT_USER_TIME, thread_info, accounting.utime); 298 - OFFSET(ACCOUNT_SYSTEM_TIME, thread_info, accounting.stime); 299 - #endif 300 293 #endif /* CONFIG_PPC64 */ 301 294 302 295 /* RTAS */ ··· 310 323 STACK_PT_REGS_OFFSET(GPR11, gpr[11]); 311 324 STACK_PT_REGS_OFFSET(GPR12, gpr[12]); 312 325 STACK_PT_REGS_OFFSET(GPR13, gpr[13]); 313 - #ifndef CONFIG_PPC64 314 - STACK_PT_REGS_OFFSET(GPR14, gpr[14]); 315 - #endif /* CONFIG_PPC64 */ 316 326 /* 317 327 * Note: these symbols include _ because they overlap with special 318 328 * register names ··· 365 381 DEFINE(_CSRR1, STACK_INT_FRAME_SIZE+offsetof(struct exception_regs, csrr1)); 366 382 DEFINE(_DSRR0, STACK_INT_FRAME_SIZE+offsetof(struct exception_regs, dsrr0)); 367 383 DEFINE(_DSRR1, STACK_INT_FRAME_SIZE+offsetof(struct exception_regs, dsrr1)); 368 - DEFINE(SAVED_KSP_LIMIT, STACK_INT_FRAME_SIZE+offsetof(struct exception_regs, saved_ksp_limit)); 369 384 #endif 370 385 #endif 371 386

+7 -8

arch/powerpc/kernel/eeh.c

··· 362 362 pa = pte_pfn(*ptep); 363 363 364 364 /* On radix we can do hugepage mappings for io, so handle that */ 365 - if (hugepage_shift) { 366 - pa <<= hugepage_shift; 367 - pa |= token & ((1ul << hugepage_shift) - 1); 368 - } else { 369 - pa <<= PAGE_SHIFT; 370 - pa |= token & (PAGE_SIZE - 1); 371 - } 365 + if (!hugepage_shift) 366 + hugepage_shift = PAGE_SHIFT; 372 367 368 + pa <<= PAGE_SHIFT; 369 + pa |= token & ((1ul << hugepage_shift) - 1); 373 370 return pa; 374 371 } 375 372 ··· 776 779 default: 777 780 eeh_pe_state_clear(pe, EEH_PE_ISOLATED | EEH_PE_CFG_BLOCKED, true); 778 781 return -EINVAL; 779 - }; 782 + } 780 783 781 784 return 0; 782 785 } ··· 1565 1568 } 1566 1569 EXPORT_SYMBOL_GPL(eeh_pe_inject_err); 1567 1570 1571 + #ifdef CONFIG_PROC_FS 1568 1572 static int proc_eeh_show(struct seq_file *m, void *v) 1569 1573 { 1570 1574 if (!eeh_enabled()) { ··· 1592 1594 1593 1595 return 0; 1594 1596 } 1597 + #endif /* CONFIG_PROC_FS */ 1595 1598 1596 1599 #ifdef CONFIG_DEBUG_FS 1597 1600

+152 -674

arch/powerpc/kernel/entry_32.S

··· 48 48 */ 49 49 .align 12 50 50 51 - #ifdef CONFIG_BOOKE 52 - .globl mcheck_transfer_to_handler 53 - mcheck_transfer_to_handler: 54 - mfspr r0,SPRN_DSRR0 55 - stw r0,_DSRR0(r11) 56 - mfspr r0,SPRN_DSRR1 57 - stw r0,_DSRR1(r11) 58 - /* fall through */ 59 - _ASM_NOKPROBE_SYMBOL(mcheck_transfer_to_handler) 60 - 61 - .globl debug_transfer_to_handler 62 - debug_transfer_to_handler: 63 - mfspr r0,SPRN_CSRR0 64 - stw r0,_CSRR0(r11) 65 - mfspr r0,SPRN_CSRR1 66 - stw r0,_CSRR1(r11) 67 - /* fall through */ 68 - _ASM_NOKPROBE_SYMBOL(debug_transfer_to_handler) 69 - 70 - .globl crit_transfer_to_handler 71 - crit_transfer_to_handler: 72 - #ifdef CONFIG_PPC_BOOK3E_MMU 73 - mfspr r0,SPRN_MAS0 74 - stw r0,MAS0(r11) 75 - mfspr r0,SPRN_MAS1 76 - stw r0,MAS1(r11) 77 - mfspr r0,SPRN_MAS2 78 - stw r0,MAS2(r11) 79 - mfspr r0,SPRN_MAS3 80 - stw r0,MAS3(r11) 81 - mfspr r0,SPRN_MAS6 82 - stw r0,MAS6(r11) 83 - #ifdef CONFIG_PHYS_64BIT 84 - mfspr r0,SPRN_MAS7 85 - stw r0,MAS7(r11) 86 - #endif /* CONFIG_PHYS_64BIT */ 87 - #endif /* CONFIG_PPC_BOOK3E_MMU */ 88 - #ifdef CONFIG_44x 89 - mfspr r0,SPRN_MMUCR 90 - stw r0,MMUCR(r11) 91 - #endif 92 - mfspr r0,SPRN_SRR0 93 - stw r0,_SRR0(r11) 94 - mfspr r0,SPRN_SRR1 95 - stw r0,_SRR1(r11) 96 - 97 - /* set the stack limit to the current stack */ 98 - mfspr r8,SPRN_SPRG_THREAD 99 - lwz r0,KSP_LIMIT(r8) 100 - stw r0,SAVED_KSP_LIMIT(r11) 101 - rlwinm r0,r1,0,0,(31 - THREAD_SHIFT) 102 - stw r0,KSP_LIMIT(r8) 103 - /* fall through */ 104 - _ASM_NOKPROBE_SYMBOL(crit_transfer_to_handler) 105 - #endif 106 - 107 - #ifdef CONFIG_40x 108 - .globl crit_transfer_to_handler 109 - crit_transfer_to_handler: 110 - lwz r0,crit_r10@l(0) 111 - stw r0,GPR10(r11) 112 - lwz r0,crit_r11@l(0) 113 - stw r0,GPR11(r11) 114 - mfspr r0,SPRN_SRR0 115 - stw r0,crit_srr0@l(0) 116 - mfspr r0,SPRN_SRR1 117 - stw r0,crit_srr1@l(0) 118 - 119 - /* set the stack limit to the current stack */ 120 - mfspr r8,SPRN_SPRG_THREAD 121 - lwz r0,KSP_LIMIT(r8) 122 - stw r0,saved_ksp_limit@l(0) 123 - rlwinm r0,r1,0,0,(31 - THREAD_SHIFT) 124 - stw r0,KSP_LIMIT(r8) 125 - /* fall through */ 126 - _ASM_NOKPROBE_SYMBOL(crit_transfer_to_handler) 127 - #endif 128 - 129 - /* 130 - * This code finishes saving the registers to the exception frame 131 - * and jumps to the appropriate handler for the exception, turning 132 - * on address translation. 133 - * Note that we rely on the caller having set cr0.eq iff the exception 134 - * occurred in kernel mode (i.e. MSR:PR = 0). 135 - */ 136 - .globl transfer_to_handler_full 137 - transfer_to_handler_full: 138 - SAVE_NVGPRS(r11) 139 - _ASM_NOKPROBE_SYMBOL(transfer_to_handler_full) 140 - /* fall through */ 141 - 142 - .globl transfer_to_handler 143 - transfer_to_handler: 144 - stw r2,GPR2(r11) 145 - stw r12,_NIP(r11) 146 - stw r9,_MSR(r11) 147 - andi. r2,r9,MSR_PR 148 - mfctr r12 149 - mfspr r2,SPRN_XER 150 - stw r12,_CTR(r11) 151 - stw r2,_XER(r11) 152 - mfspr r12,SPRN_SPRG_THREAD 153 - tovirt_vmstack r12, r12 154 - beq 2f /* if from user, fix up THREAD.regs */ 155 - addi r2, r12, -THREAD 156 - addi r11,r1,STACK_FRAME_OVERHEAD 157 - stw r11,PT_REGS(r12) 158 - #if defined(CONFIG_40x) || defined(CONFIG_BOOKE) 159 - /* Check to see if the dbcr0 register is set up to debug. Use the 160 - internal debug mode bit to do this. */ 161 - lwz r12,THREAD_DBCR0(r12) 162 - andis. r12,r12,DBCR0_IDM@h 163 - #endif 164 - ACCOUNT_CPU_USER_ENTRY(r2, r11, r12) 165 - #ifdef CONFIG_PPC_BOOK3S_32 166 - kuep_lock r11, r12 167 - #endif 168 - #if defined(CONFIG_40x) || defined(CONFIG_BOOKE) 169 - beq+ 3f 170 - /* From user and task is ptraced - load up global dbcr0 */ 171 - li r12,-1 /* clear all pending debug events */ 172 - mtspr SPRN_DBSR,r12 173 - lis r11,global_dbcr0@ha 174 - tophys(r11,r11) 175 - addi r11,r11,global_dbcr0@l 176 - #ifdef CONFIG_SMP 177 - lwz r9,TASK_CPU(r2) 178 - slwi r9,r9,2 179 - add r11,r11,r9 180 - #endif 181 - lwz r12,0(r11) 182 - mtspr SPRN_DBCR0,r12 183 - #endif 184 - 185 - b 3f 186 - 187 - 2: /* if from kernel, check interrupted DOZE/NAP mode and 188 - * check for stack overflow 189 - */ 190 - kuap_save_and_lock r11, r12, r9, r2, r6 191 - addi r2, r12, -THREAD 192 - #ifndef CONFIG_VMAP_STACK 193 - lwz r9,KSP_LIMIT(r12) 194 - cmplw r1,r9 /* if r1 <= ksp_limit */ 195 - ble- stack_ovf /* then the kernel stack overflowed */ 196 - #endif 197 - 5: 198 51 #if defined(CONFIG_PPC_BOOK3S_32) || defined(CONFIG_E500) 52 + .globl prepare_transfer_to_handler 53 + prepare_transfer_to_handler: 54 + /* if from kernel, check interrupted DOZE/NAP mode */ 199 55 lwz r12,TI_LOCAL_FLAGS(r2) 200 56 mtcrf 0x01,r12 201 57 bt- 31-TLF_NAPPING,4f 202 58 bt- 31-TLF_SLEEPING,7f 203 - #endif /* CONFIG_PPC_BOOK3S_32 || CONFIG_E500 */ 204 - .globl transfer_to_handler_cont 205 - transfer_to_handler_cont: 206 - 3: 207 - mflr r9 208 - tovirt_novmstack r2, r2 /* set r2 to current */ 209 - tovirt_vmstack r9, r9 210 - lwz r11,0(r9) /* virtual address of handler */ 211 - lwz r9,4(r9) /* where to go when done */ 212 - #if defined(CONFIG_PPC_8xx) && defined(CONFIG_PERF_EVENTS) 213 - mtspr SPRN_NRI, r0 214 - #endif 215 - #ifdef CONFIG_TRACE_IRQFLAGS 216 - /* 217 - * When tracing IRQ state (lockdep) we enable the MMU before we call 218 - * the IRQ tracing functions as they might access vmalloc space or 219 - * perform IOs for console output. 220 - * 221 - * To speed up the syscall path where interrupts stay on, let's check 222 - * first if we are changing the MSR value at all. 223 - */ 224 - tophys_novmstack r12, r1 225 - lwz r12,_MSR(r12) 226 - andi. r12,r12,MSR_EE 227 - bne 1f 59 + blr 228 60 229 - /* MSR isn't changing, just transition directly */ 230 - #endif 231 - mtspr SPRN_SRR0,r11 232 - mtspr SPRN_SRR1,r10 233 - mtlr r9 234 - rfi /* jump to handler, enable MMU */ 235 - #ifdef CONFIG_40x 236 - b . /* Prevent prefetch past rfi */ 237 - #endif 238 - 239 - #if defined (CONFIG_PPC_BOOK3S_32) || defined(CONFIG_E500) 240 61 4: rlwinm r12,r12,0,~_TLF_NAPPING 241 62 stw r12,TI_LOCAL_FLAGS(r2) 242 63 b power_save_ppc32_restore ··· 67 246 lwz r9,_MSR(r11) /* if sleeping, clear MSR.EE */ 68 247 rlwinm r9,r9,0,~MSR_EE 69 248 lwz r12,_LINK(r11) /* and return to address in LR */ 70 - kuap_restore r11, r2, r3, r4, r5 71 249 lwz r2, GPR2(r11) 72 250 b fast_exception_return 73 - #endif 74 - _ASM_NOKPROBE_SYMBOL(transfer_to_handler) 75 - _ASM_NOKPROBE_SYMBOL(transfer_to_handler_cont) 76 - 77 - #ifdef CONFIG_TRACE_IRQFLAGS 78 - 1: /* MSR is changing, re-enable MMU so we can notify lockdep. We need to 79 - * keep interrupts disabled at this point otherwise we might risk 80 - * taking an interrupt before we tell lockdep they are enabled. 81 - */ 82 - lis r12,reenable_mmu@h 83 - ori r12,r12,reenable_mmu@l 84 - LOAD_REG_IMMEDIATE(r0, MSR_KERNEL) 85 - mtspr SPRN_SRR0,r12 86 - mtspr SPRN_SRR1,r0 87 - rfi 88 - #ifdef CONFIG_40x 89 - b . /* Prevent prefetch past rfi */ 90 - #endif 91 - 92 - reenable_mmu: 93 - /* 94 - * We save a bunch of GPRs, 95 - * r3 can be different from GPR3(r1) at this point, r9 and r11 96 - * contains the old MSR and handler address respectively, 97 - * r0, r4-r8, r12, CCR, CTR, XER etc... are left 98 - * clobbered as they aren't useful past this point. 99 - */ 100 - 101 - stwu r1,-32(r1) 102 - stw r9,8(r1) 103 - stw r11,12(r1) 104 - stw r3,16(r1) 105 - 106 - /* If we are disabling interrupts (normal case), simply log it with 107 - * lockdep 108 - */ 109 - 1: bl trace_hardirqs_off 110 - lwz r3,16(r1) 111 - lwz r11,12(r1) 112 - lwz r9,8(r1) 113 - addi r1,r1,32 114 - mtctr r11 115 - mtlr r9 116 - bctr /* jump to handler */ 117 - #endif /* CONFIG_TRACE_IRQFLAGS */ 118 - 119 - #ifndef CONFIG_VMAP_STACK 120 - /* 121 - * On kernel stack overflow, load up an initial stack pointer 122 - * and call StackOverflow(regs), which should not return. 123 - */ 124 - stack_ovf: 125 - /* sometimes we use a statically-allocated stack, which is OK. */ 126 - lis r12,_end@h 127 - ori r12,r12,_end@l 128 - cmplw r1,r12 129 - ble 5b /* r1 <= &_end is OK */ 130 - SAVE_NVGPRS(r11) 131 - addi r3,r1,STACK_FRAME_OVERHEAD 132 - lis r1,init_thread_union@ha 133 - addi r1,r1,init_thread_union@l 134 - addi r1,r1,THREAD_SIZE-STACK_FRAME_OVERHEAD 135 - lis r9,StackOverflow@ha 136 - addi r9,r9,StackOverflow@l 137 - LOAD_REG_IMMEDIATE(r10,MSR_KERNEL) 138 - #if defined(CONFIG_PPC_8xx) && defined(CONFIG_PERF_EVENTS) 139 - mtspr SPRN_NRI, r0 140 - #endif 141 - mtspr SPRN_SRR0,r9 142 - mtspr SPRN_SRR1,r10 143 - rfi 144 - #ifdef CONFIG_40x 145 - b . /* Prevent prefetch past rfi */ 146 - #endif 147 - _ASM_NOKPROBE_SYMBOL(stack_ovf) 148 - #endif 251 + _ASM_NOKPROBE_SYMBOL(prepare_transfer_to_handler) 252 + #endif /* CONFIG_PPC_BOOK3S_32 || CONFIG_E500 */ 149 253 150 254 .globl transfer_to_syscall 151 255 transfer_to_syscall: 152 256 SAVE_NVGPRS(r1) 153 - #ifdef CONFIG_PPC_BOOK3S_32 154 - kuep_lock r11, r12 155 - #endif 156 257 157 258 /* Calling convention has r9 = orig r0, r10 = regs */ 158 259 addi r10,r1,STACK_FRAME_OVERHEAD 159 260 mr r9,r0 160 - stw r10,THREAD+PT_REGS(r2) 161 261 bl system_call_exception 162 262 163 263 ret_from_syscall: ··· 91 349 cmplwi cr0,r5,0 92 350 bne- 2f 93 351 #endif /* CONFIG_PPC_47x */ 94 - #ifdef CONFIG_PPC_BOOK3S_32 95 - kuep_unlock r5, r7 96 - #endif 97 - kuap_check r2, r4 98 352 lwz r4,_LINK(r1) 99 353 lwz r5,_CCR(r1) 100 354 mtlr r4 ··· 150 412 b ret_from_syscall 151 413 152 414 /* 153 - * Top-level page fault handling. 154 - * This is in assembler because if do_page_fault tells us that 155 - * it is a bad kernel page fault, we want to save the non-volatile 156 - * registers before calling bad_page_fault. 157 - */ 158 - .globl handle_page_fault 159 - handle_page_fault: 160 - addi r3,r1,STACK_FRAME_OVERHEAD 161 - bl do_page_fault 162 - cmpwi r3,0 163 - beq+ ret_from_except 164 - SAVE_NVGPRS(r1) 165 - lwz r0,_TRAP(r1) 166 - clrrwi r0,r0,1 167 - stw r0,_TRAP(r1) 168 - mr r4,r3 /* err arg for bad_page_fault */ 169 - addi r3,r1,STACK_FRAME_OVERHEAD 170 - bl __bad_page_fault 171 - b ret_from_except_full 172 - 173 - /* 174 415 * This routine switches between two different tasks. The process 175 416 * state of one is saved on its kernel stack. Then the state 176 417 * of the other is restored from its kernel stack. The memory ··· 202 485 stw r10,_CCR(r1) 203 486 stw r1,KSP(r3) /* Set old stack pointer */ 204 487 205 - kuap_check r2, r0 206 488 #ifdef CONFIG_SMP 207 489 /* We need a sync somewhere here to make sure that if the 208 490 * previous task gets rescheduled on another CPU, it sees all ··· 245 529 fast_exception_return: 246 530 #if !(defined(CONFIG_4xx) || defined(CONFIG_BOOKE)) 247 531 andi. r10,r9,MSR_RI /* check for recoverable interrupt */ 248 - beq 1f /* if not, we've got problems */ 532 + beq 3f /* if not, we've got problems */ 249 533 #endif 250 534 251 535 2: REST_4GPRS(3, r11) 252 536 lwz r10,_CCR(r11) 253 - REST_GPR(1, r11) 537 + REST_2GPRS(1, r11) 254 538 mtcr r10 255 539 lwz r10,_LINK(r11) 256 540 mtlr r10 ··· 272 556 #endif 273 557 _ASM_NOKPROBE_SYMBOL(fast_exception_return) 274 558 275 - #if !(defined(CONFIG_4xx) || defined(CONFIG_BOOKE)) 276 - /* check if the exception happened in a restartable section */ 277 - 1: lis r3,exc_exit_restart_end@ha 278 - addi r3,r3,exc_exit_restart_end@l 279 - cmplw r12,r3 280 - bge 3f 281 - lis r4,exc_exit_restart@ha 282 - addi r4,r4,exc_exit_restart@l 283 - cmplw r12,r4 284 - blt 3f 285 - lis r3,fee_restarts@ha 286 - tophys(r3,r3) 287 - lwz r5,fee_restarts@l(r3) 288 - addi r5,r5,1 289 - stw r5,fee_restarts@l(r3) 290 - mr r12,r4 /* restart at exc_exit_restart */ 291 - b 2b 292 - 293 - .section .bss 294 - .align 2 295 - fee_restarts: 296 - .space 4 297 - .previous 298 - 299 559 /* aargh, a nonrecoverable interrupt, panic */ 300 560 /* aargh, we don't know which trap this is */ 301 561 3: 302 562 li r10,-1 303 563 stw r10,_TRAP(r11) 564 + prepare_transfer_to_handler 565 + bl unrecoverable_exception 566 + trap /* should not get here */ 567 + 568 + .globl interrupt_return 569 + interrupt_return: 570 + lwz r4,_MSR(r1) 304 571 addi r3,r1,STACK_FRAME_OVERHEAD 305 - lis r10,MSR_KERNEL@h 306 - ori r10,r10,MSR_KERNEL@l 307 - bl transfer_to_handler_full 308 - .long unrecoverable_exception 309 - .long ret_from_except 310 - #endif 572 + andi. r0,r4,MSR_PR 573 + beq .Lkernel_interrupt_return 574 + bl interrupt_exit_user_prepare 575 + cmpwi r3,0 576 + bne- .Lrestore_nvgprs 311 577 312 - .globl ret_from_except_full 313 - ret_from_except_full: 314 - REST_NVGPRS(r1) 315 - /* fall through */ 316 - 317 - .globl ret_from_except 318 - ret_from_except: 319 - /* Hard-disable interrupts so that current_thread_info()->flags 320 - * can't change between when we test it and when we return 321 - * from the interrupt. */ 322 - /* Note: We don't bother telling lockdep about it */ 323 - LOAD_REG_IMMEDIATE(r10,MSR_KERNEL) 324 - mtmsr r10 /* disable interrupts */ 325 - 326 - lwz r3,_MSR(r1) /* Returning to user mode? */ 327 - andi. r0,r3,MSR_PR 328 - beq resume_kernel 329 - 330 - user_exc_return: /* r10 contains MSR_KERNEL here */ 331 - /* Check current_thread_info()->flags */ 332 - lwz r9,TI_FLAGS(r2) 333 - andi. r0,r9,_TIF_USER_WORK_MASK 334 - bne do_work 335 - 336 - restore_user: 337 - #if defined(CONFIG_4xx) || defined(CONFIG_BOOKE) 338 - /* Check whether this process has its own DBCR0 value. The internal 339 - debug mode bit tells us that dbcr0 should be loaded. */ 340 - lwz r0,THREAD+THREAD_DBCR0(r2) 341 - andis. r10,r0,DBCR0_IDM@h 342 - bnel- load_dbcr0 343 - #endif 344 - ACCOUNT_CPU_USER_EXIT(r2, r10, r11) 345 - #ifdef CONFIG_PPC_BOOK3S_32 346 - kuep_unlock r10, r11 347 - #endif 348 - 349 - b restore 350 - 351 - /* N.B. the only way to get here is from the beq following ret_from_except. */ 352 - resume_kernel: 353 - /* check current_thread_info, _TIF_EMULATE_STACK_STORE */ 354 - lwz r8,TI_FLAGS(r2) 355 - andis. r0,r8,_TIF_EMULATE_STACK_STORE@h 356 - beq+ 1f 357 - 358 - addi r8,r1,INT_FRAME_SIZE /* Get the kprobed function entry */ 359 - 360 - lwz r3,GPR1(r1) 361 - subi r3,r3,INT_FRAME_SIZE /* dst: Allocate a trampoline exception frame */ 362 - mr r4,r1 /* src: current exception frame */ 363 - mr r1,r3 /* Reroute the trampoline frame to r1 */ 364 - 365 - /* Copy from the original to the trampoline. */ 366 - li r5,INT_FRAME_SIZE/4 /* size: INT_FRAME_SIZE */ 367 - li r6,0 /* start offset: 0 */ 368 - mtctr r5 369 - 2: lwzx r0,r6,r4 370 - stwx r0,r6,r3 371 - addi r6,r6,4 372 - bdnz 2b 373 - 374 - /* Do real store operation to complete stwu */ 375 - lwz r5,GPR1(r1) 376 - stw r8,0(r5) 377 - 378 - /* Clear _TIF_EMULATE_STACK_STORE flag */ 379 - lis r11,_TIF_EMULATE_STACK_STORE@h 380 - addi r5,r2,TI_FLAGS 381 - 0: lwarx r8,0,r5 382 - andc r8,r8,r11 383 - stwcx. r8,0,r5 384 - bne- 0b 385 - 1: 386 - 387 - #ifdef CONFIG_PREEMPTION 388 - /* check current_thread_info->preempt_count */ 389 - lwz r0,TI_PREEMPT(r2) 390 - cmpwi 0,r0,0 /* if non-zero, just restore regs and return */ 391 - bne restore_kuap 392 - andi. r8,r8,_TIF_NEED_RESCHED 393 - beq+ restore_kuap 394 - lwz r3,_MSR(r1) 395 - andi. r0,r3,MSR_EE /* interrupts off? */ 396 - beq restore_kuap /* don't schedule if so */ 397 - #ifdef CONFIG_TRACE_IRQFLAGS 398 - /* Lockdep thinks irqs are enabled, we need to call 399 - * preempt_schedule_irq with IRQs off, so we inform lockdep 400 - * now that we -did- turn them off already 401 - */ 402 - bl trace_hardirqs_off 403 - #endif 404 - bl preempt_schedule_irq 405 - #ifdef CONFIG_TRACE_IRQFLAGS 406 - /* And now, to properly rebalance the above, we tell lockdep they 407 - * are being turned back on, which will happen when we return 408 - */ 409 - bl trace_hardirqs_on 410 - #endif 411 - #endif /* CONFIG_PREEMPTION */ 412 - restore_kuap: 413 - kuap_restore r1, r2, r9, r10, r0 414 - 415 - /* interrupts are hard-disabled at this point */ 416 - restore: 417 - #if defined(CONFIG_44x) && !defined(CONFIG_PPC_47x) 418 - lis r4,icache_44x_need_flush@ha 419 - lwz r5,icache_44x_need_flush@l(r4) 420 - cmplwi cr0,r5,0 421 - beq+ 1f 422 - li r6,0 423 - iccci r0,r0 424 - stw r6,icache_44x_need_flush@l(r4) 425 - 1: 426 - #endif /* CONFIG_44x */ 427 - 428 - lwz r9,_MSR(r1) 429 - #ifdef CONFIG_TRACE_IRQFLAGS 430 - /* Lockdep doesn't know about the fact that IRQs are temporarily turned 431 - * off in this assembly code while peeking at TI_FLAGS() and such. However 432 - * we need to inform it if the exception turned interrupts off, and we 433 - * are about to trun them back on. 434 - */ 435 - andi. r10,r9,MSR_EE 436 - beq 1f 437 - stwu r1,-32(r1) 438 - mflr r0 439 - stw r0,4(r1) 440 - bl trace_hardirqs_on 441 - addi r1, r1, 32 442 - lwz r9,_MSR(r1) 443 - 1: 444 - #endif /* CONFIG_TRACE_IRQFLAGS */ 445 - 446 - lwz r0,GPR0(r1) 447 - lwz r2,GPR2(r1) 448 - REST_4GPRS(3, r1) 449 - REST_2GPRS(7, r1) 450 - 451 - lwz r10,_XER(r1) 452 - lwz r11,_CTR(r1) 453 - mtspr SPRN_XER,r10 454 - mtctr r11 455 - 456 - BEGIN_FTR_SECTION 457 - lwarx r11,0,r1 458 - END_FTR_SECTION_IFSET(CPU_FTR_NEED_PAIRED_STWCX) 459 - stwcx. r0,0,r1 /* to clear the reservation */ 460 - 461 - #if !(defined(CONFIG_4xx) || defined(CONFIG_BOOKE)) 462 - andi. r10,r9,MSR_RI /* check if this exception occurred */ 463 - beql nonrecoverable /* at a bad place (MSR:RI = 0) */ 464 - 465 - lwz r10,_CCR(r1) 466 - lwz r11,_LINK(r1) 467 - mtcrf 0xFF,r10 468 - mtlr r11 469 - 470 - /* Clear the exception_marker on the stack to avoid confusing stacktrace */ 471 - li r10, 0 472 - stw r10, 8(r1) 473 - /* 474 - * Once we put values in SRR0 and SRR1, we are in a state 475 - * where exceptions are not recoverable, since taking an 476 - * exception will trash SRR0 and SRR1. Therefore we clear the 477 - * MSR:RI bit to indicate this. If we do take an exception, 478 - * we can't return to the point of the exception but we 479 - * can restart the exception exit path at the label 480 - * exc_exit_restart below. -- paulus 481 - */ 482 - LOAD_REG_IMMEDIATE(r10,MSR_KERNEL & ~MSR_RI) 483 - mtmsr r10 /* clear the RI bit */ 484 - .globl exc_exit_restart 485 - exc_exit_restart: 486 - lwz r12,_NIP(r1) 487 - mtspr SPRN_SRR0,r12 488 - mtspr SPRN_SRR1,r9 489 - REST_4GPRS(9, r1) 490 - lwz r1,GPR1(r1) 491 - .globl exc_exit_restart_end 492 - exc_exit_restart_end: 493 - rfi 494 - _ASM_NOKPROBE_SYMBOL(exc_exit_restart) 495 - _ASM_NOKPROBE_SYMBOL(exc_exit_restart_end) 496 - 497 - #else /* !(CONFIG_4xx || CONFIG_BOOKE) */ 498 - /* 499 - * This is a bit different on 4xx/Book-E because it doesn't have 500 - * the RI bit in the MSR. 501 - * The TLB miss handler checks if we have interrupted 502 - * the exception exit path and restarts it if so 503 - * (well maybe one day it will... :). 504 - */ 505 - lwz r11,_LINK(r1) 506 - mtlr r11 507 - lwz r10,_CCR(r1) 508 - mtcrf 0xff,r10 509 - /* Clear the exception_marker on the stack to avoid confusing stacktrace */ 510 - li r10, 0 511 - stw r10, 8(r1) 512 - REST_2GPRS(9, r1) 513 - .globl exc_exit_restart 514 - exc_exit_restart: 578 + .Lfast_user_interrupt_return: 515 579 lwz r11,_NIP(r1) 516 580 lwz r12,_MSR(r1) 517 581 mtspr SPRN_SRR0,r11 518 582 mtspr SPRN_SRR1,r12 583 + 584 + BEGIN_FTR_SECTION 585 + stwcx. r0,0,r1 /* to clear the reservation */ 586 + FTR_SECTION_ELSE 587 + lwarx r0,0,r1 588 + ALT_FTR_SECTION_END_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS) 589 + 590 + lwz r3,_CCR(r1) 591 + lwz r4,_LINK(r1) 592 + lwz r5,_CTR(r1) 593 + lwz r6,_XER(r1) 594 + li r0,0 595 + 596 + /* 597 + * Leaving a stale exception_marker on the stack can confuse 598 + * the reliable stack unwinder later on. Clear it. 599 + */ 600 + stw r0,8(r1) 601 + REST_4GPRS(7, r1) 519 602 REST_2GPRS(11, r1) 520 - lwz r1,GPR1(r1) 521 - .globl exc_exit_restart_end 522 - exc_exit_restart_end: 603 + 604 + mtcr r3 605 + mtlr r4 606 + mtctr r5 607 + mtspr SPRN_XER,r6 608 + 609 + REST_4GPRS(2, r1) 610 + REST_GPR(6, r1) 611 + REST_GPR(0, r1) 612 + REST_GPR(1, r1) 523 613 rfi 524 - b . /* prevent prefetch past rfi */ 525 - _ASM_NOKPROBE_SYMBOL(exc_exit_restart) 614 + #ifdef CONFIG_40x 615 + b . /* Prevent prefetch past rfi */ 616 + #endif 617 + 618 + .Lrestore_nvgprs: 619 + REST_NVGPRS(r1) 620 + b .Lfast_user_interrupt_return 621 + 622 + .Lkernel_interrupt_return: 623 + bl interrupt_exit_kernel_prepare 624 + 625 + .Lfast_kernel_interrupt_return: 626 + cmpwi cr1,r3,0 627 + lwz r11,_NIP(r1) 628 + lwz r12,_MSR(r1) 629 + mtspr SPRN_SRR0,r11 630 + mtspr SPRN_SRR1,r12 631 + 632 + BEGIN_FTR_SECTION 633 + stwcx. r0,0,r1 /* to clear the reservation */ 634 + FTR_SECTION_ELSE 635 + lwarx r0,0,r1 636 + ALT_FTR_SECTION_END_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS) 637 + 638 + lwz r3,_LINK(r1) 639 + lwz r4,_CTR(r1) 640 + lwz r5,_XER(r1) 641 + lwz r6,_CCR(r1) 642 + li r0,0 643 + 644 + REST_4GPRS(7, r1) 645 + REST_2GPRS(11, r1) 646 + 647 + mtlr r3 648 + mtctr r4 649 + mtspr SPRN_XER,r5 650 + 651 + /* 652 + * Leaving a stale exception_marker on the stack can confuse 653 + * the reliable stack unwinder later on. Clear it. 654 + */ 655 + stw r0,8(r1) 656 + 657 + REST_4GPRS(2, r1) 658 + 659 + bne- cr1,1f /* emulate stack store */ 660 + mtcr r6 661 + REST_GPR(6, r1) 662 + REST_GPR(0, r1) 663 + REST_GPR(1, r1) 664 + rfi 665 + #ifdef CONFIG_40x 666 + b . /* Prevent prefetch past rfi */ 667 + #endif 668 + 669 + 1: /* 670 + * Emulate stack store with update. New r1 value was already calculated 671 + * and updated in our interrupt regs by emulate_loadstore, but we can't 672 + * store the previous value of r1 to the stack before re-loading our 673 + * registers from it, otherwise they could be clobbered. Use 674 + * SPRG Scratch0 as temporary storage to hold the store 675 + * data, as interrupts are disabled here so it won't be clobbered. 676 + */ 677 + mtcr r6 678 + #ifdef CONFIG_BOOKE 679 + mtspr SPRN_SPRG_WSCRATCH0, r9 680 + #else 681 + mtspr SPRN_SPRG_SCRATCH0, r9 682 + #endif 683 + addi r9,r1,INT_FRAME_SIZE /* get original r1 */ 684 + REST_GPR(6, r1) 685 + REST_GPR(0, r1) 686 + REST_GPR(1, r1) 687 + stw r9,0(r1) /* perform store component of stwu */ 688 + #ifdef CONFIG_BOOKE 689 + mfspr r9, SPRN_SPRG_RSCRATCH0 690 + #else 691 + mfspr r9, SPRN_SPRG_SCRATCH0 692 + #endif 693 + rfi 694 + #ifdef CONFIG_40x 695 + b . /* Prevent prefetch past rfi */ 696 + #endif 697 + _ASM_NOKPROBE_SYMBOL(interrupt_return) 698 + 699 + #if defined(CONFIG_4xx) || defined(CONFIG_BOOKE) 526 700 527 701 /* 528 702 * Returning from a critical interrupt in user mode doesn't need ··· 443 837 REST_NVGPRS(r1); \ 444 838 lwz r3,_MSR(r1); \ 445 839 andi. r3,r3,MSR_PR; \ 446 - LOAD_REG_IMMEDIATE(r10,MSR_KERNEL); \ 447 - bne user_exc_return; \ 840 + bne interrupt_return; \ 448 841 lwz r0,GPR0(r1); \ 449 842 lwz r2,GPR2(r1); \ 450 843 REST_4GPRS(3, r1); \ ··· 511 906 #ifdef CONFIG_40x 512 907 .globl ret_from_crit_exc 513 908 ret_from_crit_exc: 514 - mfspr r9,SPRN_SPRG_THREAD 515 - lis r10,saved_ksp_limit@ha; 516 - lwz r10,saved_ksp_limit@l(r10); 517 - tovirt(r9,r9); 518 - stw r10,KSP_LIMIT(r9) 519 909 lis r9,crit_srr0@ha; 520 910 lwz r9,crit_srr0@l(r9); 521 911 lis r10,crit_srr1@ha; ··· 524 924 #ifdef CONFIG_BOOKE 525 925 .globl ret_from_crit_exc 526 926 ret_from_crit_exc: 527 - mfspr r9,SPRN_SPRG_THREAD 528 - lwz r10,SAVED_KSP_LIMIT(r1) 529 - stw r10,KSP_LIMIT(r9) 530 927 RESTORE_xSRR(SRR0,SRR1); 531 928 RESTORE_MMU_REGS; 532 929 RET_FROM_EXC_LEVEL(SPRN_CSRR0, SPRN_CSRR1, PPC_RFCI) ··· 531 934 532 935 .globl ret_from_debug_exc 533 936 ret_from_debug_exc: 534 - mfspr r9,SPRN_SPRG_THREAD 535 - lwz r10,SAVED_KSP_LIMIT(r1) 536 - stw r10,KSP_LIMIT(r9) 537 937 RESTORE_xSRR(SRR0,SRR1); 538 938 RESTORE_xSRR(CSRR0,CSRR1); 539 939 RESTORE_MMU_REGS; ··· 539 945 540 946 .globl ret_from_mcheck_exc 541 947 ret_from_mcheck_exc: 542 - mfspr r9,SPRN_SPRG_THREAD 543 - lwz r10,SAVED_KSP_LIMIT(r1) 544 - stw r10,KSP_LIMIT(r9) 545 948 RESTORE_xSRR(SRR0,SRR1); 546 949 RESTORE_xSRR(CSRR0,CSRR1); 547 950 RESTORE_xSRR(DSRR0,DSRR1); ··· 546 955 RET_FROM_EXC_LEVEL(SPRN_MCSRR0, SPRN_MCSRR1, PPC_RFMCI) 547 956 _ASM_NOKPROBE_SYMBOL(ret_from_mcheck_exc) 548 957 #endif /* CONFIG_BOOKE */ 549 - 550 - /* 551 - * Load the DBCR0 value for a task that is being ptraced, 552 - * having first saved away the global DBCR0. Note that r0 553 - * has the dbcr0 value to set upon entry to this. 554 - */ 555 - load_dbcr0: 556 - mfmsr r10 /* first disable debug exceptions */ 557 - rlwinm r10,r10,0,~MSR_DE 558 - mtmsr r10 559 - isync 560 - mfspr r10,SPRN_DBCR0 561 - lis r11,global_dbcr0@ha 562 - addi r11,r11,global_dbcr0@l 563 - #ifdef CONFIG_SMP 564 - lwz r9,TASK_CPU(r2) 565 - slwi r9,r9,2 566 - add r11,r11,r9 567 - #endif 568 - stw r10,0(r11) 569 - mtspr SPRN_DBCR0,r0 570 - li r11,-1 571 - mtspr SPRN_DBSR,r11 /* clear all pending debug events */ 572 - blr 573 - 574 - .section .bss 575 - .align 4 576 - .global global_dbcr0 577 - global_dbcr0: 578 - .space 4*NR_CPUS 579 - .previous 580 958 #endif /* !(CONFIG_4xx || CONFIG_BOOKE) */ 581 - 582 - do_work: /* r10 contains MSR_KERNEL here */ 583 - andi. r0,r9,_TIF_NEED_RESCHED 584 - beq do_user_signal 585 - 586 - do_resched: /* r10 contains MSR_KERNEL here */ 587 - #ifdef CONFIG_TRACE_IRQFLAGS 588 - bl trace_hardirqs_on 589 - mfmsr r10 590 - #endif 591 - ori r10,r10,MSR_EE 592 - mtmsr r10 /* hard-enable interrupts */ 593 - bl schedule 594 - recheck: 595 - /* Note: And we don't tell it we are disabling them again 596 - * neither. Those disable/enable cycles used to peek at 597 - * TI_FLAGS aren't advertised. 598 - */ 599 - LOAD_REG_IMMEDIATE(r10,MSR_KERNEL) 600 - mtmsr r10 /* disable interrupts */ 601 - lwz r9,TI_FLAGS(r2) 602 - andi. r0,r9,_TIF_NEED_RESCHED 603 - bne- do_resched 604 - andi. r0,r9,_TIF_USER_WORK_MASK 605 - beq restore_user 606 - do_user_signal: /* r10 contains MSR_KERNEL here */ 607 - ori r10,r10,MSR_EE 608 - mtmsr r10 /* hard-enable interrupts */ 609 - /* save r13-r31 in the exception frame, if not already done */ 610 - lwz r3,_TRAP(r1) 611 - andi. r0,r3,1 612 - beq 2f 613 - SAVE_NVGPRS(r1) 614 - rlwinm r3,r3,0,0,30 615 - stw r3,_TRAP(r1) 616 - 2: addi r3,r1,STACK_FRAME_OVERHEAD 617 - mr r4,r9 618 - bl do_notify_resume 619 - REST_NVGPRS(r1) 620 - b recheck 621 - 622 - /* 623 - * We come here when we are at the end of handling an exception 624 - * that occurred at a place where taking an exception will lose 625 - * state information, such as the contents of SRR0 and SRR1. 626 - */ 627 - nonrecoverable: 628 - lis r10,exc_exit_restart_end@ha 629 - addi r10,r10,exc_exit_restart_end@l 630 - cmplw r12,r10 631 - bge 3f 632 - lis r11,exc_exit_restart@ha 633 - addi r11,r11,exc_exit_restart@l 634 - cmplw r12,r11 635 - blt 3f 636 - lis r10,ee_restarts@ha 637 - lwz r12,ee_restarts@l(r10) 638 - addi r12,r12,1 639 - stw r12,ee_restarts@l(r10) 640 - mr r12,r11 /* restart at exc_exit_restart */ 641 - blr 642 - 3: /* OK, we can't recover, kill this process */ 643 - lwz r3,_TRAP(r1) 644 - andi. r0,r3,1 645 - beq 5f 646 - SAVE_NVGPRS(r1) 647 - rlwinm r3,r3,0,0,30 648 - stw r3,_TRAP(r1) 649 - 5: mfspr r2,SPRN_SPRG_THREAD 650 - addi r2,r2,-THREAD 651 - tovirt(r2,r2) /* set back r2 to current */ 652 - 4: addi r3,r1,STACK_FRAME_OVERHEAD 653 - bl unrecoverable_exception 654 - /* shouldn't return */ 655 - b 4b 656 - _ASM_NOKPROBE_SYMBOL(nonrecoverable) 657 - 658 - .section .bss 659 - .align 2 660 - ee_restarts: 661 - .space 4 662 - .previous 663 959 664 960 /* 665 961 * PROM code for specific machines follows. Put it ··· 566 1088 lis r6,1f@ha /* physical return address for rtas */ 567 1089 addi r6,r6,1f@l 568 1090 tophys(r6,r6) 569 - tophys_novmstack r7, r1 570 1091 lwz r8,RTASENTRY(r4) 571 1092 lwz r4,RTASBASE(r4) 572 1093 mfmsr r9 ··· 574 1097 mtmsr r0 /* disable interrupts so SRR0/1 don't get trashed */ 575 1098 li r9,MSR_KERNEL & ~(MSR_IR|MSR_DR) 576 1099 mtlr r6 577 - stw r7, THREAD + RTAS_SP(r2) 1100 + stw r1, THREAD + RTAS_SP(r2) 578 1101 mtspr SPRN_SRR0,r8 579 1102 mtspr SPRN_SRR1,r9 580 1103 rfi 581 - 1: tophys_novmstack r9, r1 582 - #ifdef CONFIG_VMAP_STACK 583 - li r0, MSR_KERNEL & ~MSR_IR /* can take DTLB miss */ 584 - mtmsr r0 585 - isync 586 - #endif 587 - lwz r8,INT_FRAME_SIZE+4(r9) /* get return address */ 588 - lwz r9,8(r9) /* original msr value */ 589 - addi r1,r1,INT_FRAME_SIZE 590 - li r0,0 591 - tophys_novmstack r7, r2 592 - stw r0, THREAD + RTAS_SP(r7) 1104 + 1: 1105 + lis r8, 1f@h 1106 + ori r8, r8, 1f@l 1107 + LOAD_REG_IMMEDIATE(r9,MSR_KERNEL) 593 1108 mtspr SPRN_SRR0,r8 594 1109 mtspr SPRN_SRR1,r9 595 - rfi /* return to caller */ 1110 + rfi /* Reactivate MMU translation */ 1111 + 1: 1112 + lwz r8,INT_FRAME_SIZE+4(r1) /* get return address */ 1113 + lwz r9,8(r1) /* original msr value */ 1114 + addi r1,r1,INT_FRAME_SIZE 1115 + li r0,0 1116 + stw r0, THREAD + RTAS_SP(r2) 1117 + mtlr r8 1118 + mtmsr r9 1119 + blr /* return to caller */ 596 1120 _ASM_NOKPROBE_SYMBOL(enter_rtas) 597 1121 #endif /* CONFIG_PPC_RTAS */

+15 -25

arch/powerpc/kernel/entry_64.S

··· 117 117 END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) 118 118 119 119 /* 120 - * RECONCILE_IRQ_STATE without calling trace_hardirqs_off(), which 121 - * would clobber syscall parameters. Also we always enter with IRQs 122 - * enabled and nothing pending. system_call_exception() will call 123 - * trace_hardirqs_off(). 124 - * 125 - * scv enters with MSR[EE]=1, so don't set PACA_IRQ_HARD_DIS. The 126 - * entry vector already sets PACAIRQSOFTMASK to IRQS_ALL_DISABLED. 120 + * scv enters with MSR[EE]=1 and is immediately considered soft-masked. 121 + * The entry vector already sets PACAIRQSOFTMASK to IRQS_ALL_DISABLED, 122 + * and interrupts may be masked and pending already. 123 + * system_call_exception() will call trace_hardirqs_off() which means 124 + * interrupts could already have been blocked before trace_hardirqs_off, 125 + * but this is the best we can do. 127 126 */ 128 127 129 128 /* Calling convention has r9 = orig r0, r10 = regs */ ··· 287 288 std r11,-16(r10) /* "regshere" marker */ 288 289 289 290 /* 290 - * RECONCILE_IRQ_STATE without calling trace_hardirqs_off(), which 291 - * would clobber syscall parameters. Also we always enter with IRQs 292 - * enabled and nothing pending. system_call_exception() will call 291 + * We always enter kernel from userspace with irq soft-mask enabled and 292 + * nothing pending. system_call_exception() will call 293 293 * trace_hardirqs_off(). 294 294 */ 295 295 li r11,IRQS_ALL_DISABLED ··· 414 416 bctrl 415 417 li r3,0 416 418 b .Lsyscall_exit 417 - 418 - #ifdef CONFIG_PPC_BOOK3E 419 - /* Save non-volatile GPRs, if not already saved. */ 420 - _GLOBAL(save_nvgprs) 421 - ld r11,_TRAP(r1) 422 - andi. r0,r11,1 423 - beqlr- 424 - SAVE_NVGPRS(r1) 425 - clrrdi r0,r11,1 426 - std r0,_TRAP(r1) 427 - blr 428 - _ASM_NOKPROBE_SYMBOL(save_nvgprs); 429 - #endif 430 419 431 420 #ifdef CONFIG_PPC_BOOK3S_64 432 421 ··· 630 645 addi r1,r1,SWITCH_FRAME_SIZE 631 646 blr 632 647 633 - #ifdef CONFIG_PPC_BOOK3S 634 648 /* 635 649 * If MSR EE/RI was never enabled, IRQs not reconciled, NVGPRs not 636 650 * touched, no exit work created, then this can be used. ··· 641 657 kuap_check_amr r3, r4 642 658 ld r5,_MSR(r1) 643 659 andi. r0,r5,MSR_PR 660 + #ifdef CONFIG_PPC_BOOK3S 644 661 bne .Lfast_user_interrupt_return_amr 645 662 kuap_kernel_restore r3, r4 646 663 andi. r0,r5,MSR_RI ··· 650 665 addi r3,r1,STACK_FRAME_OVERHEAD 651 666 bl unrecoverable_exception 652 667 b . /* should not get here */ 668 + #else 669 + bne .Lfast_user_interrupt_return 670 + b .Lfast_kernel_interrupt_return 671 + #endif 653 672 654 673 .balign IFETCH_ALIGN_BYTES 655 674 .globl interrupt_return ··· 667 678 cmpdi r3,0 668 679 bne- .Lrestore_nvgprs 669 680 681 + #ifdef CONFIG_PPC_BOOK3S 670 682 .Lfast_user_interrupt_return_amr: 671 683 kuap_user_restore r3, r4 684 + #endif 672 685 .Lfast_user_interrupt_return: 673 686 ld r11,_NIP(r1) 674 687 ld r12,_MSR(r1) ··· 779 788 780 789 RFI_TO_KERNEL 781 790 b . /* prevent speculative execution */ 782 - #endif /* CONFIG_PPC_BOOK3S */ 783 791 784 792 #ifdef CONFIG_PPC_RTAS 785 793 /*

+31 -394

arch/powerpc/kernel/exceptions-64e.S

··· 63 63 ld reg, (SPECIAL_EXC_##name * 8 + SPECIAL_EXC_FRAME_OFFS)(r1) 64 64 65 65 special_reg_save: 66 - lbz r9,PACAIRQHAPPENED(r13) 67 - RECONCILE_IRQ_STATE(r3,r4) 68 - 69 66 /* 70 67 * We only need (or have stack space) to save this stuff if 71 68 * we interrupted the kernel. ··· 116 119 mtspr SPRN_MAS5,r10 117 120 mtspr SPRN_MAS8,r10 118 121 END_FTR_SECTION_IFSET(CPU_FTR_EMB_HV) 119 - SPECIAL_EXC_STORE(r9,IRQHAPPENED) 120 - 121 122 mfspr r10,SPRN_DEAR 122 123 SPECIAL_EXC_STORE(r10,DEAR) 123 124 mfspr r10,SPRN_ESR 124 125 SPECIAL_EXC_STORE(r10,ESR) 125 126 126 - lbz r10,PACAIRQSOFTMASK(r13) 127 - SPECIAL_EXC_STORE(r10,SOFTE) 128 127 ld r10,_NIP(r1) 129 128 SPECIAL_EXC_STORE(r10,CSRR0) 130 129 ld r10,_MSR(r1) ··· 132 139 ld r3,_MSR(r1) 133 140 andi. r3,r3,MSR_PR 134 141 beq 1f 135 - b ret_from_except 142 + REST_NVGPRS(r1) 143 + b interrupt_return 136 144 1: 137 145 138 146 LOAD_REG_ADDR(r11,extlb_level_exc) ··· 186 192 SPECIAL_EXC_LOAD(r10,MAS8) 187 193 mtspr SPRN_MAS8,r10 188 194 END_FTR_SECTION_IFSET(CPU_FTR_EMB_HV) 189 - 190 - lbz r6,PACAIRQSOFTMASK(r13) 191 - ld r5,SOFTE(r1) 192 - 193 - /* Interrupts had better not already be enabled... */ 194 - tweqi r6,IRQS_ENABLED 195 - 196 - andi. r6,r5,IRQS_DISABLED 197 - bne 1f 198 - 199 - TRACE_ENABLE_INTS 200 - stb r5,PACAIRQSOFTMASK(r13) 201 - 1: 202 - /* 203 - * Restore PACAIRQHAPPENED rather than setting it based on 204 - * the return MSR[EE], since we could have interrupted 205 - * __check_irq_replay() or other inconsistent transitory 206 - * states that must remain that way. 207 - */ 208 - SPECIAL_EXC_LOAD(r10,IRQHAPPENED) 209 - stb r10,PACAIRQHAPPENED(r13) 210 195 211 196 SPECIAL_EXC_LOAD(r10,DEAR) 212 197 mtspr SPRN_DEAR,r10 ··· 390 417 std r6,_LINK(r1); \ 391 418 std r7,_CTR(r1); \ 392 419 std r8,_XER(r1); \ 393 - li r3,(n)+1; /* indicate partial regs in trap */ \ 420 + li r3,(n); /* regs.trap vector */ \ 394 421 std r9,0(r1); /* store stack frame back link */ \ 395 422 std r10,_CCR(r1); /* store orig CR in stackframe */ \ 396 423 std r9,GPR1(r1); /* store stack frame back link */ \ 397 424 std r11,SOFTE(r1); /* and save it to stackframe */ \ 398 425 std r12,STACK_FRAME_OVERHEAD-16(r1); /* mark the frame */ \ 399 426 std r3,_TRAP(r1); /* set trap number */ \ 400 - std r0,RESULT(r1); /* clear regs->result */ 427 + std r0,RESULT(r1); /* clear regs->result */ \ 428 + SAVE_NVGPRS(r1); 401 429 402 430 #define EXCEPTION_COMMON(n) \ 403 431 EXCEPTION_COMMON_LVL(n, SPRN_SPRG_GEN_SCRATCH, PACA_EXGEN) ··· 408 434 EXCEPTION_COMMON_LVL(n, SPRN_SPRG_MC_SCRATCH, PACA_EXMC) 409 435 #define EXCEPTION_COMMON_DBG(n) \ 410 436 EXCEPTION_COMMON_LVL(n, SPRN_SPRG_DBG_SCRATCH, PACA_EXDBG) 411 - 412 - /* 413 - * This is meant for exceptions that don't immediately hard-enable. We 414 - * set a bit in paca->irq_happened to ensure that a subsequent call to 415 - * arch_local_irq_restore() will properly hard-enable and avoid the 416 - * fast-path, and then reconcile irq state. 417 - */ 418 - #define INTS_DISABLE RECONCILE_IRQ_STATE(r3,r4) 419 - 420 - /* 421 - * This is called by exceptions that don't use INTS_DISABLE (that did not 422 - * touch irq indicators in the PACA). This will restore MSR:EE to it's 423 - * previous value 424 - * 425 - * XXX In the long run, we may want to open-code it in order to separate the 426 - * load from the wrtee, thus limiting the latency caused by the dependency 427 - * but at this point, I'll favor code clarity until we have a near to final 428 - * implementation 429 - */ 430 - #define INTS_RESTORE_HARD \ 431 - ld r11,_MSR(r1); \ 432 - wrtee r11; 433 437 434 438 /* XXX FIXME: Restore r14/r15 when necessary */ 435 439 #define BAD_STACK_TRAMPOLINE(n) \ ··· 457 505 START_EXCEPTION(label); \ 458 506 NORMAL_EXCEPTION_PROLOG(trapnum, intnum, PROLOG_ADDITION_MASKABLE)\ 459 507 EXCEPTION_COMMON(trapnum) \ 460 - INTS_DISABLE; \ 461 508 ack(r8); \ 462 509 CHECK_NAPPING(); \ 463 510 addi r3,r1,STACK_FRAME_OVERHEAD; \ 464 511 bl hdlr; \ 465 - b ret_from_except_lite; 512 + b interrupt_return 466 513 467 514 /* This value is used to mark exception frames on the stack. */ 468 515 .section ".toc","aw" ··· 512 561 CRIT_EXCEPTION_PROLOG(0x100, BOOKE_INTERRUPT_CRITICAL, 513 562 PROLOG_ADDITION_NONE) 514 563 EXCEPTION_COMMON_CRIT(0x100) 515 - bl save_nvgprs 516 564 bl special_reg_save 517 565 CHECK_NAPPING(); 518 566 addi r3,r1,STACK_FRAME_OVERHEAD 519 - bl unknown_exception 567 + bl unknown_nmi_exception 520 568 b ret_from_crit_except 521 569 522 570 /* Machine Check Interrupt */ ··· 523 573 MC_EXCEPTION_PROLOG(0x000, BOOKE_INTERRUPT_MACHINE_CHECK, 524 574 PROLOG_ADDITION_NONE) 525 575 EXCEPTION_COMMON_MC(0x000) 526 - bl save_nvgprs 527 576 bl special_reg_save 528 577 CHECK_NAPPING(); 529 578 addi r3,r1,STACK_FRAME_OVERHEAD ··· 536 587 mfspr r14,SPRN_DEAR 537 588 mfspr r15,SPRN_ESR 538 589 EXCEPTION_COMMON(0x300) 539 - INTS_DISABLE 540 590 b storage_fault_common 541 591 542 592 /* Instruction Storage Interrupt */ ··· 545 597 li r15,0 546 598 mr r14,r10 547 599 EXCEPTION_COMMON(0x400) 548 - INTS_DISABLE 549 600 b storage_fault_common 550 601 551 602 /* External Input Interrupt */ ··· 566 619 PROLOG_ADDITION_1REG) 567 620 mfspr r14,SPRN_ESR 568 621 EXCEPTION_COMMON(0x700) 569 - INTS_DISABLE 570 622 std r14,_DSISR(r1) 571 623 addi r3,r1,STACK_FRAME_OVERHEAD 572 624 ld r14,PACA_EXGEN+EX_R14(r13) 573 - bl save_nvgprs 574 625 bl program_check_exception 575 - b ret_from_except 626 + REST_NVGPRS(r1) 627 + b interrupt_return 576 628 577 629 /* Floating Point Unavailable Interrupt */ 578 630 START_EXCEPTION(fp_unavailable); ··· 583 637 andi. r0,r12,MSR_PR; 584 638 beq- 1f 585 639 bl load_up_fpu 586 - b fast_exception_return 587 - 1: INTS_DISABLE 588 - bl save_nvgprs 589 - addi r3,r1,STACK_FRAME_OVERHEAD 640 + b fast_interrupt_return 641 + 1: addi r3,r1,STACK_FRAME_OVERHEAD 590 642 bl kernel_fp_unavailable_exception 591 - b ret_from_except 643 + b interrupt_return 592 644 593 645 /* Altivec Unavailable Interrupt */ 594 646 START_EXCEPTION(altivec_unavailable); ··· 600 656 andi. r0,r12,MSR_PR; 601 657 beq- 1f 602 658 bl load_up_altivec 603 - b fast_exception_return 659 + b fast_interrupt_return 604 660 1: 605 661 END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) 606 662 #endif 607 - INTS_DISABLE 608 - bl save_nvgprs 609 663 addi r3,r1,STACK_FRAME_OVERHEAD 610 664 bl altivec_unavailable_exception 611 - b ret_from_except 665 + b interrupt_return 612 666 613 667 /* AltiVec Assist */ 614 668 START_EXCEPTION(altivec_assist); ··· 614 672 BOOKE_INTERRUPT_ALTIVEC_ASSIST, 615 673 PROLOG_ADDITION_NONE) 616 674 EXCEPTION_COMMON(0x220) 617 - INTS_DISABLE 618 - bl save_nvgprs 619 675 addi r3,r1,STACK_FRAME_OVERHEAD 620 676 #ifdef CONFIG_ALTIVEC 621 677 BEGIN_FTR_SECTION 622 678 bl altivec_assist_exception 623 679 END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) 680 + REST_NVGPRS(r1) 624 681 #else 625 682 bl unknown_exception 626 683 #endif 627 - b ret_from_except 684 + b interrupt_return 628 685 629 686 630 687 /* Decrementer Interrupt */ ··· 639 698 CRIT_EXCEPTION_PROLOG(0x9f0, BOOKE_INTERRUPT_WATCHDOG, 640 699 PROLOG_ADDITION_NONE) 641 700 EXCEPTION_COMMON_CRIT(0x9f0) 642 - bl save_nvgprs 643 701 bl special_reg_save 644 702 CHECK_NAPPING(); 645 703 addi r3,r1,STACK_FRAME_OVERHEAD 646 704 #ifdef CONFIG_BOOKE_WDT 647 705 bl WatchdogException 648 706 #else 649 - bl unknown_exception 707 + bl unknown_nmi_exception 650 708 #endif 651 709 b ret_from_crit_except 652 710 ··· 662 722 NORMAL_EXCEPTION_PROLOG(0xf20, BOOKE_INTERRUPT_AP_UNAVAIL, 663 723 PROLOG_ADDITION_NONE) 664 724 EXCEPTION_COMMON(0xf20) 665 - INTS_DISABLE 666 - bl save_nvgprs 667 725 addi r3,r1,STACK_FRAME_OVERHEAD 668 726 bl unknown_exception 669 - b ret_from_except 727 + b interrupt_return 670 728 671 729 /* Debug exception as a critical interrupt*/ 672 730 START_EXCEPTION(debug_crit); ··· 730 792 addi r3,r1,STACK_FRAME_OVERHEAD 731 793 ld r14,PACA_EXCRIT+EX_R14(r13) 732 794 ld r15,PACA_EXCRIT+EX_R15(r13) 733 - bl save_nvgprs 734 795 bl DebugException 735 - b ret_from_except 796 + REST_NVGPRS(r1) 797 + b interrupt_return 736 798 737 799 kernel_dbg_exc: 738 800 b . /* NYI */ ··· 797 859 */ 798 860 mfspr r14,SPRN_DBSR 799 861 EXCEPTION_COMMON_DBG(0xd08) 800 - INTS_DISABLE 801 862 std r14,_DSISR(r1) 802 863 addi r3,r1,STACK_FRAME_OVERHEAD 803 864 ld r14,PACA_EXDBG+EX_R14(r13) 804 865 ld r15,PACA_EXDBG+EX_R15(r13) 805 - bl save_nvgprs 806 866 bl DebugException 807 - b ret_from_except 867 + REST_NVGPRS(r1) 868 + b interrupt_return 808 869 809 870 START_EXCEPTION(perfmon); 810 871 NORMAL_EXCEPTION_PROLOG(0x260, BOOKE_INTERRUPT_PERFORMANCE_MONITOR, 811 872 PROLOG_ADDITION_NONE) 812 873 EXCEPTION_COMMON(0x260) 813 - INTS_DISABLE 814 874 CHECK_NAPPING() 815 875 addi r3,r1,STACK_FRAME_OVERHEAD 816 876 bl performance_monitor_exception 817 - b ret_from_except_lite 877 + b interrupt_return 818 878 819 879 /* Doorbell interrupt */ 820 880 MASKABLE_EXCEPTION(0x280, BOOKE_INTERRUPT_DOORBELL, ··· 823 887 CRIT_EXCEPTION_PROLOG(0x2a0, BOOKE_INTERRUPT_DOORBELL_CRITICAL, 824 888 PROLOG_ADDITION_NONE) 825 889 EXCEPTION_COMMON_CRIT(0x2a0) 826 - bl save_nvgprs 827 890 bl special_reg_save 828 891 CHECK_NAPPING(); 829 892 addi r3,r1,STACK_FRAME_OVERHEAD 830 - bl unknown_exception 893 + bl unknown_nmi_exception 831 894 b ret_from_crit_except 832 895 833 896 /* ··· 838 903 PROLOG_ADDITION_NONE) 839 904 EXCEPTION_COMMON(0x2c0) 840 905 addi r3,r1,STACK_FRAME_OVERHEAD 841 - bl save_nvgprs 842 - INTS_RESTORE_HARD 843 906 bl unknown_exception 844 - b ret_from_except 907 + b interrupt_return 845 908 846 909 /* Guest Doorbell critical Interrupt */ 847 910 START_EXCEPTION(guest_doorbell_crit); 848 911 CRIT_EXCEPTION_PROLOG(0x2e0, BOOKE_INTERRUPT_GUEST_DBELL_CRIT, 849 912 PROLOG_ADDITION_NONE) 850 913 EXCEPTION_COMMON_CRIT(0x2e0) 851 - bl save_nvgprs 852 914 bl special_reg_save 853 915 CHECK_NAPPING(); 854 916 addi r3,r1,STACK_FRAME_OVERHEAD 855 - bl unknown_exception 917 + bl unknown_nmi_exception 856 918 b ret_from_crit_except 857 919 858 920 /* Hypervisor call */ ··· 858 926 PROLOG_ADDITION_NONE) 859 927 EXCEPTION_COMMON(0x310) 860 928 addi r3,r1,STACK_FRAME_OVERHEAD 861 - bl save_nvgprs 862 - INTS_RESTORE_HARD 863 929 bl unknown_exception 864 - b ret_from_except 930 + b interrupt_return 865 931 866 932 /* Embedded Hypervisor priviledged */ 867 933 START_EXCEPTION(ehpriv); ··· 867 937 PROLOG_ADDITION_NONE) 868 938 EXCEPTION_COMMON(0x320) 869 939 addi r3,r1,STACK_FRAME_OVERHEAD 870 - bl save_nvgprs 871 - INTS_RESTORE_HARD 872 940 bl unknown_exception 873 - b ret_from_except 941 + b interrupt_return 874 942 875 943 /* LRAT Error interrupt */ 876 944 START_EXCEPTION(lrat_error); ··· 876 948 PROLOG_ADDITION_NONE) 877 949 EXCEPTION_COMMON(0x340) 878 950 addi r3,r1,STACK_FRAME_OVERHEAD 879 - bl save_nvgprs 880 - INTS_RESTORE_HARD 881 951 bl unknown_exception 882 - b ret_from_except 952 + b interrupt_return 883 953 884 954 /* 885 955 * An interrupt came in while soft-disabled; We mark paca->irq_happened ··· 937 1011 ld r14,PACA_EXGEN+EX_R14(r13) 938 1012 ld r15,PACA_EXGEN+EX_R15(r13) 939 1013 bl do_page_fault 940 - cmpdi r3,0 941 - bne- 1f 942 - b ret_from_except_lite 943 - 1: bl save_nvgprs 944 - mr r4,r3 945 - addi r3,r1,STACK_FRAME_OVERHEAD 946 - bl __bad_page_fault 947 - b ret_from_except 1014 + b interrupt_return 948 1015 949 1016 /* 950 1017 * Alignment exception doesn't fit entirely in the 0x100 bytes so it ··· 949 1030 addi r3,r1,STACK_FRAME_OVERHEAD 950 1031 ld r14,PACA_EXGEN+EX_R14(r13) 951 1032 ld r15,PACA_EXGEN+EX_R15(r13) 952 - bl save_nvgprs 953 - INTS_RESTORE_HARD 954 1033 bl alignment_exception 955 - b ret_from_except 956 - 957 - .align 7 958 - _GLOBAL(ret_from_except) 959 - ld r11,_TRAP(r1) 960 - andi. r0,r11,1 961 - bne ret_from_except_lite 962 1034 REST_NVGPRS(r1) 963 - 964 - _GLOBAL(ret_from_except_lite) 965 - /* 966 - * Disable interrupts so that current_thread_info()->flags 967 - * can't change between when we test it and when we return 968 - * from the interrupt. 969 - */ 970 - wrteei 0 971 - 972 - ld r9, PACA_THREAD_INFO(r13) 973 - ld r3,_MSR(r1) 974 - ld r10,PACACURRENT(r13) 975 - ld r4,TI_FLAGS(r9) 976 - andi. r3,r3,MSR_PR 977 - beq resume_kernel 978 - lwz r3,(THREAD+THREAD_DBCR0)(r10) 979 - 980 - /* Check current_thread_info()->flags */ 981 - andi. r0,r4,_TIF_USER_WORK_MASK 982 - bne 1f 983 - /* 984 - * Check to see if the dbcr0 register is set up to debug. 985 - * Use the internal debug mode bit to do this. 986 - */ 987 - andis. r0,r3,DBCR0_IDM@h 988 - beq restore 989 - mfmsr r0 990 - rlwinm r0,r0,0,~MSR_DE /* Clear MSR.DE */ 991 - mtmsr r0 992 - mtspr SPRN_DBCR0,r3 993 - li r10, -1 994 - mtspr SPRN_DBSR,r10 995 - b restore 996 - 1: andi. r0,r4,_TIF_NEED_RESCHED 997 - beq 2f 998 - bl restore_interrupts 999 - SCHEDULE_USER 1000 - b ret_from_except_lite 1001 - 2: 1002 - bl save_nvgprs 1003 - /* 1004 - * Use a non volatile GPR to save and restore our thread_info flags 1005 - * across the call to restore_interrupts. 1006 - */ 1007 - mr r30,r4 1008 - bl restore_interrupts 1009 - mr r4,r30 1010 - addi r3,r1,STACK_FRAME_OVERHEAD 1011 - bl do_notify_resume 1012 - b ret_from_except 1013 - 1014 - resume_kernel: 1015 - /* check current_thread_info, _TIF_EMULATE_STACK_STORE */ 1016 - andis. r8,r4,_TIF_EMULATE_STACK_STORE@h 1017 - beq+ 1f 1018 - 1019 - addi r8,r1,INT_FRAME_SIZE /* Get the kprobed function entry */ 1020 - 1021 - ld r3,GPR1(r1) 1022 - subi r3,r3,INT_FRAME_SIZE /* dst: Allocate a trampoline exception frame */ 1023 - mr r4,r1 /* src: current exception frame */ 1024 - mr r1,r3 /* Reroute the trampoline frame to r1 */ 1025 - 1026 - /* Copy from the original to the trampoline. */ 1027 - li r5,INT_FRAME_SIZE/8 /* size: INT_FRAME_SIZE */ 1028 - li r6,0 /* start offset: 0 */ 1029 - mtctr r5 1030 - 2: ldx r0,r6,r4 1031 - stdx r0,r6,r3 1032 - addi r6,r6,8 1033 - bdnz 2b 1034 - 1035 - /* Do real store operation to complete stdu */ 1036 - ld r5,GPR1(r1) 1037 - std r8,0(r5) 1038 - 1039 - /* Clear _TIF_EMULATE_STACK_STORE flag */ 1040 - lis r11,_TIF_EMULATE_STACK_STORE@h 1041 - addi r5,r9,TI_FLAGS 1042 - 0: ldarx r4,0,r5 1043 - andc r4,r4,r11 1044 - stdcx. r4,0,r5 1045 - bne- 0b 1046 - 1: 1047 - 1048 - #ifdef CONFIG_PREEMPT 1049 - /* Check if we need to preempt */ 1050 - andi. r0,r4,_TIF_NEED_RESCHED 1051 - beq+ restore 1052 - /* Check that preempt_count() == 0 and interrupts are enabled */ 1053 - lwz r8,TI_PREEMPT(r9) 1054 - cmpwi cr0,r8,0 1055 - bne restore 1056 - ld r0,SOFTE(r1) 1057 - andi. r0,r0,IRQS_DISABLED 1058 - bne restore 1059 - 1060 - /* 1061 - * Here we are preempting the current task. We want to make 1062 - * sure we are soft-disabled first and reconcile irq state. 1063 - */ 1064 - RECONCILE_IRQ_STATE(r3,r4) 1065 - bl preempt_schedule_irq 1066 - 1067 - /* 1068 - * arch_local_irq_restore() from preempt_schedule_irq above may 1069 - * enable hard interrupt but we really should disable interrupts 1070 - * when we return from the interrupt, and so that we don't get 1071 - * interrupted after loading SRR0/1. 1072 - */ 1073 - wrteei 0 1074 - #endif /* CONFIG_PREEMPT */ 1075 - 1076 - restore: 1077 - /* 1078 - * This is the main kernel exit path. First we check if we 1079 - * are about to re-enable interrupts 1080 - */ 1081 - ld r5,SOFTE(r1) 1082 - lbz r6,PACAIRQSOFTMASK(r13) 1083 - andi. r5,r5,IRQS_DISABLED 1084 - bne .Lrestore_irq_off 1085 - 1086 - /* We are enabling, were we already enabled ? Yes, just return */ 1087 - andi. r6,r6,IRQS_DISABLED 1088 - beq cr0,fast_exception_return 1089 - 1090 - /* 1091 - * We are about to soft-enable interrupts (we are hard disabled 1092 - * at this point). We check if there's anything that needs to 1093 - * be replayed first. 1094 - */ 1095 - lbz r0,PACAIRQHAPPENED(r13) 1096 - cmpwi cr0,r0,0 1097 - bne- .Lrestore_check_irq_replay 1098 - 1099 - /* 1100 - * Get here when nothing happened while soft-disabled, just 1101 - * soft-enable and move-on. We will hard-enable as a side 1102 - * effect of rfi 1103 - */ 1104 - .Lrestore_no_replay: 1105 - TRACE_ENABLE_INTS 1106 - li r0,IRQS_ENABLED 1107 - stb r0,PACAIRQSOFTMASK(r13); 1108 - 1109 - /* This is the return from load_up_fpu fast path which could do with 1110 - * less GPR restores in fact, but for now we have a single return path 1111 - */ 1112 - fast_exception_return: 1113 - wrteei 0 1114 - 1: mr r0,r13 1115 - ld r10,_MSR(r1) 1116 - REST_4GPRS(2, r1) 1117 - andi. r6,r10,MSR_PR 1118 - REST_2GPRS(6, r1) 1119 - beq 1f 1120 - ACCOUNT_CPU_USER_EXIT(r13, r10, r11) 1121 - ld r0,GPR13(r1) 1122 - 1123 - 1: stdcx. r0,0,r1 /* to clear the reservation */ 1124 - 1125 - ld r8,_CCR(r1) 1126 - ld r9,_LINK(r1) 1127 - ld r10,_CTR(r1) 1128 - ld r11,_XER(r1) 1129 - mtcr r8 1130 - mtlr r9 1131 - mtctr r10 1132 - mtxer r11 1133 - REST_2GPRS(8, r1) 1134 - ld r10,GPR10(r1) 1135 - ld r11,GPR11(r1) 1136 - ld r12,GPR12(r1) 1137 - mtspr SPRN_SPRG_GEN_SCRATCH,r0 1138 - 1139 - std r10,PACA_EXGEN+EX_R10(r13); 1140 - std r11,PACA_EXGEN+EX_R11(r13); 1141 - ld r10,_NIP(r1) 1142 - ld r11,_MSR(r1) 1143 - ld r0,GPR0(r1) 1144 - ld r1,GPR1(r1) 1145 - mtspr SPRN_SRR0,r10 1146 - mtspr SPRN_SRR1,r11 1147 - ld r10,PACA_EXGEN+EX_R10(r13) 1148 - ld r11,PACA_EXGEN+EX_R11(r13) 1149 - mfspr r13,SPRN_SPRG_GEN_SCRATCH 1150 - rfi 1151 - 1152 - /* 1153 - * We are returning to a context with interrupts soft disabled. 1154 - * 1155 - * However, we may also about to hard enable, so we need to 1156 - * make sure that in this case, we also clear PACA_IRQ_HARD_DIS 1157 - * or that bit can get out of sync and bad things will happen 1158 - */ 1159 - .Lrestore_irq_off: 1160 - ld r3,_MSR(r1) 1161 - lbz r7,PACAIRQHAPPENED(r13) 1162 - andi. r0,r3,MSR_EE 1163 - beq 1f 1164 - rlwinm r7,r7,0,~PACA_IRQ_HARD_DIS 1165 - stb r7,PACAIRQHAPPENED(r13) 1166 - 1: 1167 - #if defined(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG) && defined(CONFIG_BUG) 1168 - /* The interrupt should not have soft enabled. */ 1169 - lbz r7,PACAIRQSOFTMASK(r13) 1170 - 1: tdeqi r7,IRQS_ENABLED 1171 - EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,BUGFLAG_WARNING 1172 - #endif 1173 - b fast_exception_return 1174 - 1175 - /* 1176 - * Something did happen, check if a re-emit is needed 1177 - * (this also clears paca->irq_happened) 1178 - */ 1179 - .Lrestore_check_irq_replay: 1180 - /* XXX: We could implement a fast path here where we check 1181 - * for irq_happened being just 0x01, in which case we can 1182 - * clear it and return. That means that we would potentially 1183 - * miss a decrementer having wrapped all the way around. 1184 - * 1185 - * Still, this might be useful for things like hash_page 1186 - */ 1187 - bl __check_irq_replay 1188 - cmpwi cr0,r3,0 1189 - beq .Lrestore_no_replay 1190 - 1191 - /* 1192 - * We need to re-emit an interrupt. We do so by re-using our 1193 - * existing exception frame. We first change the trap value, 1194 - * but we need to ensure we preserve the low nibble of it 1195 - */ 1196 - ld r4,_TRAP(r1) 1197 - clrldi r4,r4,60 1198 - or r4,r4,r3 1199 - std r4,_TRAP(r1) 1200 - 1201 - /* 1202 - * PACA_IRQ_HARD_DIS won't always be set here, so set it now 1203 - * to reconcile the IRQ state. Tracing is already accounted for. 1204 - */ 1205 - lbz r4,PACAIRQHAPPENED(r13) 1206 - ori r4,r4,PACA_IRQ_HARD_DIS 1207 - stb r4,PACAIRQHAPPENED(r13) 1208 - 1209 - /* 1210 - * Then find the right handler and call it. Interrupts are 1211 - * still soft-disabled and we keep them that way. 1212 - */ 1213 - cmpwi cr0,r3,0x500 1214 - bne 1f 1215 - addi r3,r1,STACK_FRAME_OVERHEAD; 1216 - bl do_IRQ 1217 - b ret_from_except 1218 - 1: cmpwi cr0,r3,0x900 1219 - bne 1f 1220 - addi r3,r1,STACK_FRAME_OVERHEAD; 1221 - bl timer_interrupt 1222 - b ret_from_except 1223 - #ifdef CONFIG_PPC_DOORBELL 1224 - 1: 1225 - cmpwi cr0,r3,0x280 1226 - bne 1f 1227 - addi r3,r1,STACK_FRAME_OVERHEAD; 1228 - bl doorbell_exception 1229 - #endif /* CONFIG_PPC_DOORBELL */ 1230 - 1: b ret_from_except /* What else to do here ? */ 1231 - 1232 - _ASM_NOKPROBE_SYMBOL(ret_from_except); 1233 - _ASM_NOKPROBE_SYMBOL(ret_from_except_lite); 1234 - _ASM_NOKPROBE_SYMBOL(resume_kernel); 1235 - _ASM_NOKPROBE_SYMBOL(restore); 1236 - _ASM_NOKPROBE_SYMBOL(fast_exception_return); 1035 + b interrupt_return 1237 1036 1238 1037 /* 1239 1038 * Trampolines used when spotting a bad kernel stack pointer in

+7 -53

arch/powerpc/kernel/exceptions-64s.S

··· 693 693 .endm 694 694 695 695 /* 696 - * When the idle code in power4_idle puts the CPU into NAP mode, 697 - * it has to do so in a loop, and relies on the external interrupt 698 - * and decrementer interrupt entry code to get it out of the loop. 699 - * It sets the _TLF_NAPPING bit in current_thread_info()->local_flags 700 - * to signal that it is in the loop and needs help to get out. 701 - */ 702 - #ifdef CONFIG_PPC_970_NAP 703 - #define FINISH_NAP \ 704 - BEGIN_FTR_SECTION \ 705 - ld r11, PACA_THREAD_INFO(r13); \ 706 - ld r9,TI_LOCAL_FLAGS(r11); \ 707 - andi. r10,r9,_TLF_NAPPING; \ 708 - bnel power4_fixup_nap; \ 709 - END_FTR_SECTION_IFSET(CPU_FTR_CAN_NAP) 710 - #else 711 - #define FINISH_NAP 712 - #endif 713 - 714 - /* 715 696 * There are a few constraints to be concerned with. 716 697 * - Real mode exceptions code/data must be located at their physical location. 717 698 * - Virtual mode exceptions must be mapped at their 0xc000... location. ··· 1229 1248 */ 1230 1249 GEN_COMMON machine_check 1231 1250 1232 - FINISH_NAP 1233 1251 /* Enable MSR_RI when finished with PACA_EXMC */ 1234 1252 li r10,MSR_RI 1235 1253 mtmsrd r10,1 ··· 1551 1571 EXC_VIRT_END(hardware_interrupt, 0x4500, 0x100) 1552 1572 EXC_COMMON_BEGIN(hardware_interrupt_common) 1553 1573 GEN_COMMON hardware_interrupt 1554 - FINISH_NAP 1555 1574 addi r3,r1,STACK_FRAME_OVERHEAD 1556 1575 bl do_IRQ 1557 1576 b interrupt_return ··· 1780 1801 EXC_VIRT_END(decrementer, 0x4900, 0x80) 1781 1802 EXC_COMMON_BEGIN(decrementer_common) 1782 1803 GEN_COMMON decrementer 1783 - FINISH_NAP 1784 1804 addi r3,r1,STACK_FRAME_OVERHEAD 1785 1805 bl timer_interrupt 1786 1806 b interrupt_return ··· 1864 1886 EXC_VIRT_END(doorbell_super, 0x4a00, 0x100) 1865 1887 EXC_COMMON_BEGIN(doorbell_super_common) 1866 1888 GEN_COMMON doorbell_super 1867 - FINISH_NAP 1868 1889 addi r3,r1,STACK_FRAME_OVERHEAD 1869 1890 #ifdef CONFIG_PPC_DOORBELL 1870 1891 bl doorbell_exception ··· 2214 2237 2215 2238 EXC_COMMON_BEGIN(hmi_exception_common) 2216 2239 GEN_COMMON hmi_exception 2217 - FINISH_NAP 2218 2240 addi r3,r1,STACK_FRAME_OVERHEAD 2219 2241 bl handle_hmi_exception 2220 2242 b interrupt_return ··· 2242 2266 EXC_VIRT_END(h_doorbell, 0x4e80, 0x20) 2243 2267 EXC_COMMON_BEGIN(h_doorbell_common) 2244 2268 GEN_COMMON h_doorbell 2245 - FINISH_NAP 2246 2269 addi r3,r1,STACK_FRAME_OVERHEAD 2247 2270 #ifdef CONFIG_PPC_DOORBELL 2248 2271 bl doorbell_exception ··· 2274 2299 EXC_VIRT_END(h_virt_irq, 0x4ea0, 0x20) 2275 2300 EXC_COMMON_BEGIN(h_virt_irq_common) 2276 2301 GEN_COMMON h_virt_irq 2277 - FINISH_NAP 2278 2302 addi r3,r1,STACK_FRAME_OVERHEAD 2279 2303 bl do_IRQ 2280 2304 b interrupt_return ··· 2319 2345 EXC_VIRT_END(performance_monitor, 0x4f00, 0x20) 2320 2346 EXC_COMMON_BEGIN(performance_monitor_common) 2321 2347 GEN_COMMON performance_monitor 2322 - FINISH_NAP 2323 2348 addi r3,r1,STACK_FRAME_OVERHEAD 2324 2349 bl performance_monitor_exception 2325 2350 b interrupt_return ··· 2503 2530 INT_DEFINE_BEGIN(cbe_system_error) 2504 2531 IVEC=0x1200 2505 2532 IHSRR=1 2506 - IKVM_SKIP=1 2507 - IKVM_REAL=1 2508 2533 INT_DEFINE_END(cbe_system_error) 2509 2534 2510 2535 EXC_REAL_BEGIN(cbe_system_error, 0x1200, 0x100) ··· 2522 2551 EXC_VIRT_NONE(0x5200, 0x100) 2523 2552 #endif 2524 2553 2525 - 2554 + /** 2555 + * Interrupt 0x1300 - Instruction Address Breakpoint Interrupt. 2556 + * This has been removed from the ISA before 2.01, which is the earliest 2557 + * 64-bit BookS ISA supported, however the G5 / 970 implements this 2558 + * interrupt with a non-architected feature available through the support 2559 + * processor interface. 2560 + */ 2526 2561 INT_DEFINE_BEGIN(instruction_breakpoint) 2527 2562 IVEC=0x1300 2528 2563 #ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 2529 - IKVM_SKIP=1 2530 2564 IKVM_REAL=1 2531 2565 #endif 2532 2566 INT_DEFINE_END(instruction_breakpoint) ··· 2677 2701 INT_DEFINE_BEGIN(cbe_maintenance) 2678 2702 IVEC=0x1600 2679 2703 IHSRR=1 2680 - IKVM_SKIP=1 2681 - IKVM_REAL=1 2682 2704 INT_DEFINE_END(cbe_maintenance) 2683 2705 2684 2706 EXC_REAL_BEGIN(cbe_maintenance, 0x1600, 0x100) ··· 2728 2754 INT_DEFINE_BEGIN(cbe_thermal) 2729 2755 IVEC=0x1800 2730 2756 IHSRR=1 2731 - IKVM_SKIP=1 2732 - IKVM_REAL=1 2733 2757 INT_DEFINE_END(cbe_thermal) 2734 2758 2735 2759 EXC_REAL_BEGIN(cbe_thermal, 0x1800, 0x100) ··· 3067 3095 .globl __end_interrupts 3068 3096 __end_interrupts: 3069 3097 DEFINE_FIXED_SYMBOL(__end_interrupts) 3070 - 3071 - #ifdef CONFIG_PPC_970_NAP 3072 - /* 3073 - * Called by exception entry code if _TLF_NAPPING was set, this clears 3074 - * the NAPPING flag, and redirects the exception exit to 3075 - * power4_fixup_nap_return. 3076 - */ 3077 - .globl power4_fixup_nap 3078 - EXC_COMMON_BEGIN(power4_fixup_nap) 3079 - andc r9,r9,r10 3080 - std r9,TI_LOCAL_FLAGS(r11) 3081 - LOAD_REG_ADDR(r10, power4_idle_nap_return) 3082 - std r10,_NIP(r1) 3083 - blr 3084 - 3085 - power4_idle_nap_return: 3086 - blr 3087 - #endif 3088 3098 3089 3099 CLOSE_FIXED_SECTION(real_vectors); 3090 3100 CLOSE_FIXED_SECTION(real_trampolines);

+9 -9

arch/powerpc/kernel/fadump.c

··· 31 31 #include <asm/fadump.h> 32 32 #include <asm/fadump-internal.h> 33 33 #include <asm/setup.h> 34 + #include <asm/interrupt.h> 34 35 35 36 /* 36 37 * The CPU who acquired the lock to trigger the fadump crash should ··· 45 44 46 45 static void __init fadump_reserve_crash_area(u64 base); 47 46 48 - struct kobject *fadump_kobj; 49 - 50 47 #ifndef CONFIG_PRESERVE_FA_DUMP 48 + 49 + static struct kobject *fadump_kobj; 51 50 52 51 static atomic_t cpus_in_fadump; 53 52 static DEFINE_MUTEX(fadump_mutex); 54 53 55 - struct fadump_mrange_info crash_mrange_info = { "crash", NULL, 0, 0, 0, false }; 54 + static struct fadump_mrange_info crash_mrange_info = { "crash", NULL, 0, 0, 0, false }; 56 55 57 56 #define RESERVED_RNGS_SZ 16384 /* 16K - 128 entries */ 58 57 #define RESERVED_RNGS_CNT (RESERVED_RNGS_SZ / \ 59 58 sizeof(struct fadump_memory_range)) 60 59 static struct fadump_memory_range rngs[RESERVED_RNGS_CNT]; 61 - struct fadump_mrange_info reserved_mrange_info = { "reserved", rngs, 62 - RESERVED_RNGS_SZ, 0, 63 - RESERVED_RNGS_CNT, true }; 60 + static struct fadump_mrange_info 61 + reserved_mrange_info = { "reserved", rngs, RESERVED_RNGS_SZ, 0, RESERVED_RNGS_CNT, true }; 64 62 65 63 static void __init early_init_dt_scan_reserved_ranges(unsigned long node); 66 64 ··· 79 79 * But for some reason even if it fails we still have the memory reservation 80 80 * with us and we can still continue doing fadump. 81 81 */ 82 - int __init fadump_cma_init(void) 82 + static int __init fadump_cma_init(void) 83 83 { 84 84 unsigned long long base, size; 85 85 int rc; ··· 292 292 * that is required for a kernel to boot successfully. 293 293 * 294 294 */ 295 - static inline u64 fadump_calculate_reserve_size(void) 295 + static __init u64 fadump_calculate_reserve_size(void) 296 296 { 297 297 u64 base, size, bootmem_min; 298 298 int ret; ··· 728 728 * If we came in via system reset, wait a while for the secondary 729 729 * CPUs to enter. 730 730 */ 731 - if (TRAP(&(fdh->regs)) == 0x100) { 731 + if (TRAP(&(fdh->regs)) == INTERRUPT_SYSTEM_RESET) { 732 732 msecs = CRASH_TIMEOUT; 733 733 while ((atomic_read(&cpus_in_fadump) < ncpus) && (--msecs > 0)) 734 734 mdelay(1);

-2

arch/powerpc/kernel/fpu.S

··· 92 92 /* enable use of FP after return */ 93 93 #ifdef CONFIG_PPC32 94 94 mfspr r5,SPRN_SPRG_THREAD /* current task's THREAD (phys) */ 95 - #ifdef CONFIG_VMAP_STACK 96 95 tovirt(r5, r5) 97 - #endif 98 96 lwz r4,THREAD_FPEXC_MODE(r5) 99 97 ori r9,r9,MSR_FP /* enable FP for current */ 100 98 or r9,r9,r4

+79 -119

arch/powerpc/kernel/head_32.h

··· 10 10 * We assume sprg3 has the physical address of the current 11 11 * task's thread_struct. 12 12 */ 13 - .macro EXCEPTION_PROLOG handle_dar_dsisr=0 13 + .macro EXCEPTION_PROLOG trapno name handle_dar_dsisr=0 14 14 EXCEPTION_PROLOG_0 handle_dar_dsisr=\handle_dar_dsisr 15 15 EXCEPTION_PROLOG_1 16 - EXCEPTION_PROLOG_2 handle_dar_dsisr=\handle_dar_dsisr 16 + EXCEPTION_PROLOG_2 \trapno \name handle_dar_dsisr=\handle_dar_dsisr 17 17 .endm 18 18 19 19 .macro EXCEPTION_PROLOG_0 handle_dar_dsisr=0 20 20 mtspr SPRN_SPRG_SCRATCH0,r10 21 21 mtspr SPRN_SPRG_SCRATCH1,r11 22 - #ifdef CONFIG_VMAP_STACK 23 22 mfspr r10, SPRN_SPRG_THREAD 24 23 .if \handle_dar_dsisr 24 + #ifdef CONFIG_40x 25 + mfspr r11, SPRN_DEAR 26 + #else 25 27 mfspr r11, SPRN_DAR 28 + #endif 26 29 stw r11, DAR(r10) 30 + #ifdef CONFIG_40x 31 + mfspr r11, SPRN_ESR 32 + #else 27 33 mfspr r11, SPRN_DSISR 34 + #endif 28 35 stw r11, DSISR(r10) 29 36 .endif 30 37 mfspr r11, SPRN_SRR0 31 38 stw r11, SRR0(r10) 32 - #endif 33 39 mfspr r11, SPRN_SRR1 /* check whether user or kernel */ 34 - #ifdef CONFIG_VMAP_STACK 35 40 stw r11, SRR1(r10) 36 - #endif 37 41 mfcr r10 38 42 andi. r11, r11, MSR_PR 39 43 .endm 40 44 41 - .macro EXCEPTION_PROLOG_1 for_rtas=0 42 - #ifdef CONFIG_VMAP_STACK 45 + .macro EXCEPTION_PROLOG_1 43 46 mtspr SPRN_SPRG_SCRATCH2,r1 44 47 subi r1, r1, INT_FRAME_SIZE /* use r1 if kernel */ 45 48 beq 1f ··· 50 47 lwz r1,TASK_STACK-THREAD(r1) 51 48 addi r1, r1, THREAD_SIZE - INT_FRAME_SIZE 52 49 1: 50 + #ifdef CONFIG_VMAP_STACK 53 51 mtcrf 0x3f, r1 54 - bt 32 - THREAD_ALIGN_SHIFT, stack_overflow 55 - #else 56 - subi r11, r1, INT_FRAME_SIZE /* use r1 if kernel */ 57 - beq 1f 58 - mfspr r11,SPRN_SPRG_THREAD 59 - lwz r11,TASK_STACK-THREAD(r11) 60 - addi r11, r11, THREAD_SIZE - INT_FRAME_SIZE 61 - 1: tophys(r11, r11) 52 + bt 32 - THREAD_ALIGN_SHIFT, vmap_stack_overflow 62 53 #endif 63 54 .endm 64 55 65 - .macro EXCEPTION_PROLOG_2 handle_dar_dsisr=0 66 - #ifdef CONFIG_VMAP_STACK 67 - li r11, MSR_KERNEL & ~(MSR_IR | MSR_RI) /* can take DTLB miss */ 68 - mtmsr r11 69 - isync 56 + .macro EXCEPTION_PROLOG_2 trapno name handle_dar_dsisr=0 57 + #ifdef CONFIG_PPC_8xx 58 + .if \handle_dar_dsisr 59 + li r11, RPN_PATTERN 60 + mtspr SPRN_DAR, r11 /* Tag DAR, to be used in DTLB Error */ 61 + .endif 62 + #endif 63 + LOAD_REG_IMMEDIATE(r11, MSR_KERNEL & ~MSR_RI) /* re-enable MMU */ 64 + mtspr SPRN_SRR1, r11 65 + lis r11, 1f@h 66 + ori r11, r11, 1f@l 67 + mtspr SPRN_SRR0, r11 70 68 mfspr r11, SPRN_SPRG_SCRATCH2 69 + rfi 70 + 71 + .text 72 + \name\()_virt: 73 + 1: 71 74 stw r11,GPR1(r1) 72 75 stw r11,0(r1) 73 76 mr r11, r1 74 - #else 75 - stw r1,GPR1(r11) 76 - stw r1,0(r11) 77 - tovirt(r1, r11) /* set new kernel sp */ 78 - #endif 79 77 stw r10,_CCR(r11) /* save registers */ 80 78 stw r12,GPR12(r11) 81 79 stw r9,GPR9(r11) ··· 86 82 stw r12,GPR11(r11) 87 83 mflr r10 88 84 stw r10,_LINK(r11) 89 - #ifdef CONFIG_VMAP_STACK 90 85 mfspr r12, SPRN_SPRG_THREAD 91 86 tovirt(r12, r12) 92 87 .if \handle_dar_dsisr ··· 96 93 .endif 97 94 lwz r9, SRR1(r12) 98 95 lwz r12, SRR0(r12) 99 - #else 100 - mfspr r12,SPRN_SRR0 101 - mfspr r9,SPRN_SRR1 102 - #endif 103 96 #ifdef CONFIG_40x 104 97 rlwinm r9,r9,0,14,12 /* clear MSR_WE (necessary?) */ 98 + #elif defined(CONFIG_PPC_8xx) 99 + mtspr SPRN_EID, r2 /* Set MSR_RI */ 105 100 #else 106 - #ifdef CONFIG_VMAP_STACK 107 - li r10, MSR_KERNEL & ~MSR_IR /* can take exceptions */ 108 - #else 109 - li r10,MSR_KERNEL & ~(MSR_IR|MSR_DR) /* can take exceptions */ 110 - #endif 101 + li r10, MSR_KERNEL /* can take exceptions */ 111 102 mtmsr r10 /* (except for mach check in rtas) */ 112 103 #endif 113 - stw r0,GPR0(r11) 104 + COMMON_EXCEPTION_PROLOG_END \trapno 105 + _ASM_NOKPROBE_SYMBOL(\name\()_virt) 106 + .endm 107 + 108 + .macro COMMON_EXCEPTION_PROLOG_END trapno 109 + stw r0,GPR0(r1) 114 110 lis r10,STACK_FRAME_REGS_MARKER@ha /* exception frame marker */ 115 111 addi r10,r10,STACK_FRAME_REGS_MARKER@l 116 - stw r10,8(r11) 117 - SAVE_4GPRS(3, r11) 118 - SAVE_2GPRS(7, r11) 112 + stw r10,8(r1) 113 + li r10, \trapno 114 + stw r10,_TRAP(r1) 115 + SAVE_4GPRS(3, r1) 116 + SAVE_2GPRS(7, r1) 117 + SAVE_NVGPRS(r1) 118 + stw r2,GPR2(r1) 119 + stw r12,_NIP(r1) 120 + stw r9,_MSR(r1) 121 + mfctr r10 122 + mfspr r2,SPRN_SPRG_THREAD 123 + stw r10,_CTR(r1) 124 + tovirt(r2, r2) 125 + mfspr r10,SPRN_XER 126 + addi r2, r2, -THREAD 127 + stw r10,_XER(r1) 128 + addi r3,r1,STACK_FRAME_OVERHEAD 129 + .endm 130 + 131 + .macro prepare_transfer_to_handler 132 + #ifdef CONFIG_PPC_BOOK3S_32 133 + andi. r12,r9,MSR_PR 134 + bne 777f 135 + bl prepare_transfer_to_handler 136 + 777: 137 + #endif 119 138 .endm 120 139 121 140 .macro SYSCALL_ENTRY trapno ··· 181 156 b transfer_to_syscall /* jump to handler */ 182 157 .endm 183 158 184 - .macro save_dar_dsisr_on_stack reg1, reg2, sp 185 - #ifndef CONFIG_VMAP_STACK 186 - mfspr \reg1, SPRN_DAR 187 - mfspr \reg2, SPRN_DSISR 188 - stw \reg1, _DAR(\sp) 189 - stw \reg2, _DSISR(\sp) 190 - #endif 191 - .endm 192 - 193 - .macro get_and_save_dar_dsisr_on_stack reg1, reg2, sp 194 - #ifdef CONFIG_VMAP_STACK 195 - lwz \reg1, _DAR(\sp) 196 - lwz \reg2, _DSISR(\sp) 197 - #else 198 - save_dar_dsisr_on_stack \reg1, \reg2, \sp 199 - #endif 200 - .endm 201 - 202 - .macro tovirt_vmstack dst, src 203 - #ifdef CONFIG_VMAP_STACK 204 - tovirt(\dst, \src) 205 - #else 206 - .ifnc \dst, \src 207 - mr \dst, \src 208 - .endif 209 - #endif 210 - .endm 211 - 212 - .macro tovirt_novmstack dst, src 213 - #ifndef CONFIG_VMAP_STACK 214 - tovirt(\dst, \src) 215 - #else 216 - .ifnc \dst, \src 217 - mr \dst, \src 218 - .endif 219 - #endif 220 - .endm 221 - 222 - .macro tophys_novmstack dst, src 223 - #ifndef CONFIG_VMAP_STACK 224 - tophys(\dst, \src) 225 - #else 226 - .ifnc \dst, \src 227 - mr \dst, \src 228 - .endif 229 - #endif 230 - .endm 231 - 232 159 /* 233 160 * Note: code which follows this uses cr0.eq (set if from kernel), 234 161 * r11, r12 (SRR0), and r9 (SRR1). ··· 194 217 */ 195 218 #ifdef CONFIG_PPC_BOOK3S 196 219 #define START_EXCEPTION(n, label) \ 220 + __HEAD; \ 197 221 . = n; \ 198 222 DO_KVM n; \ 199 223 label: 200 224 201 225 #else 202 226 #define START_EXCEPTION(n, label) \ 227 + __HEAD; \ 203 228 . = n; \ 204 229 label: 205 230 206 231 #endif 207 232 208 - #define EXCEPTION(n, label, hdlr, xfer) \ 233 + #define EXCEPTION(n, label, hdlr) \ 209 234 START_EXCEPTION(n, label) \ 210 - EXCEPTION_PROLOG; \ 211 - addi r3,r1,STACK_FRAME_OVERHEAD; \ 212 - xfer(n, hdlr) 213 - 214 - #define EXC_XFER_TEMPLATE(hdlr, trap, msr, tfer, ret) \ 215 - li r10,trap; \ 216 - stw r10,_TRAP(r11); \ 217 - LOAD_REG_IMMEDIATE(r10, msr); \ 218 - bl tfer; \ 219 - .long hdlr; \ 220 - .long ret 221 - 222 - #define EXC_XFER_STD(n, hdlr) \ 223 - EXC_XFER_TEMPLATE(hdlr, n, MSR_KERNEL, transfer_to_handler_full, \ 224 - ret_from_except_full) 225 - 226 - #define EXC_XFER_LITE(n, hdlr) \ 227 - EXC_XFER_TEMPLATE(hdlr, n+1, MSR_KERNEL, transfer_to_handler, \ 228 - ret_from_except) 235 + EXCEPTION_PROLOG n label; \ 236 + prepare_transfer_to_handler; \ 237 + bl hdlr; \ 238 + b interrupt_return 229 239 230 240 .macro vmap_stack_overflow_exception 231 - #ifdef CONFIG_VMAP_STACK 241 + __HEAD 242 + vmap_stack_overflow: 232 243 #ifdef CONFIG_SMP 233 244 mfspr r1, SPRN_SPRG_THREAD 234 245 lwz r1, TASK_CPU - THREAD(r1) ··· 226 261 lis r1, emergency_ctx@ha 227 262 #endif 228 263 lwz r1, emergency_ctx@l(r1) 229 - cmpwi cr1, r1, 0 230 - bne cr1, 1f 231 - lis r1, init_thread_union@ha 232 - addi r1, r1, init_thread_union@l 233 - 1: addi r1, r1, THREAD_SIZE - INT_FRAME_SIZE 234 - EXCEPTION_PROLOG_2 235 - SAVE_NVGPRS(r11) 236 - addi r3, r1, STACK_FRAME_OVERHEAD 237 - EXC_XFER_STD(0, stack_overflow_exception) 238 - #endif 264 + addi r1, r1, THREAD_SIZE - INT_FRAME_SIZE 265 + EXCEPTION_PROLOG_2 0 vmap_stack_overflow 266 + prepare_transfer_to_handler 267 + bl stack_overflow_exception 268 + b interrupt_return 239 269 .endm 240 270 241 271 #endif /* __HEAD_32_H__ */

+147 -124

arch/powerpc/kernel/head_40x.S

··· 89 89 .space 4 90 90 _ENTRY(crit_srr1) 91 91 .space 4 92 - _ENTRY(saved_ksp_limit) 92 + _ENTRY(crit_r1) 93 + .space 4 94 + _ENTRY(crit_dear) 95 + .space 4 96 + _ENTRY(crit_esr) 93 97 .space 4 94 98 95 99 /* ··· 104 100 * Instead we use a couple of words of memory at low physical addresses. 105 101 * This is OK since we don't support SMP on these processors. 106 102 */ 107 - #define CRITICAL_EXCEPTION_PROLOG \ 108 - stw r10,crit_r10@l(0); /* save two registers to work with */\ 109 - stw r11,crit_r11@l(0); \ 110 - mfcr r10; /* save CR in r10 for now */\ 111 - mfspr r11,SPRN_SRR3; /* check whether user or kernel */\ 112 - andi. r11,r11,MSR_PR; \ 113 - lis r11,critirq_ctx@ha; \ 114 - tophys(r11,r11); \ 115 - lwz r11,critirq_ctx@l(r11); \ 116 - beq 1f; \ 117 - /* COMING FROM USER MODE */ \ 118 - mfspr r11,SPRN_SPRG_THREAD; /* if from user, start at top of */\ 119 - lwz r11,TASK_STACK-THREAD(r11); /* this thread's kernel stack */\ 120 - 1: addi r11,r11,THREAD_SIZE-INT_FRAME_SIZE; /* Alloc an excpt frm */\ 121 - tophys(r11,r11); \ 122 - stw r10,_CCR(r11); /* save various registers */\ 123 - stw r12,GPR12(r11); \ 124 - stw r9,GPR9(r11); \ 125 - mflr r10; \ 126 - stw r10,_LINK(r11); \ 127 - mfspr r12,SPRN_DEAR; /* save DEAR and ESR in the frame */\ 128 - stw r12,_DEAR(r11); /* since they may have had stuff */\ 129 - mfspr r9,SPRN_ESR; /* in them at the point where the */\ 130 - stw r9,_ESR(r11); /* exception was taken */\ 131 - mfspr r12,SPRN_SRR2; \ 132 - stw r1,GPR1(r11); \ 133 - mfspr r9,SPRN_SRR3; \ 134 - stw r1,0(r11); \ 135 - tovirt(r1,r11); \ 136 - rlwinm r9,r9,0,14,12; /* clear MSR_WE (necessary?) */\ 137 - stw r0,GPR0(r11); \ 138 - lis r10, STACK_FRAME_REGS_MARKER@ha; /* exception frame marker */\ 139 - addi r10, r10, STACK_FRAME_REGS_MARKER@l; \ 140 - stw r10, 8(r11); \ 141 - SAVE_4GPRS(3, r11); \ 142 - SAVE_2GPRS(7, r11) 103 + .macro CRITICAL_EXCEPTION_PROLOG trapno name 104 + stw r10,crit_r10@l(0) /* save two registers to work with */ 105 + stw r11,crit_r11@l(0) 106 + mfspr r10,SPRN_SRR0 107 + mfspr r11,SPRN_SRR1 108 + stw r10,crit_srr0@l(0) 109 + stw r11,crit_srr1@l(0) 110 + mfspr r10,SPRN_DEAR 111 + mfspr r11,SPRN_ESR 112 + stw r10,crit_dear@l(0) 113 + stw r11,crit_esr@l(0) 114 + mfcr r10 /* save CR in r10 for now */ 115 + mfspr r11,SPRN_SRR3 /* check whether user or kernel */ 116 + andi. r11,r11,MSR_PR 117 + lis r11,(critirq_ctx-PAGE_OFFSET)@ha 118 + lwz r11,(critirq_ctx-PAGE_OFFSET)@l(r11) 119 + beq 1f 120 + /* COMING FROM USER MODE */ 121 + mfspr r11,SPRN_SPRG_THREAD /* if from user, start at top of */ 122 + lwz r11,TASK_STACK-THREAD(r11) /* this thread's kernel stack */ 123 + 1: stw r1,crit_r1@l(0) 124 + addi r1,r11,THREAD_SIZE-INT_FRAME_SIZE /* Alloc an excpt frm */ 125 + LOAD_REG_IMMEDIATE(r11, MSR_KERNEL & ~(MSR_ME|MSR_DE|MSR_CE)) /* re-enable MMU */ 126 + mtspr SPRN_SRR1, r11 127 + lis r11, 1f@h 128 + ori r11, r11, 1f@l 129 + mtspr SPRN_SRR0, r11 130 + rfi 131 + 132 + .text 133 + 1: 134 + \name\()_virt: 135 + lwz r11,crit_r1@l(0) 136 + stw r11,GPR1(r1) 137 + stw r11,0(r1) 138 + mr r11,r1 139 + stw r10,_CCR(r11) /* save various registers */ 140 + stw r12,GPR12(r11) 141 + stw r9,GPR9(r11) 142 + mflr r10 143 + stw r10,_LINK(r11) 144 + lis r9,PAGE_OFFSET@ha 145 + lwz r10,crit_r10@l(r9) 146 + lwz r12,crit_r11@l(r9) 147 + stw r10,GPR10(r11) 148 + stw r12,GPR11(r11) 149 + lwz r12,crit_dear@l(r9) 150 + lwz r9,crit_esr@l(r9) 151 + stw r12,_DEAR(r11) /* since they may have had stuff */ 152 + stw r9,_ESR(r11) /* exception was taken */ 153 + mfspr r12,SPRN_SRR2 154 + mfspr r9,SPRN_SRR3 155 + rlwinm r9,r9,0,14,12 /* clear MSR_WE (necessary?) */ 156 + COMMON_EXCEPTION_PROLOG_END \trapno + 2 157 + _ASM_NOKPROBE_SYMBOL(\name\()_virt) 158 + .endm 143 159 144 160 /* 145 161 * State at this point: ··· 179 155 */ 180 156 #define CRITICAL_EXCEPTION(n, label, hdlr) \ 181 157 START_EXCEPTION(n, label); \ 182 - CRITICAL_EXCEPTION_PROLOG; \ 183 - addi r3,r1,STACK_FRAME_OVERHEAD; \ 184 - EXC_XFER_TEMPLATE(hdlr, n+2, (MSR_KERNEL & ~(MSR_ME|MSR_DE|MSR_CE)), \ 185 - crit_transfer_to_handler, ret_from_crit_exc) 158 + CRITICAL_EXCEPTION_PROLOG n label; \ 159 + prepare_transfer_to_handler; \ 160 + bl hdlr; \ 161 + b ret_from_crit_exc 186 162 187 163 /* 188 164 * 0x0100 - Critical Interrupt Exception ··· 202 178 * if they can't resolve the lightweight TLB fault. 203 179 */ 204 180 START_EXCEPTION(0x0300, DataStorage) 205 - EXCEPTION_PROLOG 206 - mfspr r5, SPRN_ESR /* Grab the ESR, save it */ 207 - stw r5, _ESR(r11) 208 - mfspr r4, SPRN_DEAR /* Grab the DEAR, save it */ 209 - stw r4, _DEAR(r11) 210 - EXC_XFER_LITE(0x300, handle_page_fault) 181 + EXCEPTION_PROLOG 0x300 DataStorage handle_dar_dsisr=1 182 + prepare_transfer_to_handler 183 + bl do_page_fault 184 + b interrupt_return 211 185 212 186 /* 213 187 * 0x0400 - Instruction Storage Exception 214 188 * This is caused by a fetch from non-execute or guarded pages. 215 189 */ 216 190 START_EXCEPTION(0x0400, InstructionAccess) 217 - EXCEPTION_PROLOG 191 + EXCEPTION_PROLOG 0x400 InstructionAccess 218 192 li r5,0 219 193 stw r5, _ESR(r11) /* Zero ESR */ 220 194 stw r12, _DEAR(r11) /* SRR0 as DEAR */ 221 - EXC_XFER_LITE(0x400, handle_page_fault) 195 + prepare_transfer_to_handler 196 + bl do_page_fault 197 + b interrupt_return 222 198 223 199 /* 0x0500 - External Interrupt Exception */ 224 - EXCEPTION(0x0500, HardwareInterrupt, do_IRQ, EXC_XFER_LITE) 200 + EXCEPTION(0x0500, HardwareInterrupt, do_IRQ) 225 201 226 202 /* 0x0600 - Alignment Exception */ 227 203 START_EXCEPTION(0x0600, Alignment) 228 - EXCEPTION_PROLOG 229 - mfspr r4,SPRN_DEAR /* Grab the DEAR and save it */ 230 - stw r4,_DEAR(r11) 231 - addi r3,r1,STACK_FRAME_OVERHEAD 232 - EXC_XFER_STD(0x600, alignment_exception) 204 + EXCEPTION_PROLOG 0x600 Alignment handle_dar_dsisr=1 205 + prepare_transfer_to_handler 206 + bl alignment_exception 207 + REST_NVGPRS(r1) 208 + b interrupt_return 233 209 234 210 /* 0x0700 - Program Exception */ 235 211 START_EXCEPTION(0x0700, ProgramCheck) 236 - EXCEPTION_PROLOG 237 - mfspr r4,SPRN_ESR /* Grab the ESR and save it */ 238 - stw r4,_ESR(r11) 239 - addi r3,r1,STACK_FRAME_OVERHEAD 240 - EXC_XFER_STD(0x700, program_check_exception) 212 + EXCEPTION_PROLOG 0x700 ProgramCheck handle_dar_dsisr=1 213 + prepare_transfer_to_handler 214 + bl program_check_exception 215 + REST_NVGPRS(r1) 216 + b interrupt_return 241 217 242 - EXCEPTION(0x0800, Trap_08, unknown_exception, EXC_XFER_STD) 243 - EXCEPTION(0x0900, Trap_09, unknown_exception, EXC_XFER_STD) 244 - EXCEPTION(0x0A00, Trap_0A, unknown_exception, EXC_XFER_STD) 245 - EXCEPTION(0x0B00, Trap_0B, unknown_exception, EXC_XFER_STD) 218 + EXCEPTION(0x0800, Trap_08, unknown_exception) 219 + EXCEPTION(0x0900, Trap_09, unknown_exception) 220 + EXCEPTION(0x0A00, Trap_0A, unknown_exception) 221 + EXCEPTION(0x0B00, Trap_0B, unknown_exception) 246 222 247 223 /* 0x0C00 - System Call Exception */ 248 224 START_EXCEPTION(0x0C00, SystemCall) 249 225 SYSCALL_ENTRY 0xc00 250 226 /* Trap_0D is commented out to get more space for system call exception */ 251 227 252 - /* EXCEPTION(0x0D00, Trap_0D, unknown_exception, EXC_XFER_STD) */ 253 - EXCEPTION(0x0E00, Trap_0E, unknown_exception, EXC_XFER_STD) 254 - EXCEPTION(0x0F00, Trap_0F, unknown_exception, EXC_XFER_STD) 228 + /* EXCEPTION(0x0D00, Trap_0D, unknown_exception) */ 229 + EXCEPTION(0x0E00, Trap_0E, unknown_exception) 230 + EXCEPTION(0x0F00, Trap_0F, unknown_exception) 255 231 256 232 /* 0x1000 - Programmable Interval Timer (PIT) Exception */ 257 - . = 0x1000 233 + START_EXCEPTION(0x1000, DecrementerTrap) 258 234 b Decrementer 259 235 260 - /* 0x1010 - Fixed Interval Timer (FIT) Exception 261 - */ 262 - . = 0x1010 236 + /* 0x1010 - Fixed Interval Timer (FIT) Exception */ 237 + START_EXCEPTION(0x1010, FITExceptionTrap) 263 238 b FITException 264 239 265 - /* 0x1020 - Watchdog Timer (WDT) Exception 266 - */ 267 - . = 0x1020 240 + /* 0x1020 - Watchdog Timer (WDT) Exception */ 241 + START_EXCEPTION(0x1020, WDTExceptionTrap) 268 242 b WDTException 269 243 270 244 /* 0x1100 - Data TLB Miss Exception ··· 271 249 * load TLB entries from the page table if they exist. 272 250 */ 273 251 START_EXCEPTION(0x1100, DTLBMiss) 274 - mtspr SPRN_SPRG_SCRATCH0, r10 /* Save some working registers */ 275 - mtspr SPRN_SPRG_SCRATCH1, r11 252 + mtspr SPRN_SPRG_SCRATCH5, r10 /* Save some working registers */ 253 + mtspr SPRN_SPRG_SCRATCH6, r11 276 254 mtspr SPRN_SPRG_SCRATCH3, r12 277 255 mtspr SPRN_SPRG_SCRATCH4, r9 278 256 mfcr r12 279 257 mfspr r9, SPRN_PID 280 - mtspr SPRN_SPRG_SCRATCH5, r9 258 + rlwimi r12, r9, 0, 0xff 281 259 mfspr r10, SPRN_DEAR /* Get faulting address */ 282 260 283 261 /* If we are faulting a kernel address, we have to use the ··· 338 316 /* The bailout. Restore registers to pre-exception conditions 339 317 * and call the heavyweights to help us out. 340 318 */ 341 - mfspr r9, SPRN_SPRG_SCRATCH5 342 - mtspr SPRN_PID, r9 343 - mtcr r12 319 + mtspr SPRN_PID, r12 320 + mtcrf 0x80, r12 344 321 mfspr r9, SPRN_SPRG_SCRATCH4 345 322 mfspr r12, SPRN_SPRG_SCRATCH3 346 - mfspr r11, SPRN_SPRG_SCRATCH1 347 - mfspr r10, SPRN_SPRG_SCRATCH0 323 + mfspr r11, SPRN_SPRG_SCRATCH6 324 + mfspr r10, SPRN_SPRG_SCRATCH5 348 325 b DataStorage 349 326 350 327 /* 0x1200 - Instruction TLB Miss Exception ··· 351 330 * registers and bailout to a different point. 352 331 */ 353 332 START_EXCEPTION(0x1200, ITLBMiss) 354 - mtspr SPRN_SPRG_SCRATCH0, r10 /* Save some working registers */ 355 - mtspr SPRN_SPRG_SCRATCH1, r11 333 + mtspr SPRN_SPRG_SCRATCH5, r10 /* Save some working registers */ 334 + mtspr SPRN_SPRG_SCRATCH6, r11 356 335 mtspr SPRN_SPRG_SCRATCH3, r12 357 336 mtspr SPRN_SPRG_SCRATCH4, r9 358 337 mfcr r12 359 338 mfspr r9, SPRN_PID 360 - mtspr SPRN_SPRG_SCRATCH5, r9 339 + rlwimi r12, r9, 0, 0xff 361 340 mfspr r10, SPRN_SRR0 /* Get faulting address */ 362 341 363 342 /* If we are faulting a kernel address, we have to use the ··· 418 397 /* The bailout. Restore registers to pre-exception conditions 419 398 * and call the heavyweights to help us out. 420 399 */ 421 - mfspr r9, SPRN_SPRG_SCRATCH5 422 - mtspr SPRN_PID, r9 423 - mtcr r12 400 + mtspr SPRN_PID, r12 401 + mtcrf 0x80, r12 424 402 mfspr r9, SPRN_SPRG_SCRATCH4 425 403 mfspr r12, SPRN_SPRG_SCRATCH3 426 - mfspr r11, SPRN_SPRG_SCRATCH1 427 - mfspr r10, SPRN_SPRG_SCRATCH0 404 + mfspr r11, SPRN_SPRG_SCRATCH6 405 + mfspr r10, SPRN_SPRG_SCRATCH5 428 406 b InstructionAccess 429 407 430 - EXCEPTION(0x1300, Trap_13, unknown_exception, EXC_XFER_STD) 431 - EXCEPTION(0x1400, Trap_14, unknown_exception, EXC_XFER_STD) 432 - EXCEPTION(0x1500, Trap_15, unknown_exception, EXC_XFER_STD) 433 - EXCEPTION(0x1600, Trap_16, unknown_exception, EXC_XFER_STD) 434 - EXCEPTION(0x1700, Trap_17, unknown_exception, EXC_XFER_STD) 435 - EXCEPTION(0x1800, Trap_18, unknown_exception, EXC_XFER_STD) 436 - EXCEPTION(0x1900, Trap_19, unknown_exception, EXC_XFER_STD) 437 - EXCEPTION(0x1A00, Trap_1A, unknown_exception, EXC_XFER_STD) 438 - EXCEPTION(0x1B00, Trap_1B, unknown_exception, EXC_XFER_STD) 439 - EXCEPTION(0x1C00, Trap_1C, unknown_exception, EXC_XFER_STD) 440 - EXCEPTION(0x1D00, Trap_1D, unknown_exception, EXC_XFER_STD) 441 - EXCEPTION(0x1E00, Trap_1E, unknown_exception, EXC_XFER_STD) 442 - EXCEPTION(0x1F00, Trap_1F, unknown_exception, EXC_XFER_STD) 408 + EXCEPTION(0x1300, Trap_13, unknown_exception) 409 + EXCEPTION(0x1400, Trap_14, unknown_exception) 410 + EXCEPTION(0x1500, Trap_15, unknown_exception) 411 + EXCEPTION(0x1600, Trap_16, unknown_exception) 412 + EXCEPTION(0x1700, Trap_17, unknown_exception) 413 + EXCEPTION(0x1800, Trap_18, unknown_exception) 414 + EXCEPTION(0x1900, Trap_19, unknown_exception) 415 + EXCEPTION(0x1A00, Trap_1A, unknown_exception) 416 + EXCEPTION(0x1B00, Trap_1B, unknown_exception) 417 + EXCEPTION(0x1C00, Trap_1C, unknown_exception) 418 + EXCEPTION(0x1D00, Trap_1D, unknown_exception) 419 + EXCEPTION(0x1E00, Trap_1E, unknown_exception) 420 + EXCEPTION(0x1F00, Trap_1F, unknown_exception) 443 421 444 422 /* Check for a single step debug exception while in an exception 445 423 * handler before state has been saved. This is to catch the case ··· 455 435 */ 456 436 /* 0x2000 - Debug Exception */ 457 437 START_EXCEPTION(0x2000, DebugTrap) 458 - CRITICAL_EXCEPTION_PROLOG 438 + CRITICAL_EXCEPTION_PROLOG 0x2000 DebugTrap 459 439 460 440 /* 461 441 * If this is a single step or branch-taken exception in an ··· 497 477 /* continue normal handling for a critical exception... */ 498 478 2: mfspr r4,SPRN_DBSR 499 479 stw r4,_ESR(r11) /* DebugException takes DBSR in _ESR */ 500 - addi r3,r1,STACK_FRAME_OVERHEAD 501 - EXC_XFER_TEMPLATE(DebugException, 0x2002, \ 502 - (MSR_KERNEL & ~(MSR_ME|MSR_DE|MSR_CE)), \ 503 - crit_transfer_to_handler, ret_from_crit_exc) 480 + prepare_transfer_to_handler 481 + bl DebugException 482 + b ret_from_crit_exc 504 483 505 484 /* Programmable Interval Timer (PIT) Exception. (from 0x1000) */ 485 + __HEAD 506 486 Decrementer: 507 - EXCEPTION_PROLOG 487 + EXCEPTION_PROLOG 0x1000 Decrementer 508 488 lis r0,TSR_PIS@h 509 489 mtspr SPRN_TSR,r0 /* Clear the PIT exception */ 510 - addi r3,r1,STACK_FRAME_OVERHEAD 511 - EXC_XFER_LITE(0x1000, timer_interrupt) 490 + prepare_transfer_to_handler 491 + bl timer_interrupt 492 + b interrupt_return 512 493 513 494 /* Fixed Interval Timer (FIT) Exception. (from 0x1010) */ 495 + __HEAD 514 496 FITException: 515 - EXCEPTION_PROLOG 516 - addi r3,r1,STACK_FRAME_OVERHEAD; 517 - EXC_XFER_STD(0x1010, unknown_exception) 497 + EXCEPTION_PROLOG 0x1010 FITException 498 + prepare_transfer_to_handler 499 + bl unknown_exception 500 + b interrupt_return 518 501 519 502 /* Watchdog Timer (WDT) Exception. (from 0x1020) */ 503 + __HEAD 520 504 WDTException: 521 - CRITICAL_EXCEPTION_PROLOG; 522 - addi r3,r1,STACK_FRAME_OVERHEAD; 523 - EXC_XFER_TEMPLATE(WatchdogException, 0x1020+2, 524 - (MSR_KERNEL & ~(MSR_ME|MSR_DE|MSR_CE)), 525 - crit_transfer_to_handler, ret_from_crit_exc) 505 + CRITICAL_EXCEPTION_PROLOG 0x1020 WDTException 506 + prepare_transfer_to_handler 507 + bl WatchdogException 508 + b ret_from_crit_exc 526 509 527 510 /* Other PowerPC processors, namely those derived from the 6xx-series 528 511 * have vectors from 0x2100 through 0x2F00 defined, but marked as reserved. ··· 533 510 * reserved. 534 511 */ 535 512 513 + __HEAD 536 514 /* Damn, I came up one instruction too many to fit into the 537 515 * exception space :-). Both the instruction and data TLB 538 516 * miss get to this point to load the TLB. ··· 567 543 568 544 /* Done...restore registers and get out of here. 569 545 */ 570 - mfspr r9, SPRN_SPRG_SCRATCH5 571 - mtspr SPRN_PID, r9 572 - mtcr r12 546 + mtspr SPRN_PID, r12 547 + mtcrf 0x80, r12 573 548 mfspr r9, SPRN_SPRG_SCRATCH4 574 549 mfspr r12, SPRN_SPRG_SCRATCH3 575 - mfspr r11, SPRN_SPRG_SCRATCH1 576 - mfspr r10, SPRN_SPRG_SCRATCH0 550 + mfspr r11, SPRN_SPRG_SCRATCH6 551 + mfspr r10, SPRN_SPRG_SCRATCH5 577 552 rfi /* Should sync shadow TLBs */ 578 553 b . /* prevent prefetch past rfi */ 579 554

+4 -6

arch/powerpc/kernel/head_44x.S

··· 263 263 INSTRUCTION_STORAGE_EXCEPTION 264 264 265 265 /* External Input Interrupt */ 266 - EXCEPTION(0x0500, BOOKE_INTERRUPT_EXTERNAL, ExternalInput, \ 267 - do_IRQ, EXC_XFER_LITE) 266 + EXCEPTION(0x0500, BOOKE_INTERRUPT_EXTERNAL, ExternalInput, do_IRQ) 268 267 269 268 /* Alignment Interrupt */ 270 269 ALIGNMENT_EXCEPTION ··· 276 277 FP_UNAVAILABLE_EXCEPTION 277 278 #else 278 279 EXCEPTION(0x2010, BOOKE_INTERRUPT_FP_UNAVAIL, \ 279 - FloatingPointUnavailable, unknown_exception, EXC_XFER_STD) 280 + FloatingPointUnavailable, unknown_exception) 280 281 #endif 281 282 /* System Call Interrupt */ 282 283 START_EXCEPTION(SystemCall) ··· 284 285 285 286 /* Auxiliary Processor Unavailable Interrupt */ 286 287 EXCEPTION(0x2020, BOOKE_INTERRUPT_AP_UNAVAIL, \ 287 - AuxillaryProcessorUnavailable, unknown_exception, EXC_XFER_STD) 288 + AuxillaryProcessorUnavailable, unknown_exception) 288 289 289 290 /* Decrementer Interrupt */ 290 291 DECREMENTER_EXCEPTION 291 292 292 293 /* Fixed Internal Timer Interrupt */ 293 294 /* TODO: Add FIT support */ 294 - EXCEPTION(0x1010, BOOKE_INTERRUPT_FIT, FixedIntervalTimer, \ 295 - unknown_exception, EXC_XFER_STD) 295 + EXCEPTION(0x1010, BOOKE_INTERRUPT_FIT, FixedIntervalTimer, unknown_exception) 296 296 297 297 /* Watchdog Timer Interrupt */ 298 298 /* TODO: Add watchdog support */

+71 -85

arch/powerpc/kernel/head_8xx.S

··· 29 29 #include <asm/ptrace.h> 30 30 #include <asm/export.h> 31 31 #include <asm/code-patching-asm.h> 32 + #include <asm/interrupt.h> 33 + 34 + /* 35 + * Value for the bits that have fixed value in RPN entries. 36 + * Also used for tagging DAR for DTLBerror. 37 + */ 38 + #define RPN_PATTERN 0x00f0 32 39 33 40 #include "head_32.h" 34 41 ··· 48 41 cmpli cr0, \scratch, PAGE_OFFSET@h 49 42 #endif 50 43 .endm 51 - 52 - /* 53 - * Value for the bits that have fixed value in RPN entries. 54 - * Also used for tagging DAR for DTLBerror. 55 - */ 56 - #define RPN_PATTERN 0x00f0 57 44 58 45 #define PAGE_SHIFT_512K 19 59 46 #define PAGE_SHIFT_8M 23 ··· 119 118 #endif 120 119 121 120 /* System reset */ 122 - EXCEPTION(0x100, Reset, system_reset_exception, EXC_XFER_STD) 121 + EXCEPTION(INTERRUPT_SYSTEM_RESET, Reset, system_reset_exception) 123 122 124 123 /* Machine check */ 125 - . = 0x200 126 - MachineCheck: 127 - EXCEPTION_PROLOG handle_dar_dsisr=1 128 - save_dar_dsisr_on_stack r4, r5, r11 129 - li r6, RPN_PATTERN 130 - mtspr SPRN_DAR, r6 /* Tag DAR, to be used in DTLB Error */ 131 - addi r3,r1,STACK_FRAME_OVERHEAD 132 - EXC_XFER_STD(0x200, machine_check_exception) 124 + START_EXCEPTION(INTERRUPT_MACHINE_CHECK, MachineCheck) 125 + EXCEPTION_PROLOG INTERRUPT_MACHINE_CHECK MachineCheck handle_dar_dsisr=1 126 + prepare_transfer_to_handler 127 + bl machine_check_exception 128 + b interrupt_return 133 129 134 130 /* External interrupt */ 135 - EXCEPTION(0x500, HardwareInterrupt, do_IRQ, EXC_XFER_LITE) 131 + EXCEPTION(INTERRUPT_EXTERNAL, HardwareInterrupt, do_IRQ) 136 132 137 133 /* Alignment exception */ 138 - . = 0x600 139 - Alignment: 140 - EXCEPTION_PROLOG handle_dar_dsisr=1 141 - save_dar_dsisr_on_stack r4, r5, r11 142 - li r6, RPN_PATTERN 143 - mtspr SPRN_DAR, r6 /* Tag DAR, to be used in DTLB Error */ 144 - addi r3,r1,STACK_FRAME_OVERHEAD 145 - b .Lalignment_exception_ool 134 + START_EXCEPTION(INTERRUPT_ALIGNMENT, Alignment) 135 + EXCEPTION_PROLOG INTERRUPT_ALIGNMENT Alignment handle_dar_dsisr=1 136 + prepare_transfer_to_handler 137 + bl alignment_exception 138 + REST_NVGPRS(r1) 139 + b interrupt_return 146 140 147 141 /* Program check exception */ 148 - EXCEPTION(0x700, ProgramCheck, program_check_exception, EXC_XFER_STD) 142 + START_EXCEPTION(INTERRUPT_PROGRAM, ProgramCheck) 143 + EXCEPTION_PROLOG INTERRUPT_PROGRAM ProgramCheck 144 + prepare_transfer_to_handler 145 + bl program_check_exception 146 + REST_NVGPRS(r1) 147 + b interrupt_return 149 148 150 149 /* Decrementer */ 151 - EXCEPTION(0x900, Decrementer, timer_interrupt, EXC_XFER_LITE) 152 - 153 - /* With VMAP_STACK there's not enough room for this at 0x600 */ 154 - . = 0xa00 155 - .Lalignment_exception_ool: 156 - EXC_XFER_STD(0x600, alignment_exception) 150 + EXCEPTION(INTERRUPT_DECREMENTER, Decrementer, timer_interrupt) 157 151 158 152 /* System call */ 159 - . = 0xc00 160 - SystemCall: 161 - SYSCALL_ENTRY 0xc00 153 + START_EXCEPTION(INTERRUPT_SYSCALL, SystemCall) 154 + SYSCALL_ENTRY INTERRUPT_SYSCALL 162 155 163 156 /* Single step - not used on 601 */ 164 - EXCEPTION(0xd00, SingleStep, single_step_exception, EXC_XFER_STD) 157 + EXCEPTION(INTERRUPT_TRACE, SingleStep, single_step_exception) 165 158 166 159 /* On the MPC8xx, this is a software emulation interrupt. It occurs 167 160 * for all unimplemented and illegal instructions. 168 161 */ 169 - EXCEPTION(0x1000, SoftEmu, emulation_assist_interrupt, EXC_XFER_STD) 162 + START_EXCEPTION(INTERRUPT_SOFT_EMU_8xx, SoftEmu) 163 + EXCEPTION_PROLOG INTERRUPT_SOFT_EMU_8xx SoftEmu 164 + prepare_transfer_to_handler 165 + bl emulation_assist_interrupt 166 + REST_NVGPRS(r1) 167 + b interrupt_return 170 168 171 - . = 0x1100 172 169 /* 173 170 * For the MPC8xx, this is a software tablewalk to load the instruction 174 171 * TLB. The task switch loads the M_TWB register with the pointer to the first ··· 188 189 #define INVALIDATE_ADJACENT_PAGES_CPU15(addr, tmp) 189 190 #endif 190 191 191 - InstructionTLBMiss: 192 + START_EXCEPTION(INTERRUPT_INST_TLB_MISS_8xx, InstructionTLBMiss) 192 193 mtspr SPRN_SPRG_SCRATCH2, r10 193 194 mtspr SPRN_M_TW, r11 194 195 ··· 244 245 rfi 245 246 #endif 246 247 247 - . = 0x1200 248 - DataStoreTLBMiss: 248 + START_EXCEPTION(INTERRUPT_DATA_TLB_MISS_8xx, DataStoreTLBMiss) 249 249 mtspr SPRN_SPRG_SCRATCH2, r10 250 250 mtspr SPRN_M_TW, r11 251 251 mfcr r11 ··· 307 309 * to many reasons, such as executing guarded memory or illegal instruction 308 310 * addresses. There is nothing to do but handle a big time error fault. 309 311 */ 310 - . = 0x1300 311 - InstructionTLBError: 312 - EXCEPTION_PROLOG 312 + START_EXCEPTION(INTERRUPT_INST_TLB_ERROR_8xx, InstructionTLBError) 313 + /* 0x400 is InstructionAccess exception, needed by bad_page_fault() */ 314 + EXCEPTION_PROLOG INTERRUPT_INST_STORAGE InstructionTLBError 313 315 andis. r5,r9,DSISR_SRR1_MATCH_32S@h /* Filter relevant SRR1 bits */ 314 316 andis. r10,r9,SRR1_ISI_NOPT@h 315 317 beq+ .Litlbie 316 318 tlbie r12 317 - /* 0x400 is InstructionAccess exception, needed by bad_page_fault() */ 318 319 .Litlbie: 319 320 stw r12, _DAR(r11) 320 321 stw r5, _DSISR(r11) 321 - EXC_XFER_LITE(0x400, handle_page_fault) 322 + prepare_transfer_to_handler 323 + bl do_page_fault 324 + b interrupt_return 322 325 323 326 /* This is the data TLB error on the MPC8xx. This could be due to 324 327 * many reasons, including a dirty update to a pte. We bail out to 325 328 * a higher level function that can handle it. 326 329 */ 327 - . = 0x1400 328 - DataTLBError: 330 + START_EXCEPTION(INTERRUPT_DATA_TLB_ERROR_8xx, DataTLBError) 329 331 EXCEPTION_PROLOG_0 handle_dar_dsisr=1 330 332 mfspr r11, SPRN_DAR 331 333 cmpwi cr1, r11, RPN_PATTERN 332 334 beq- cr1, FixupDAR /* must be a buggy dcbX, icbi insn. */ 333 335 DARFixed:/* Return from dcbx instruction bug workaround */ 334 - #ifdef CONFIG_VMAP_STACK 335 - li r11, RPN_PATTERN 336 - mtspr SPRN_DAR, r11 /* Tag DAR, to be used in DTLB Error */ 337 - #endif 338 336 EXCEPTION_PROLOG_1 339 - EXCEPTION_PROLOG_2 handle_dar_dsisr=1 340 - get_and_save_dar_dsisr_on_stack r4, r5, r11 337 + /* 0x300 is DataAccess exception, needed by bad_page_fault() */ 338 + EXCEPTION_PROLOG_2 INTERRUPT_DATA_STORAGE DataTLBError handle_dar_dsisr=1 339 + lwz r4, _DAR(r11) 340 + lwz r5, _DSISR(r11) 341 341 andis. r10,r5,DSISR_NOHPTE@h 342 342 beq+ .Ldtlbie 343 343 tlbie r4 344 344 .Ldtlbie: 345 - #ifndef CONFIG_VMAP_STACK 346 - li r10,RPN_PATTERN 347 - mtspr SPRN_DAR,r10 /* Tag DAR, to be used in DTLB Error */ 348 - #endif 349 - /* 0x300 is DataAccess exception, needed by bad_page_fault() */ 350 - EXC_XFER_LITE(0x300, handle_page_fault) 345 + prepare_transfer_to_handler 346 + bl do_page_fault 347 + b interrupt_return 351 348 352 - stack_overflow: 349 + #ifdef CONFIG_VMAP_STACK 353 350 vmap_stack_overflow_exception 351 + #endif 354 352 355 353 /* On the MPC8xx, these next four traps are used for development 356 354 * support of breakpoints and such. Someday I will get around to 357 355 * using them. 358 356 */ 359 - do_databreakpoint: 360 - EXCEPTION_PROLOG_1 361 - EXCEPTION_PROLOG_2 handle_dar_dsisr=1 362 - addi r3,r1,STACK_FRAME_OVERHEAD 363 - mfspr r4,SPRN_BAR 364 - stw r4,_DAR(r11) 365 - #ifndef CONFIG_VMAP_STACK 366 - mfspr r5,SPRN_DSISR 367 - stw r5,_DSISR(r11) 368 - #endif 369 - EXC_XFER_STD(0x1c00, do_break) 370 - 371 - . = 0x1c00 372 - DataBreakpoint: 357 + START_EXCEPTION(INTERRUPT_DATA_BREAKPOINT_8xx, DataBreakpoint) 373 358 EXCEPTION_PROLOG_0 handle_dar_dsisr=1 374 359 mfspr r11, SPRN_SRR0 375 360 cmplwi cr1, r11, (.Ldtlbie - PAGE_OFFSET)@l 376 361 cmplwi cr7, r11, (.Litlbie - PAGE_OFFSET)@l 377 362 cror 4*cr1+eq, 4*cr1+eq, 4*cr7+eq 378 - bne cr1, do_databreakpoint 363 + bne cr1, 1f 379 364 mtcr r10 380 365 mfspr r10, SPRN_SPRG_SCRATCH0 381 366 mfspr r11, SPRN_SPRG_SCRATCH1 382 367 rfi 383 368 369 + 1: EXCEPTION_PROLOG_1 370 + EXCEPTION_PROLOG_2 INTERRUPT_DATA_BREAKPOINT_8xx DataBreakpoint handle_dar_dsisr=1 371 + mfspr r4,SPRN_BAR 372 + stw r4,_DAR(r11) 373 + prepare_transfer_to_handler 374 + bl do_break 375 + REST_NVGPRS(r1) 376 + b interrupt_return 377 + 384 378 #ifdef CONFIG_PERF_EVENTS 385 - . = 0x1d00 386 - InstructionBreakpoint: 379 + START_EXCEPTION(INTERRUPT_INST_BREAKPOINT_8xx, InstructionBreakpoint) 387 380 mtspr SPRN_SPRG_SCRATCH0, r10 388 381 lwz r10, (instruction_counter - PAGE_OFFSET)@l(0) 389 382 addi r10, r10, -1 ··· 385 396 mfspr r10, SPRN_SPRG_SCRATCH0 386 397 rfi 387 398 #else 388 - EXCEPTION(0x1d00, Trap_1d, unknown_exception, EXC_XFER_STD) 399 + EXCEPTION(INTERRUPT_INST_BREAKPOINT_8xx, Trap_1d, unknown_exception) 389 400 #endif 390 - EXCEPTION(0x1e00, Trap_1e, unknown_exception, EXC_XFER_STD) 391 - EXCEPTION(0x1f00, Trap_1f, unknown_exception, EXC_XFER_STD) 401 + EXCEPTION(0x1e00, Trap_1e, unknown_exception) 402 + EXCEPTION(0x1f00, Trap_1f, unknown_exception) 392 403 404 + __HEAD 393 405 . = 0x2000 394 406 395 407 /* This is the procedure to calculate the data EA for buggy dcbx,dcbi instructions ··· 500 510 152: 501 511 mfdar r11 502 512 mtctr r11 /* restore ctr reg from DAR */ 503 - #ifdef CONFIG_VMAP_STACK 504 513 mfspr r11, SPRN_SPRG_THREAD 505 514 stw r10, DAR(r11) 506 515 mfspr r10, SPRN_DSISR 507 516 stw r10, DSISR(r11) 508 - #else 509 - mtdar r10 /* save fault EA to DAR */ 510 - #endif 511 517 mfspr r10,SPRN_M_TW 512 518 b DARFixed /* Go back to normal TLB handling */ 513 519 ··· 805 819 swapper_pg_dir: 806 820 .space PGD_TABLE_SIZE 807 821 808 - /* Room for two PTE table poiners, usually the kernel and current user 822 + /* Room for two PTE table pointers, usually the kernel and current user 809 823 * pointer to their respective root page table (pgdir). 810 824 */ 811 825 .globl abatron_pteptrs

+99 -143

arch/powerpc/kernel/head_book3s_32.S

··· 31 31 #include <asm/kvm_book3s_asm.h> 32 32 #include <asm/export.h> 33 33 #include <asm/feature-fixups.h> 34 + #include <asm/interrupt.h> 34 35 35 36 #include "head_32.h" 36 37 ··· 240 239 /* System reset */ 241 240 /* core99 pmac starts the seconary here by changing the vector, and 242 241 putting it back to what it was (unknown_async_exception) when done. */ 243 - EXCEPTION(0x100, Reset, unknown_async_exception, EXC_XFER_STD) 242 + EXCEPTION(INTERRUPT_SYSTEM_RESET, Reset, unknown_async_exception) 244 243 245 244 /* Machine check */ 246 245 /* ··· 256 255 * pointer when we take an exception from supervisor mode.) 257 256 * -- paulus. 258 257 */ 259 - . = 0x200 260 - DO_KVM 0x200 261 - MachineCheck: 258 + START_EXCEPTION(INTERRUPT_MACHINE_CHECK, MachineCheck) 262 259 EXCEPTION_PROLOG_0 263 260 #ifdef CONFIG_PPC_CHRP 264 - #ifdef CONFIG_VMAP_STACK 265 261 mtspr SPRN_SPRG_SCRATCH2,r1 266 262 mfspr r1, SPRN_SPRG_THREAD 267 263 lwz r1, RTAS_SP(r1) 268 264 cmpwi cr1, r1, 0 269 265 bne cr1, 7f 270 266 mfspr r1, SPRN_SPRG_SCRATCH2 271 - #else 272 - mfspr r11, SPRN_SPRG_THREAD 273 - lwz r11, RTAS_SP(r11) 274 - cmpwi cr1, r11, 0 275 - bne cr1, 7f 276 - #endif 277 267 #endif /* CONFIG_PPC_CHRP */ 278 - EXCEPTION_PROLOG_1 for_rtas=1 279 - 7: EXCEPTION_PROLOG_2 280 - addi r3,r1,STACK_FRAME_OVERHEAD 268 + EXCEPTION_PROLOG_1 269 + 7: EXCEPTION_PROLOG_2 0x200 MachineCheck 281 270 #ifdef CONFIG_PPC_CHRP 282 - beq cr1, machine_check_tramp 271 + beq cr1, 1f 283 272 twi 31, 0, 0 284 - #else 285 - b machine_check_tramp 286 273 #endif 274 + 1: prepare_transfer_to_handler 275 + bl machine_check_exception 276 + b interrupt_return 287 277 288 278 /* Data access exception. */ 289 - . = 0x300 290 - DO_KVM 0x300 291 - DataAccess: 292 - #ifdef CONFIG_VMAP_STACK 279 + START_EXCEPTION(INTERRUPT_DATA_STORAGE, DataAccess) 293 280 #ifdef CONFIG_PPC_BOOK3S_604 294 281 BEGIN_MMU_FTR_SECTION 295 282 mtspr SPRN_SPRG_SCRATCH2,r10 ··· 298 309 #endif 299 310 1: EXCEPTION_PROLOG_0 handle_dar_dsisr=1 300 311 EXCEPTION_PROLOG_1 301 - b handle_page_fault_tramp_1 302 - #else /* CONFIG_VMAP_STACK */ 303 - EXCEPTION_PROLOG handle_dar_dsisr=1 304 - get_and_save_dar_dsisr_on_stack r4, r5, r11 305 - #ifdef CONFIG_PPC_BOOK3S_604 306 - BEGIN_MMU_FTR_SECTION 307 - andis. r0, r5, (DSISR_BAD_FAULT_32S | DSISR_DABRMATCH)@h 308 - bne handle_page_fault_tramp_2 /* if not, try to put a PTE */ 309 - rlwinm r3, r5, 32 - 15, 21, 21 /* DSISR_STORE -> _PAGE_RW */ 310 - bl hash_page 311 - b handle_page_fault_tramp_1 312 - MMU_FTR_SECTION_ELSE 313 - #endif 314 - b handle_page_fault_tramp_2 315 - #ifdef CONFIG_PPC_BOOK3S_604 316 - ALT_MMU_FTR_SECTION_END_IFSET(MMU_FTR_HPTE_TABLE) 317 - #endif 318 - #endif /* CONFIG_VMAP_STACK */ 312 + EXCEPTION_PROLOG_2 INTERRUPT_DATA_STORAGE DataAccess handle_dar_dsisr=1 313 + prepare_transfer_to_handler 314 + lwz r5, _DSISR(r11) 315 + andis. r0, r5, DSISR_DABRMATCH@h 316 + bne- 1f 317 + bl do_page_fault 318 + b interrupt_return 319 + 1: bl do_break 320 + REST_NVGPRS(r1) 321 + b interrupt_return 322 + 319 323 320 324 /* Instruction access exception. */ 321 - . = 0x400 322 - DO_KVM 0x400 323 - InstructionAccess: 324 - #ifdef CONFIG_VMAP_STACK 325 + START_EXCEPTION(INTERRUPT_INST_STORAGE, InstructionAccess) 325 326 mtspr SPRN_SPRG_SCRATCH0,r10 326 327 mtspr SPRN_SPRG_SCRATCH1,r11 327 328 mfspr r10, SPRN_SPRG_THREAD ··· 331 352 andi. r11, r11, MSR_PR 332 353 333 354 EXCEPTION_PROLOG_1 334 - EXCEPTION_PROLOG_2 335 - #else /* CONFIG_VMAP_STACK */ 336 - EXCEPTION_PROLOG 337 - andis. r0,r9,SRR1_ISI_NOPT@h /* no pte found? */ 338 - beq 1f /* if so, try to put a PTE */ 339 - li r3,0 /* into the hash table */ 340 - mr r4,r12 /* SRR0 is fault address */ 341 - #ifdef CONFIG_PPC_BOOK3S_604 342 - BEGIN_MMU_FTR_SECTION 343 - bl hash_page 344 - END_MMU_FTR_SECTION_IFSET(MMU_FTR_HPTE_TABLE) 345 - #endif 346 - #endif /* CONFIG_VMAP_STACK */ 355 + EXCEPTION_PROLOG_2 INTERRUPT_INST_STORAGE InstructionAccess 347 356 andis. r5,r9,DSISR_SRR1_MATCH_32S@h /* Filter relevant SRR1 bits */ 348 357 stw r5, _DSISR(r11) 349 358 stw r12, _DAR(r11) 350 - EXC_XFER_LITE(0x400, handle_page_fault) 359 + prepare_transfer_to_handler 360 + bl do_page_fault 361 + b interrupt_return 351 362 352 363 /* External interrupt */ 353 - EXCEPTION(0x500, HardwareInterrupt, do_IRQ, EXC_XFER_LITE) 364 + EXCEPTION(INTERRUPT_EXTERNAL, HardwareInterrupt, do_IRQ) 354 365 355 366 /* Alignment exception */ 356 - . = 0x600 357 - DO_KVM 0x600 358 - Alignment: 359 - EXCEPTION_PROLOG handle_dar_dsisr=1 360 - save_dar_dsisr_on_stack r4, r5, r11 361 - addi r3,r1,STACK_FRAME_OVERHEAD 362 - b alignment_exception_tramp 367 + START_EXCEPTION(INTERRUPT_ALIGNMENT, Alignment) 368 + EXCEPTION_PROLOG INTERRUPT_ALIGNMENT Alignment handle_dar_dsisr=1 369 + prepare_transfer_to_handler 370 + bl alignment_exception 371 + REST_NVGPRS(r1) 372 + b interrupt_return 363 373 364 374 /* Program check exception */ 365 - EXCEPTION(0x700, ProgramCheck, program_check_exception, EXC_XFER_STD) 375 + START_EXCEPTION(INTERRUPT_PROGRAM, ProgramCheck) 376 + EXCEPTION_PROLOG INTERRUPT_PROGRAM ProgramCheck 377 + prepare_transfer_to_handler 378 + bl program_check_exception 379 + REST_NVGPRS(r1) 380 + b interrupt_return 366 381 367 382 /* Floating-point unavailable */ 368 - . = 0x800 369 - DO_KVM 0x800 370 - FPUnavailable: 383 + START_EXCEPTION(0x800, FPUnavailable) 371 384 #ifdef CONFIG_PPC_FPU 372 385 BEGIN_FTR_SECTION 373 386 /* ··· 368 397 */ 369 398 b ProgramCheck 370 399 END_FTR_SECTION_IFSET(CPU_FTR_FPU_UNAVAILABLE) 371 - EXCEPTION_PROLOG 400 + EXCEPTION_PROLOG INTERRUPT_FP_UNAVAIL FPUnavailable 372 401 beq 1f 373 402 bl load_up_fpu /* if from user, just load it up */ 374 403 b fast_exception_return 375 - 1: addi r3,r1,STACK_FRAME_OVERHEAD 376 - EXC_XFER_LITE(0x800, kernel_fp_unavailable_exception) 404 + 1: prepare_transfer_to_handler 405 + bl kernel_fp_unavailable_exception 406 + b interrupt_return 377 407 #else 378 408 b ProgramCheck 379 409 #endif 380 410 381 411 /* Decrementer */ 382 - EXCEPTION(0x900, Decrementer, timer_interrupt, EXC_XFER_LITE) 412 + EXCEPTION(INTERRUPT_DECREMENTER, Decrementer, timer_interrupt) 383 413 384 - EXCEPTION(0xa00, Trap_0a, unknown_exception, EXC_XFER_STD) 385 - EXCEPTION(0xb00, Trap_0b, unknown_exception, EXC_XFER_STD) 414 + EXCEPTION(0xa00, Trap_0a, unknown_exception) 415 + EXCEPTION(0xb00, Trap_0b, unknown_exception) 386 416 387 417 /* System call */ 388 - . = 0xc00 389 - DO_KVM 0xc00 390 - SystemCall: 391 - SYSCALL_ENTRY 0xc00 418 + START_EXCEPTION(INTERRUPT_SYSCALL, SystemCall) 419 + SYSCALL_ENTRY INTERRUPT_SYSCALL 392 420 393 - EXCEPTION(0xd00, SingleStep, single_step_exception, EXC_XFER_STD) 394 - EXCEPTION(0xe00, Trap_0e, unknown_exception, EXC_XFER_STD) 421 + EXCEPTION(INTERRUPT_TRACE, SingleStep, single_step_exception) 422 + EXCEPTION(0xe00, Trap_0e, unknown_exception) 395 423 396 424 /* 397 425 * The Altivec unavailable trap is at 0x0f20. Foo. ··· 400 430 * non-altivec kernel running on a machine with altivec just 401 431 * by executing an altivec instruction. 402 432 */ 403 - . = 0xf00 404 - DO_KVM 0xf00 433 + START_EXCEPTION(INTERRUPT_PERFMON, PerformanceMonitorTrap) 405 434 b PerformanceMonitor 406 435 407 - . = 0xf20 408 - DO_KVM 0xf20 436 + START_EXCEPTION(INTERRUPT_ALTIVEC_UNAVAIL, AltiVecUnavailableTrap) 409 437 b AltiVecUnavailable 410 438 439 + __HEAD 411 440 /* 412 441 * Handle TLB miss for instruction on 603/603e. 413 442 * Note: we get an alternate set of r0 - r3 to use automatically. 414 443 */ 415 - . = 0x1000 444 + . = INTERRUPT_INST_TLB_MISS_603 416 445 InstructionTLBMiss: 417 446 /* 418 447 * r0: scratch ··· 477 508 /* 478 509 * Handle TLB miss for DATA Load operation on 603/603e 479 510 */ 480 - . = 0x1100 511 + . = INTERRUPT_DATA_LOAD_TLB_MISS_603 481 512 DataLoadTLBMiss: 482 513 /* 483 514 * r0: scratch ··· 555 586 /* 556 587 * Handle TLB miss for DATA Store on 603/603e 557 588 */ 558 - . = 0x1200 589 + . = INTERRUPT_DATA_STORE_TLB_MISS_603 559 590 DataStoreTLBMiss: 560 591 /* 561 592 * r0: scratch ··· 619 650 #define TAUException unknown_async_exception 620 651 #endif 621 652 622 - EXCEPTION(0x1300, Trap_13, instruction_breakpoint_exception, EXC_XFER_STD) 623 - EXCEPTION(0x1400, SMI, SMIException, EXC_XFER_STD) 624 - EXCEPTION(0x1500, Trap_15, unknown_exception, EXC_XFER_STD) 625 - EXCEPTION(0x1600, Trap_16, altivec_assist_exception, EXC_XFER_STD) 626 - EXCEPTION(0x1700, Trap_17, TAUException, EXC_XFER_STD) 627 - EXCEPTION(0x1800, Trap_18, unknown_exception, EXC_XFER_STD) 628 - EXCEPTION(0x1900, Trap_19, unknown_exception, EXC_XFER_STD) 629 - EXCEPTION(0x1a00, Trap_1a, unknown_exception, EXC_XFER_STD) 630 - EXCEPTION(0x1b00, Trap_1b, unknown_exception, EXC_XFER_STD) 631 - EXCEPTION(0x1c00, Trap_1c, unknown_exception, EXC_XFER_STD) 632 - EXCEPTION(0x1d00, Trap_1d, unknown_exception, EXC_XFER_STD) 633 - EXCEPTION(0x1e00, Trap_1e, unknown_exception, EXC_XFER_STD) 634 - EXCEPTION(0x1f00, Trap_1f, unknown_exception, EXC_XFER_STD) 635 - EXCEPTION(0x2000, RunMode, RunModeException, EXC_XFER_STD) 636 - EXCEPTION(0x2100, Trap_21, unknown_exception, EXC_XFER_STD) 637 - EXCEPTION(0x2200, Trap_22, unknown_exception, EXC_XFER_STD) 638 - EXCEPTION(0x2300, Trap_23, unknown_exception, EXC_XFER_STD) 639 - EXCEPTION(0x2400, Trap_24, unknown_exception, EXC_XFER_STD) 640 - EXCEPTION(0x2500, Trap_25, unknown_exception, EXC_XFER_STD) 641 - EXCEPTION(0x2600, Trap_26, unknown_exception, EXC_XFER_STD) 642 - EXCEPTION(0x2700, Trap_27, unknown_exception, EXC_XFER_STD) 643 - EXCEPTION(0x2800, Trap_28, unknown_exception, EXC_XFER_STD) 644 - EXCEPTION(0x2900, Trap_29, unknown_exception, EXC_XFER_STD) 645 - EXCEPTION(0x2a00, Trap_2a, unknown_exception, EXC_XFER_STD) 646 - EXCEPTION(0x2b00, Trap_2b, unknown_exception, EXC_XFER_STD) 647 - EXCEPTION(0x2c00, Trap_2c, unknown_exception, EXC_XFER_STD) 648 - EXCEPTION(0x2d00, Trap_2d, unknown_exception, EXC_XFER_STD) 649 - EXCEPTION(0x2e00, Trap_2e, unknown_exception, EXC_XFER_STD) 650 - EXCEPTION(0x2f00, Trap_2f, unknown_exception, EXC_XFER_STD) 653 + EXCEPTION(0x1300, Trap_13, instruction_breakpoint_exception) 654 + EXCEPTION(0x1400, SMI, SMIException) 655 + EXCEPTION(0x1500, Trap_15, unknown_exception) 656 + EXCEPTION(0x1600, Trap_16, altivec_assist_exception) 657 + EXCEPTION(0x1700, Trap_17, TAUException) 658 + EXCEPTION(0x1800, Trap_18, unknown_exception) 659 + EXCEPTION(0x1900, Trap_19, unknown_exception) 660 + EXCEPTION(0x1a00, Trap_1a, unknown_exception) 661 + EXCEPTION(0x1b00, Trap_1b, unknown_exception) 662 + EXCEPTION(0x1c00, Trap_1c, unknown_exception) 663 + EXCEPTION(0x1d00, Trap_1d, unknown_exception) 664 + EXCEPTION(0x1e00, Trap_1e, unknown_exception) 665 + EXCEPTION(0x1f00, Trap_1f, unknown_exception) 666 + EXCEPTION(0x2000, RunMode, RunModeException) 667 + EXCEPTION(0x2100, Trap_21, unknown_exception) 668 + EXCEPTION(0x2200, Trap_22, unknown_exception) 669 + EXCEPTION(0x2300, Trap_23, unknown_exception) 670 + EXCEPTION(0x2400, Trap_24, unknown_exception) 671 + EXCEPTION(0x2500, Trap_25, unknown_exception) 672 + EXCEPTION(0x2600, Trap_26, unknown_exception) 673 + EXCEPTION(0x2700, Trap_27, unknown_exception) 674 + EXCEPTION(0x2800, Trap_28, unknown_exception) 675 + EXCEPTION(0x2900, Trap_29, unknown_exception) 676 + EXCEPTION(0x2a00, Trap_2a, unknown_exception) 677 + EXCEPTION(0x2b00, Trap_2b, unknown_exception) 678 + EXCEPTION(0x2c00, Trap_2c, unknown_exception) 679 + EXCEPTION(0x2d00, Trap_2d, unknown_exception) 680 + EXCEPTION(0x2e00, Trap_2e, unknown_exception) 681 + EXCEPTION(0x2f00, Trap_2f, unknown_exception) 651 682 683 + __HEAD 652 684 . = 0x3000 653 685 654 - machine_check_tramp: 655 - EXC_XFER_STD(0x200, machine_check_exception) 656 - 657 - alignment_exception_tramp: 658 - EXC_XFER_STD(0x600, alignment_exception) 659 - 660 - handle_page_fault_tramp_1: 661 - #ifdef CONFIG_VMAP_STACK 662 - EXCEPTION_PROLOG_2 handle_dar_dsisr=1 663 - #endif 664 - lwz r5, _DSISR(r11) 665 - /* fall through */ 666 - handle_page_fault_tramp_2: 667 - andis. r0, r5, DSISR_DABRMATCH@h 668 - bne- 1f 669 - EXC_XFER_LITE(0x300, handle_page_fault) 670 - 1: EXC_XFER_STD(0x300, do_break) 671 - 672 - #ifdef CONFIG_VMAP_STACK 673 686 #ifdef CONFIG_PPC_BOOK3S_604 674 687 .macro save_regs_thread thread 675 688 stw r0, THR0(\thread) ··· 726 775 rfi 727 776 #endif /* CONFIG_PPC_BOOK3S_604 */ 728 777 729 - stack_overflow: 778 + #ifdef CONFIG_VMAP_STACK 730 779 vmap_stack_overflow_exception 731 780 #endif 732 781 782 + __HEAD 733 783 AltiVecUnavailable: 734 - EXCEPTION_PROLOG 784 + EXCEPTION_PROLOG 0xf20 AltiVecUnavailable 735 785 #ifdef CONFIG_ALTIVEC 736 786 beq 1f 737 787 bl load_up_altivec /* if from user, just load it up */ 738 788 b fast_exception_return 739 789 #endif /* CONFIG_ALTIVEC */ 740 - 1: addi r3,r1,STACK_FRAME_OVERHEAD 741 - EXC_XFER_LITE(0xf20, altivec_unavailable_exception) 790 + 1: prepare_transfer_to_handler 791 + bl altivec_unavailable_exception 792 + b interrupt_return 742 793 794 + __HEAD 743 795 PerformanceMonitor: 744 - EXCEPTION_PROLOG 745 - addi r3,r1,STACK_FRAME_OVERHEAD 746 - EXC_XFER_STD(0xf00, performance_monitor_exception) 796 + EXCEPTION_PROLOG 0xf00 PerformanceMonitor 797 + prepare_transfer_to_handler 798 + bl performance_monitor_exception 799 + b interrupt_return 747 800 748 801 802 + __HEAD 749 803 /* 750 804 * This code is jumped to from the startup code to copy 751 805 * the kernel image to physical address PHYSICAL_START.

+134 -69

arch/powerpc/kernel/head_booke.h

··· 44 44 #endif 45 45 46 46 47 - #define NORMAL_EXCEPTION_PROLOG(intno) \ 47 + #define NORMAL_EXCEPTION_PROLOG(trapno, intno) \ 48 48 mtspr SPRN_SPRG_WSCRATCH0, r10; /* save one register */ \ 49 49 mfspr r10, SPRN_SPRG_THREAD; \ 50 50 stw r11, THREAD_NORMSAVE(0)(r10); \ ··· 53 53 mfspr r11, SPRN_SRR1; \ 54 54 DO_KVM BOOKE_INTERRUPT_##intno SPRN_SRR1; \ 55 55 andi. r11, r11, MSR_PR; /* check whether user or kernel */\ 56 + LOAD_REG_IMMEDIATE(r11, MSR_KERNEL); \ 57 + mtmsr r11; \ 56 58 mr r11, r1; \ 57 59 beq 1f; \ 58 60 BOOKE_CLEAR_BTB(r11) \ ··· 78 76 stw r1, 0(r11); \ 79 77 mr r1, r11; \ 80 78 rlwinm r9,r9,0,14,12; /* clear MSR_WE (necessary?) */\ 81 - stw r0,GPR0(r11); \ 82 - lis r10, STACK_FRAME_REGS_MARKER@ha;/* exception frame marker */ \ 83 - addi r10, r10, STACK_FRAME_REGS_MARKER@l; \ 84 - stw r10, 8(r11); \ 85 - SAVE_4GPRS(3, r11); \ 86 - SAVE_2GPRS(7, r11) 79 + COMMON_EXCEPTION_PROLOG_END trapno 80 + 81 + .macro COMMON_EXCEPTION_PROLOG_END trapno 82 + stw r0,GPR0(r1) 83 + lis r10, STACK_FRAME_REGS_MARKER@ha /* exception frame marker */ 84 + addi r10, r10, STACK_FRAME_REGS_MARKER@l 85 + stw r10, 8(r1) 86 + li r10, \trapno 87 + stw r10,_TRAP(r1) 88 + SAVE_4GPRS(3, r1) 89 + SAVE_2GPRS(7, r1) 90 + SAVE_NVGPRS(r1) 91 + stw r2,GPR2(r1) 92 + stw r12,_NIP(r1) 93 + stw r9,_MSR(r1) 94 + mfctr r10 95 + mfspr r2,SPRN_SPRG_THREAD 96 + stw r10,_CTR(r1) 97 + tovirt(r2, r2) 98 + mfspr r10,SPRN_XER 99 + addi r2, r2, -THREAD 100 + stw r10,_XER(r1) 101 + addi r3,r1,STACK_FRAME_OVERHEAD 102 + .endm 103 + 104 + .macro prepare_transfer_to_handler 105 + #ifdef CONFIG_E500 106 + andi. r12,r9,MSR_PR 107 + bne 777f 108 + bl prepare_transfer_to_handler 109 + 777: 110 + #endif 111 + .endm 87 112 88 113 .macro SYSCALL_ENTRY trapno intno srr1 89 114 mfspr r10, SPRN_SPRG_THREAD ··· 209 180 * registers as the normal prolog above. Instead we use a portion of the 210 181 * critical/machine check exception stack at low physical addresses. 211 182 */ 212 - #define EXC_LEVEL_EXCEPTION_PROLOG(exc_level, intno, exc_level_srr0, exc_level_srr1) \ 183 + #define EXC_LEVEL_EXCEPTION_PROLOG(exc_level, trapno, intno, exc_level_srr0, exc_level_srr1) \ 213 184 mtspr SPRN_SPRG_WSCRATCH_##exc_level,r8; \ 214 185 BOOKE_LOAD_EXC_LEVEL_STACK(exc_level);/* r8 points to the exc_level stack*/ \ 215 186 stw r9,GPR9(r8); /* save various registers */\ ··· 221 192 DO_KVM BOOKE_INTERRUPT_##intno exc_level_srr1; \ 222 193 BOOKE_CLEAR_BTB(r10) \ 223 194 andi. r11,r11,MSR_PR; \ 195 + LOAD_REG_IMMEDIATE(r11, MSR_KERNEL & ~(MSR_ME|MSR_DE|MSR_CE)); \ 196 + mtmsr r11; \ 224 197 mfspr r11,SPRN_SPRG_THREAD; /* if from user, start at top of */\ 225 198 lwz r11, TASK_STACK - THREAD(r11); /* this thread's kernel stack */\ 226 199 addi r11,r11,EXC_LVL_FRAME_OVERHEAD; /* allocate stack frame */\ ··· 252 221 stw r1,0(r11); \ 253 222 mr r1,r11; \ 254 223 rlwinm r9,r9,0,14,12; /* clear MSR_WE (necessary?) */\ 255 - stw r0,GPR0(r11); \ 256 - SAVE_4GPRS(3, r11); \ 257 - SAVE_2GPRS(7, r11) 224 + COMMON_EXCEPTION_PROLOG_END trapno 258 225 259 - #define CRITICAL_EXCEPTION_PROLOG(intno) \ 260 - EXC_LEVEL_EXCEPTION_PROLOG(CRIT, intno, SPRN_CSRR0, SPRN_CSRR1) 261 - #define DEBUG_EXCEPTION_PROLOG \ 262 - EXC_LEVEL_EXCEPTION_PROLOG(DBG, DEBUG, SPRN_DSRR0, SPRN_DSRR1) 263 - #define MCHECK_EXCEPTION_PROLOG \ 264 - EXC_LEVEL_EXCEPTION_PROLOG(MC, MACHINE_CHECK, \ 226 + #define SAVE_xSRR(xSRR) \ 227 + mfspr r0,SPRN_##xSRR##0; \ 228 + stw r0,_##xSRR##0(r1); \ 229 + mfspr r0,SPRN_##xSRR##1; \ 230 + stw r0,_##xSRR##1(r1) 231 + 232 + 233 + .macro SAVE_MMU_REGS 234 + #ifdef CONFIG_PPC_BOOK3E_MMU 235 + mfspr r0,SPRN_MAS0 236 + stw r0,MAS0(r1) 237 + mfspr r0,SPRN_MAS1 238 + stw r0,MAS1(r1) 239 + mfspr r0,SPRN_MAS2 240 + stw r0,MAS2(r1) 241 + mfspr r0,SPRN_MAS3 242 + stw r0,MAS3(r1) 243 + mfspr r0,SPRN_MAS6 244 + stw r0,MAS6(r1) 245 + #ifdef CONFIG_PHYS_64BIT 246 + mfspr r0,SPRN_MAS7 247 + stw r0,MAS7(r1) 248 + #endif /* CONFIG_PHYS_64BIT */ 249 + #endif /* CONFIG_PPC_BOOK3E_MMU */ 250 + #ifdef CONFIG_44x 251 + mfspr r0,SPRN_MMUCR 252 + stw r0,MMUCR(r1) 253 + #endif 254 + .endm 255 + 256 + #define CRITICAL_EXCEPTION_PROLOG(trapno, intno) \ 257 + EXC_LEVEL_EXCEPTION_PROLOG(CRIT, trapno+2, intno, SPRN_CSRR0, SPRN_CSRR1) 258 + #define DEBUG_EXCEPTION_PROLOG(trapno) \ 259 + EXC_LEVEL_EXCEPTION_PROLOG(DBG, trapno+8, DEBUG, SPRN_DSRR0, SPRN_DSRR1) 260 + #define MCHECK_EXCEPTION_PROLOG(trapno) \ 261 + EXC_LEVEL_EXCEPTION_PROLOG(MC, trapno+4, MACHINE_CHECK, \ 265 262 SPRN_MCSRR0, SPRN_MCSRR1) 266 263 267 264 /* ··· 316 257 .align 5; \ 317 258 label: 318 259 319 - #define EXCEPTION(n, intno, label, hdlr, xfer) \ 260 + #define EXCEPTION(n, intno, label, hdlr) \ 320 261 START_EXCEPTION(label); \ 321 - NORMAL_EXCEPTION_PROLOG(intno); \ 322 - addi r3,r1,STACK_FRAME_OVERHEAD; \ 323 - xfer(n, hdlr) 262 + NORMAL_EXCEPTION_PROLOG(n, intno); \ 263 + prepare_transfer_to_handler; \ 264 + bl hdlr; \ 265 + b interrupt_return 324 266 325 267 #define CRITICAL_EXCEPTION(n, intno, label, hdlr) \ 326 268 START_EXCEPTION(label); \ 327 - CRITICAL_EXCEPTION_PROLOG(intno); \ 328 - addi r3,r1,STACK_FRAME_OVERHEAD; \ 329 - EXC_XFER_TEMPLATE(hdlr, n+2, (MSR_KERNEL & ~(MSR_ME|MSR_DE|MSR_CE)), \ 330 - crit_transfer_to_handler, ret_from_crit_exc) 269 + CRITICAL_EXCEPTION_PROLOG(n, intno); \ 270 + SAVE_MMU_REGS; \ 271 + SAVE_xSRR(SRR); \ 272 + prepare_transfer_to_handler; \ 273 + bl hdlr; \ 274 + b ret_from_crit_exc 331 275 332 276 #define MCHECK_EXCEPTION(n, label, hdlr) \ 333 277 START_EXCEPTION(label); \ 334 - MCHECK_EXCEPTION_PROLOG; \ 278 + MCHECK_EXCEPTION_PROLOG(n); \ 335 279 mfspr r5,SPRN_ESR; \ 336 280 stw r5,_ESR(r11); \ 337 - addi r3,r1,STACK_FRAME_OVERHEAD; \ 338 - EXC_XFER_TEMPLATE(hdlr, n+4, (MSR_KERNEL & ~(MSR_ME|MSR_DE|MSR_CE)), \ 339 - mcheck_transfer_to_handler, ret_from_mcheck_exc) 340 - 341 - #define EXC_XFER_TEMPLATE(hdlr, trap, msr, tfer, ret) \ 342 - li r10,trap; \ 343 - stw r10,_TRAP(r11); \ 344 - lis r10,msr@h; \ 345 - ori r10,r10,msr@l; \ 346 - bl tfer; \ 347 - .long hdlr; \ 348 - .long ret 349 - 350 - #define EXC_XFER_STD(n, hdlr) \ 351 - EXC_XFER_TEMPLATE(hdlr, n, MSR_KERNEL, transfer_to_handler_full, \ 352 - ret_from_except_full) 353 - 354 - #define EXC_XFER_LITE(n, hdlr) \ 355 - EXC_XFER_TEMPLATE(hdlr, n+1, MSR_KERNEL, transfer_to_handler, \ 356 - ret_from_except) 281 + SAVE_xSRR(DSRR); \ 282 + SAVE_xSRR(CSRR); \ 283 + SAVE_MMU_REGS; \ 284 + SAVE_xSRR(SRR); \ 285 + prepare_transfer_to_handler; \ 286 + bl hdlr; \ 287 + b ret_from_mcheck_exc 357 288 358 289 /* Check for a single step debug exception while in an exception 359 290 * handler before state has been saved. This is to catch the case ··· 360 311 */ 361 312 #define DEBUG_DEBUG_EXCEPTION \ 362 313 START_EXCEPTION(DebugDebug); \ 363 - DEBUG_EXCEPTION_PROLOG; \ 314 + DEBUG_EXCEPTION_PROLOG(2000); \ 364 315 \ 365 316 /* \ 366 317 * If there is a single step or branch-taken exception in an \ ··· 409 360 /* continue normal handling for a debug exception... */ \ 410 361 2: mfspr r4,SPRN_DBSR; \ 411 362 stw r4,_ESR(r11); /* DebugException takes DBSR in _ESR */\ 412 - addi r3,r1,STACK_FRAME_OVERHEAD; \ 413 - EXC_XFER_TEMPLATE(DebugException, 0x2008, (MSR_KERNEL & ~(MSR_ME|MSR_DE|MSR_CE)), debug_transfer_to_handler, ret_from_debug_exc) 363 + SAVE_xSRR(CSRR); \ 364 + SAVE_MMU_REGS; \ 365 + SAVE_xSRR(SRR); \ 366 + prepare_transfer_to_handler; \ 367 + bl DebugException; \ 368 + b ret_from_debug_exc 414 369 415 370 #define DEBUG_CRIT_EXCEPTION \ 416 371 START_EXCEPTION(DebugCrit); \ 417 - CRITICAL_EXCEPTION_PROLOG(DEBUG); \ 372 + CRITICAL_EXCEPTION_PROLOG(2000,DEBUG); \ 418 373 \ 419 374 /* \ 420 375 * If there is a single step or branch-taken exception in an \ ··· 467 414 /* continue normal handling for a critical exception... */ \ 468 415 2: mfspr r4,SPRN_DBSR; \ 469 416 stw r4,_ESR(r11); /* DebugException takes DBSR in _ESR */\ 470 - addi r3,r1,STACK_FRAME_OVERHEAD; \ 471 - EXC_XFER_TEMPLATE(DebugException, 0x2002, (MSR_KERNEL & ~(MSR_ME|MSR_DE|MSR_CE)), crit_transfer_to_handler, ret_from_crit_exc) 417 + SAVE_MMU_REGS; \ 418 + SAVE_xSRR(SRR); \ 419 + prepare_transfer_to_handler; \ 420 + bl DebugException; \ 421 + b ret_from_crit_exc 472 422 473 423 #define DATA_STORAGE_EXCEPTION \ 474 424 START_EXCEPTION(DataStorage) \ 475 - NORMAL_EXCEPTION_PROLOG(DATA_STORAGE); \ 425 + NORMAL_EXCEPTION_PROLOG(0x300, DATA_STORAGE); \ 476 426 mfspr r5,SPRN_ESR; /* Grab the ESR and save it */ \ 477 427 stw r5,_ESR(r11); \ 478 428 mfspr r4,SPRN_DEAR; /* Grab the DEAR */ \ 479 429 stw r4, _DEAR(r11); \ 480 - EXC_XFER_LITE(0x0300, handle_page_fault) 430 + prepare_transfer_to_handler; \ 431 + bl do_page_fault; \ 432 + b interrupt_return 481 433 482 434 #define INSTRUCTION_STORAGE_EXCEPTION \ 483 435 START_EXCEPTION(InstructionStorage) \ 484 - NORMAL_EXCEPTION_PROLOG(INST_STORAGE); \ 436 + NORMAL_EXCEPTION_PROLOG(0x400, INST_STORAGE); \ 485 437 mfspr r5,SPRN_ESR; /* Grab the ESR and save it */ \ 486 438 stw r5,_ESR(r11); \ 487 439 stw r12, _DEAR(r11); /* Pass SRR0 as arg2 */ \ 488 - EXC_XFER_LITE(0x0400, handle_page_fault) 440 + prepare_transfer_to_handler; \ 441 + bl do_page_fault; \ 442 + b interrupt_return 489 443 490 444 #define ALIGNMENT_EXCEPTION \ 491 445 START_EXCEPTION(Alignment) \ 492 - NORMAL_EXCEPTION_PROLOG(ALIGNMENT); \ 446 + NORMAL_EXCEPTION_PROLOG(0x600, ALIGNMENT); \ 493 447 mfspr r4,SPRN_DEAR; /* Grab the DEAR and save it */ \ 494 448 stw r4,_DEAR(r11); \ 495 - addi r3,r1,STACK_FRAME_OVERHEAD; \ 496 - EXC_XFER_STD(0x0600, alignment_exception) 449 + prepare_transfer_to_handler; \ 450 + bl alignment_exception; \ 451 + REST_NVGPRS(r1); \ 452 + b interrupt_return 497 453 498 454 #define PROGRAM_EXCEPTION \ 499 455 START_EXCEPTION(Program) \ 500 - NORMAL_EXCEPTION_PROLOG(PROGRAM); \ 456 + NORMAL_EXCEPTION_PROLOG(0x700, PROGRAM); \ 501 457 mfspr r4,SPRN_ESR; /* Grab the ESR and save it */ \ 502 458 stw r4,_ESR(r11); \ 503 - addi r3,r1,STACK_FRAME_OVERHEAD; \ 504 - EXC_XFER_STD(0x0700, program_check_exception) 459 + prepare_transfer_to_handler; \ 460 + bl program_check_exception; \ 461 + REST_NVGPRS(r1); \ 462 + b interrupt_return 505 463 506 464 #define DECREMENTER_EXCEPTION \ 507 465 START_EXCEPTION(Decrementer) \ 508 - NORMAL_EXCEPTION_PROLOG(DECREMENTER); \ 466 + NORMAL_EXCEPTION_PROLOG(0x900, DECREMENTER); \ 509 467 lis r0,TSR_DIS@h; /* Setup the DEC interrupt mask */ \ 510 468 mtspr SPRN_TSR,r0; /* Clear the DEC interrupt */ \ 511 - addi r3,r1,STACK_FRAME_OVERHEAD; \ 512 - EXC_XFER_LITE(0x0900, timer_interrupt) 469 + prepare_transfer_to_handler; \ 470 + bl timer_interrupt; \ 471 + b interrupt_return 513 472 514 473 #define FP_UNAVAILABLE_EXCEPTION \ 515 474 START_EXCEPTION(FloatingPointUnavailable) \ 516 - NORMAL_EXCEPTION_PROLOG(FP_UNAVAIL); \ 475 + NORMAL_EXCEPTION_PROLOG(0x800, FP_UNAVAIL); \ 517 476 beq 1f; \ 518 477 bl load_up_fpu; /* if from user, just load it up */ \ 519 478 b fast_exception_return; \ 520 - 1: addi r3,r1,STACK_FRAME_OVERHEAD; \ 521 - EXC_XFER_STD(0x800, kernel_fp_unavailable_exception) 479 + 1: prepare_transfer_to_handler; \ 480 + bl kernel_fp_unavailable_exception; \ 481 + b interrupt_return 522 482 523 483 #else /* __ASSEMBLY__ */ 524 484 struct exception_regs { ··· 547 481 unsigned long csrr1; 548 482 unsigned long dsrr0; 549 483 unsigned long dsrr1; 550 - unsigned long saved_ksp_limit; 551 484 }; 552 485 553 486 /* ensure this structure is always sized to a multiple of the stack alignment */

+38 -32

arch/powerpc/kernel/head_fsl_booke.S

··· 113 113 114 114 1: 115 115 /* 116 - * We have the runtime (virutal) address of our base. 116 + * We have the runtime (virtual) address of our base. 117 117 * We calculate our shift of offset from a 64M page. 118 118 * We could map the 64M page we belong to at PAGE_OFFSET and 119 119 * get going from there. ··· 363 363 364 364 /* Data Storage Interrupt */ 365 365 START_EXCEPTION(DataStorage) 366 - NORMAL_EXCEPTION_PROLOG(DATA_STORAGE) 366 + NORMAL_EXCEPTION_PROLOG(0x300, DATA_STORAGE) 367 367 mfspr r5,SPRN_ESR /* Grab the ESR, save it */ 368 368 stw r5,_ESR(r11) 369 369 mfspr r4,SPRN_DEAR /* Grab the DEAR, save it */ 370 370 stw r4, _DEAR(r11) 371 371 andis. r10,r5,(ESR_ILK|ESR_DLK)@h 372 372 bne 1f 373 - EXC_XFER_LITE(0x0300, handle_page_fault) 373 + prepare_transfer_to_handler 374 + bl do_page_fault 375 + b interrupt_return 374 376 1: 375 - addi r3,r1,STACK_FRAME_OVERHEAD 376 - EXC_XFER_LITE(0x0300, CacheLockingException) 377 + prepare_transfer_to_handler 378 + bl CacheLockingException 379 + b interrupt_return 377 380 378 381 /* Instruction Storage Interrupt */ 379 382 INSTRUCTION_STORAGE_EXCEPTION 380 383 381 384 /* External Input Interrupt */ 382 - EXCEPTION(0x0500, EXTERNAL, ExternalInput, do_IRQ, EXC_XFER_LITE) 385 + EXCEPTION(0x0500, EXTERNAL, ExternalInput, do_IRQ) 383 386 384 387 /* Alignment Interrupt */ 385 388 ALIGNMENT_EXCEPTION ··· 394 391 #ifdef CONFIG_PPC_FPU 395 392 FP_UNAVAILABLE_EXCEPTION 396 393 #else 397 - EXCEPTION(0x0800, FP_UNAVAIL, FloatingPointUnavailable, \ 398 - unknown_exception, EXC_XFER_STD) 394 + EXCEPTION(0x0800, FP_UNAVAIL, FloatingPointUnavailable, unknown_exception) 399 395 #endif 400 396 401 397 /* System Call Interrupt */ ··· 402 400 SYSCALL_ENTRY 0xc00 BOOKE_INTERRUPT_SYSCALL SPRN_SRR1 403 401 404 402 /* Auxiliary Processor Unavailable Interrupt */ 405 - EXCEPTION(0x2900, AP_UNAVAIL, AuxillaryProcessorUnavailable, \ 406 - unknown_exception, EXC_XFER_STD) 403 + EXCEPTION(0x2900, AP_UNAVAIL, AuxillaryProcessorUnavailable, unknown_exception) 407 404 408 405 /* Decrementer Interrupt */ 409 406 DECREMENTER_EXCEPTION 410 407 411 408 /* Fixed Internal Timer Interrupt */ 412 409 /* TODO: Add FIT support */ 413 - EXCEPTION(0x3100, FIT, FixedIntervalTimer, \ 414 - unknown_exception, EXC_XFER_STD) 410 + EXCEPTION(0x3100, FIT, FixedIntervalTimer, unknown_exception) 415 411 416 412 /* Watchdog Timer Interrupt */ 417 413 #ifdef CONFIG_BOOKE_WDT ··· 497 497 #endif 498 498 #endif 499 499 500 - bne 2f /* Bail if permission/valid mismach */ 500 + bne 2f /* Bail if permission/valid mismatch */ 501 501 502 502 /* Jump to common tlb load */ 503 503 b finish_tlb_load ··· 592 592 #endif 593 593 #endif 594 594 595 - bne 2f /* Bail if permission mismach */ 595 + bne 2f /* Bail if permission mismatch */ 596 596 597 597 /* Jump to common TLB load point */ 598 598 b finish_tlb_load ··· 614 614 #ifdef CONFIG_SPE 615 615 /* SPE Unavailable */ 616 616 START_EXCEPTION(SPEUnavailable) 617 - NORMAL_EXCEPTION_PROLOG(SPE_UNAVAIL) 617 + NORMAL_EXCEPTION_PROLOG(0x2010, SPE_UNAVAIL) 618 618 beq 1f 619 619 bl load_up_spe 620 620 b fast_exception_return 621 - 1: addi r3,r1,STACK_FRAME_OVERHEAD 622 - EXC_XFER_LITE(0x2010, KernelSPE) 621 + 1: prepare_transfer_to_handler 622 + bl KernelSPE 623 + b interrupt_return 623 624 #elif defined(CONFIG_SPE_POSSIBLE) 624 - EXCEPTION(0x2020, SPE_UNAVAIL, SPEUnavailable, \ 625 - unknown_exception, EXC_XFER_STD) 625 + EXCEPTION(0x2020, SPE_UNAVAIL, SPEUnavailable, unknown_exception) 626 626 #endif /* CONFIG_SPE_POSSIBLE */ 627 627 628 628 /* SPE Floating Point Data */ 629 629 #ifdef CONFIG_SPE 630 - EXCEPTION(0x2030, SPE_FP_DATA, SPEFloatingPointData, 631 - SPEFloatingPointException, EXC_XFER_STD) 630 + START_EXCEPTION(SPEFloatingPointData) 631 + NORMAL_EXCEPTION_PROLOG(0x2030, SPE_FP_DATA) 632 + prepare_transfer_to_handler 633 + bl SPEFloatingPointException 634 + REST_NVGPRS(r1) 635 + b interrupt_return 632 636 633 637 /* SPE Floating Point Round */ 634 - EXCEPTION(0x2050, SPE_FP_ROUND, SPEFloatingPointRound, \ 635 - SPEFloatingPointRoundException, EXC_XFER_STD) 638 + START_EXCEPTION(SPEFloatingPointRound) 639 + NORMAL_EXCEPTION_PROLOG(0x2050, SPE_FP_ROUND) 640 + prepare_transfer_to_handler 641 + bl SPEFloatingPointRoundException 642 + REST_NVGPRS(r1) 643 + b interrupt_return 636 644 #elif defined(CONFIG_SPE_POSSIBLE) 637 - EXCEPTION(0x2040, SPE_FP_DATA, SPEFloatingPointData, 638 - unknown_exception, EXC_XFER_STD) 639 - EXCEPTION(0x2050, SPE_FP_ROUND, SPEFloatingPointRound, \ 640 - unknown_exception, EXC_XFER_STD) 645 + EXCEPTION(0x2040, SPE_FP_DATA, SPEFloatingPointData, unknown_exception) 646 + EXCEPTION(0x2050, SPE_FP_ROUND, SPEFloatingPointRound, unknown_exception) 641 647 #endif /* CONFIG_SPE_POSSIBLE */ 642 648 643 649 644 650 /* Performance Monitor */ 645 651 EXCEPTION(0x2060, PERFORMANCE_MONITOR, PerformanceMonitor, \ 646 - performance_monitor_exception, EXC_XFER_STD) 652 + performance_monitor_exception) 647 653 648 - EXCEPTION(0x2070, DOORBELL, Doorbell, doorbell_exception, EXC_XFER_STD) 654 + EXCEPTION(0x2070, DOORBELL, Doorbell, doorbell_exception) 649 655 650 656 CRITICAL_EXCEPTION(0x2080, DOORBELL_CRITICAL, \ 651 657 CriticalDoorbell, unknown_exception) ··· 666 660 unknown_exception) 667 661 668 662 /* Hypercall */ 669 - EXCEPTION(0, HV_SYSCALL, Hypercall, unknown_exception, EXC_XFER_STD) 663 + EXCEPTION(0, HV_SYSCALL, Hypercall, unknown_exception) 670 664 671 665 /* Embedded Hypervisor Privilege */ 672 - EXCEPTION(0, HV_PRIV, Ehvpriv, unknown_exception, EXC_XFER_STD) 666 + EXCEPTION(0, HV_PRIV, Ehvpriv, unknown_exception) 673 667 674 668 interrupt_end: 675 669 ··· 860 854 lwz r5,_NIP(r1) 861 855 bl printk 862 856 #endif 863 - b ret_from_except 857 + b interrupt_return 864 858 #ifdef CONFIG_PRINTK 865 859 87: .string "SPE used in kernel (task=%p, pc=%x) \n" 866 860 #endif

+1 -1

arch/powerpc/kernel/hw_breakpoint_constraints.c

··· 141 141 { 142 142 struct instruction_op op; 143 143 144 - if (__get_user_instr_inatomic(*instr, (void __user *)regs->nip)) 144 + if (__get_user_instr(*instr, (void __user *)regs->nip)) 145 145 return; 146 146 147 147 analyse_instr(&op, regs, *instr);

+2 -12

arch/powerpc/kernel/idle_6xx.S

··· 145 145 146 146 /* 147 147 * Return from NAP/DOZE mode, restore some CPU specific registers, 148 - * we are called with DR/IR still off and r2 containing physical 149 - * address of current. R11 points to the exception frame (physical 150 - * address). We have to preserve r10. 148 + * R11 points to the exception frame. We have to preserve r10. 151 149 */ 152 150 _GLOBAL(power_save_ppc32_restore) 153 151 lwz r9,_LINK(r11) /* interrupted in ppc6xx_idle: */ ··· 164 166 mfspr r9,SPRN_HID0 165 167 andis. r9,r9,HID0_NAP@h 166 168 beq 1f 167 - #ifdef CONFIG_VMAP_STACK 168 169 addis r9, r11, nap_save_msscr0@ha 169 - #else 170 - addis r9,r11,(nap_save_msscr0-KERNELBASE)@ha 171 - #endif 172 170 lwz r9,nap_save_msscr0@l(r9) 173 171 mtspr SPRN_MSSCR0, r9 174 172 sync ··· 172 178 1: 173 179 END_FTR_SECTION_IFSET(CPU_FTR_NAP_DISABLE_L2_PR) 174 180 BEGIN_FTR_SECTION 175 - #ifdef CONFIG_VMAP_STACK 176 181 addis r9, r11, nap_save_hid1@ha 177 - #else 178 - addis r9,r11,(nap_save_hid1-KERNELBASE)@ha 179 - #endif 180 182 lwz r9,nap_save_hid1@l(r9) 181 183 mtspr SPRN_HID1, r9 182 184 END_FTR_SECTION_IFSET(CPU_FTR_DUAL_PLL_750FX) 183 - b transfer_to_handler_cont 185 + blr 184 186 _ASM_NOKPROBE_SYMBOL(power_save_ppc32_restore) 185 187 186 188 .data

+4

arch/powerpc/kernel/idle_book3s.S

··· 209 209 mtmsrd r7 210 210 isync 211 211 b 1b 212 + 213 + .globl power4_idle_nap_return 214 + power4_idle_nap_return: 215 + blr 212 216 #endif

+3 -11

arch/powerpc/kernel/idle_e500.S

··· 74 74 75 75 /* 76 76 * Return from NAP/DOZE mode, restore some CPU specific registers, 77 - * r2 containing physical address of current. 78 - * r11 points to the exception frame (physical address). 77 + * r2 containing address of current. 78 + * r11 points to the exception frame. 79 79 * We have to preserve r10. 80 80 */ 81 81 _GLOBAL(power_save_ppc32_restore) 82 82 lwz r9,_LINK(r11) /* interrupted in e500_idle */ 83 83 stw r9,_NIP(r11) /* make it do a blr */ 84 - 85 - #ifdef CONFIG_SMP 86 - lwz r11,TASK_CPU(r2) /* get cpu number * 4 */ 87 - slwi r11,r11,2 88 - #else 89 - li r11,0 90 - #endif 91 - 92 - b transfer_to_handler_cont 84 + blr 93 85 _ASM_NOKPROBE_SYMBOL(power_save_ppc32_restore)

+27 -40

arch/powerpc/kernel/interrupt.c

··· 20 20 #include <asm/time.h> 21 21 #include <asm/unistd.h> 22 22 23 + #if defined(CONFIG_PPC_ADV_DEBUG_REGS) && defined(CONFIG_PPC32) 24 + unsigned long global_dbcr0[NR_CPUS]; 25 + #endif 26 + 23 27 typedef long (*syscall_fn)(long, long, long, long, long, long); 24 28 25 29 /* Has to run notrace because it is entered not completely "reconciled" */ ··· 33 29 { 34 30 syscall_fn f; 35 31 32 + kuep_lock(); 33 + #ifdef CONFIG_PPC32 34 + kuap_save_and_lock(regs); 35 + #endif 36 + 36 37 regs->orig_gpr3 = r3; 37 38 38 39 if (IS_ENABLED(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG)) 39 40 BUG_ON(irq_soft_mask_return() != IRQS_ALL_DISABLED); 40 41 42 + trace_hardirqs_off(); /* finish reconciling */ 43 + 41 44 CT_WARN_ON(ct_state() == CONTEXT_KERNEL); 42 45 user_exit_irqoff(); 43 - 44 - trace_hardirqs_off(); /* finish reconciling */ 45 46 46 47 if (!IS_ENABLED(CONFIG_BOOKE) && !IS_ENABLED(CONFIG_40x)) 47 48 BUG_ON(!(regs->msr & MSR_RI)); 48 49 BUG_ON(!(regs->msr & MSR_PR)); 49 - BUG_ON(!FULL_REGS(regs)); 50 50 BUG_ON(arch_irq_disabled_regs(regs)); 51 51 52 52 #ifdef CONFIG_PPC_PKEY ··· 77 69 isync(); 78 70 } else 79 71 #endif 80 - #ifdef CONFIG_PPC64 81 - kuap_check_amr(); 82 - #endif 72 + kuap_assert_locked(); 83 73 84 74 booke_restore_dbcr0(); 85 75 ··· 253 247 254 248 CT_WARN_ON(ct_state() == CONTEXT_USER); 255 249 256 - #ifdef CONFIG_PPC64 257 - kuap_check_amr(); 258 - #endif 250 + kuap_assert_locked(); 259 251 260 252 regs->result = r3; 261 253 ··· 348 344 349 345 account_cpu_user_exit(); 350 346 351 - #ifdef CONFIG_PPC_BOOK3S_64 /* BOOK3E and ppc32 not using this */ 352 - /* 353 - * We do this at the end so that we do context switch with KERNEL AMR 354 - */ 347 + /* Restore user access locks last */ 355 348 kuap_user_restore(regs); 356 - #endif 349 + kuep_unlock(); 350 + 357 351 return ret; 358 352 } 359 353 360 - #ifndef CONFIG_PPC_BOOK3E_64 /* BOOK3E not yet using this */ 361 354 notrace unsigned long interrupt_exit_user_prepare(struct pt_regs *regs, unsigned long msr) 362 355 { 363 356 unsigned long ti_flags; ··· 364 363 if (!IS_ENABLED(CONFIG_BOOKE) && !IS_ENABLED(CONFIG_40x)) 365 364 BUG_ON(!(regs->msr & MSR_RI)); 366 365 BUG_ON(!(regs->msr & MSR_PR)); 367 - BUG_ON(!FULL_REGS(regs)); 368 366 BUG_ON(arch_irq_disabled_regs(regs)); 369 367 CT_WARN_ON(ct_state() == CONTEXT_USER); 370 368 ··· 371 371 * We don't need to restore AMR on the way back to userspace for KUAP. 372 372 * AMR can only have been unlocked if we interrupted the kernel. 373 373 */ 374 - #ifdef CONFIG_PPC64 375 - kuap_check_amr(); 376 - #endif 374 + kuap_assert_locked(); 377 375 378 376 local_irq_save(flags); 379 377 ··· 390 392 ti_flags = READ_ONCE(current_thread_info()->flags); 391 393 } 392 394 393 - if (IS_ENABLED(CONFIG_PPC_BOOK3S) && IS_ENABLED(CONFIG_PPC_FPU)) { 395 + if (IS_ENABLED(CONFIG_PPC_BOOK3S_64) && IS_ENABLED(CONFIG_PPC_FPU)) { 394 396 if (IS_ENABLED(CONFIG_PPC_TRANSACTIONAL_MEM) && 395 397 unlikely((ti_flags & _TIF_RESTORE_TM))) { 396 398 restore_tm_state(regs); ··· 425 427 426 428 account_cpu_user_exit(); 427 429 428 - /* 429 - * We do this at the end so that we do context switch with KERNEL AMR 430 - */ 431 - #ifdef CONFIG_PPC64 430 + /* Restore user access locks last */ 432 431 kuap_user_restore(regs); 433 - #endif 432 + 434 433 return ret; 435 434 } 436 435 ··· 437 442 { 438 443 unsigned long flags; 439 444 unsigned long ret = 0; 440 - #ifdef CONFIG_PPC64 441 - unsigned long amr; 442 - #endif 445 + unsigned long kuap; 443 446 444 447 if (!IS_ENABLED(CONFIG_BOOKE) && !IS_ENABLED(CONFIG_40x) && 445 448 unlikely(!(regs->msr & MSR_RI))) 446 449 unrecoverable_exception(regs); 447 450 BUG_ON(regs->msr & MSR_PR); 448 - BUG_ON(!FULL_REGS(regs)); 449 451 /* 450 452 * CT_WARN_ON comes here via program_check_exception, 451 453 * so avoid recursion. 452 454 */ 453 - if (TRAP(regs) != 0x700) 455 + if (TRAP(regs) != INTERRUPT_PROGRAM) 454 456 CT_WARN_ON(ct_state() == CONTEXT_USER); 455 457 456 - #ifdef CONFIG_PPC64 457 - amr = kuap_get_and_check_amr(); 458 - #endif 458 + kuap = kuap_get_and_assert_locked(); 459 459 460 460 if (unlikely(current_thread_info()->flags & _TIF_EMULATE_STACK_STORE)) { 461 461 clear_bits(_TIF_EMULATE_STACK_STORE, &current_thread_info()->flags); ··· 488 498 #endif 489 499 490 500 /* 491 - * Don't want to mfspr(SPRN_AMR) here, because this comes after mtmsr, 492 - * which would cause Read-After-Write stalls. Hence, we take the AMR 493 - * value from the check above. 501 + * 64s does not want to mfspr(SPRN_AMR) here, because this comes after 502 + * mtmsr, which would cause Read-After-Write stalls. Hence, take the 503 + * AMR value from the check above. 494 504 */ 495 - #ifdef CONFIG_PPC64 496 - kuap_kernel_restore(regs, amr); 497 - #endif 505 + kuap_kernel_restore(regs, kuap); 498 506 499 507 return ret; 500 508 } 501 - #endif

+24 -22

arch/powerpc/kernel/iommu.c

··· 72 72 73 73 sprintf(name, "%08lx", tbl->it_index); 74 74 liobn_entry = debugfs_lookup(name, iommu_debugfs_dir); 75 - if (liobn_entry) 76 - debugfs_remove(liobn_entry); 75 + debugfs_remove(liobn_entry); 77 76 } 78 77 #else 79 78 static void iommu_debugfs_add(struct iommu_table *tbl){} ··· 292 293 pool_nr = (pool_nr + 1) & (tbl->nr_pools - 1); 293 294 pool = &tbl->pools[pool_nr]; 294 295 spin_lock(&(pool->lock)); 296 + pool->hint = pool->start; 297 + pass++; 298 + goto again; 299 + 300 + } else if (pass == tbl->nr_pools + 1) { 301 + /* Last resort: try largepool */ 302 + spin_unlock(&pool->lock); 303 + pool = &tbl->large_pool; 304 + spin_lock(&pool->lock); 295 305 pool->hint = pool->start; 296 306 pass++; 297 307 goto again; ··· 727 719 { 728 720 unsigned long sz; 729 721 static int welcomed = 0; 730 - struct page *page; 731 722 unsigned int i; 732 723 struct iommu_pool *p; 733 724 ··· 735 728 /* number of bytes needed for the bitmap */ 736 729 sz = BITS_TO_LONGS(tbl->it_size) * sizeof(unsigned long); 737 730 738 - page = alloc_pages_node(nid, GFP_KERNEL, get_order(sz)); 739 - if (!page) 740 - panic("iommu_init_table: Can't allocate %ld bytes\n", sz); 741 - tbl->it_map = page_address(page); 742 - memset(tbl->it_map, 0, sz); 731 + tbl->it_map = vzalloc_node(sz, nid); 732 + if (!tbl->it_map) { 733 + pr_err("%s: Can't allocate %ld bytes\n", __func__, sz); 734 + return NULL; 735 + } 743 736 744 737 iommu_table_reserve_pages(tbl, res_start, res_end); 745 738 ··· 781 774 782 775 static void iommu_table_free(struct kref *kref) 783 776 { 784 - unsigned long bitmap_sz; 785 - unsigned int order; 786 777 struct iommu_table *tbl; 787 778 788 779 tbl = container_of(kref, struct iommu_table, it_kref); ··· 801 796 if (!bitmap_empty(tbl->it_map, tbl->it_size)) 802 797 pr_warn("%s: Unexpected TCEs\n", __func__); 803 798 804 - /* calculate bitmap size in bytes */ 805 - bitmap_sz = BITS_TO_LONGS(tbl->it_size) * sizeof(unsigned long); 806 - 807 799 /* free bitmap */ 808 - order = get_order(bitmap_sz); 809 - free_pages((unsigned long) tbl->it_map, order); 800 + vfree(tbl->it_map); 810 801 811 802 /* free table */ 812 803 kfree(tbl); ··· 898 897 unsigned int order; 899 898 unsigned int nio_pages, io_order; 900 899 struct page *page; 900 + size_t size_io = size; 901 901 902 902 size = PAGE_ALIGN(size); 903 903 order = get_order(size); ··· 925 923 memset(ret, 0, size); 926 924 927 925 /* Set up tces to cover the allocated range */ 928 - nio_pages = size >> tbl->it_page_shift; 929 - io_order = get_iommu_order(size, tbl); 926 + size_io = IOMMU_PAGE_ALIGN(size_io, tbl); 927 + nio_pages = size_io >> tbl->it_page_shift; 928 + io_order = get_iommu_order(size_io, tbl); 930 929 mapping = iommu_alloc(dev, tbl, ret, nio_pages, DMA_BIDIRECTIONAL, 931 930 mask >> tbl->it_page_shift, io_order, 0); 932 931 if (mapping == DMA_MAPPING_ERROR) { ··· 942 939 void *vaddr, dma_addr_t dma_handle) 943 940 { 944 941 if (tbl) { 945 - unsigned int nio_pages; 942 + size_t size_io = IOMMU_PAGE_ALIGN(size, tbl); 943 + unsigned int nio_pages = size_io >> tbl->it_page_shift; 946 944 947 - size = PAGE_ALIGN(size); 948 - nio_pages = size >> tbl->it_page_shift; 949 945 iommu_free(tbl, dma_handle, nio_pages); 950 946 size = PAGE_ALIGN(size); 951 947 free_pages((unsigned long)vaddr, get_order(size)); ··· 1098 1096 1099 1097 spin_lock_irqsave(&tbl->large_pool.lock, flags); 1100 1098 for (i = 0; i < tbl->nr_pools; i++) 1101 - spin_lock(&tbl->pools[i].lock); 1099 + spin_lock_nest_lock(&tbl->pools[i].lock, &tbl->large_pool.lock); 1102 1100 1103 1101 iommu_table_release_pages(tbl); 1104 1102 ··· 1126 1124 1127 1125 spin_lock_irqsave(&tbl->large_pool.lock, flags); 1128 1126 for (i = 0; i < tbl->nr_pools; i++) 1129 - spin_lock(&tbl->pools[i].lock); 1127 + spin_lock_nest_lock(&tbl->pools[i].lock, &tbl->large_pool.lock); 1130 1128 1131 1129 memset(tbl->it_map, 0, sz); 1132 1130

+47 -85

arch/powerpc/kernel/irq.c

··· 104 104 return happened; 105 105 } 106 106 107 - #ifdef CONFIG_PPC_BOOK3E 108 - 109 - /* This is called whenever we are re-enabling interrupts 110 - * and returns either 0 (nothing to do) or 500/900/280 if 111 - * there's an EE, DEC or DBELL to generate. 112 - * 113 - * This is called in two contexts: From arch_local_irq_restore() 114 - * before soft-enabling interrupts, and from the exception exit 115 - * path when returning from an interrupt from a soft-disabled to 116 - * a soft enabled context. In both case we have interrupts hard 117 - * disabled. 118 - * 119 - * We take care of only clearing the bits we handled in the 120 - * PACA irq_happened field since we can only re-emit one at a 121 - * time and we don't want to "lose" one. 122 - */ 123 - notrace unsigned int __check_irq_replay(void) 124 - { 125 - /* 126 - * We use local_paca rather than get_paca() to avoid all 127 - * the debug_smp_processor_id() business in this low level 128 - * function 129 - */ 130 - unsigned char happened = local_paca->irq_happened; 131 - 132 - /* 133 - * We are responding to the next interrupt, so interrupt-off 134 - * latencies should be reset here. 135 - */ 136 - trace_hardirqs_on(); 137 - trace_hardirqs_off(); 138 - 139 - if (happened & PACA_IRQ_DEC) { 140 - local_paca->irq_happened &= ~PACA_IRQ_DEC; 141 - return 0x900; 142 - } 143 - 144 - if (happened & PACA_IRQ_EE) { 145 - local_paca->irq_happened &= ~PACA_IRQ_EE; 146 - return 0x500; 147 - } 148 - 149 - if (happened & PACA_IRQ_DBELL) { 150 - local_paca->irq_happened &= ~PACA_IRQ_DBELL; 151 - return 0x280; 152 - } 153 - 154 - if (happened & PACA_IRQ_HARD_DIS) 155 - local_paca->irq_happened &= ~PACA_IRQ_HARD_DIS; 156 - 157 - /* There should be nothing left ! */ 158 - BUG_ON(local_paca->irq_happened != 0); 159 - 160 - return 0; 161 - } 162 - 163 - /* 164 - * This is specifically called by assembly code to re-enable interrupts 165 - * if they are currently disabled. This is typically called before 166 - * schedule() or do_signal() when returning to userspace. We do it 167 - * in C to avoid the burden of dealing with lockdep etc... 168 - * 169 - * NOTE: This is called with interrupts hard disabled but not marked 170 - * as such in paca->irq_happened, so we need to resync this. 171 - */ 172 - void notrace restore_interrupts(void) 173 - { 174 - if (irqs_disabled()) { 175 - local_paca->irq_happened |= PACA_IRQ_HARD_DIS; 176 - local_irq_enable(); 177 - } else 178 - __hard_irq_enable(); 179 - } 180 - 181 - #endif /* CONFIG_PPC_BOOK3E */ 182 - 183 107 void replay_soft_interrupts(void) 184 108 { 185 109 struct pt_regs regs; ··· 142 218 */ 143 219 if (IS_ENABLED(CONFIG_PPC_BOOK3S) && (local_paca->irq_happened & PACA_IRQ_HMI)) { 144 220 local_paca->irq_happened &= ~PACA_IRQ_HMI; 145 - regs.trap = 0xe60; 221 + regs.trap = INTERRUPT_HMI; 146 222 handle_hmi_exception(&regs); 147 223 if (!(local_paca->irq_happened & PACA_IRQ_HARD_DIS)) 148 224 hard_irq_disable(); ··· 150 226 151 227 if (local_paca->irq_happened & PACA_IRQ_DEC) { 152 228 local_paca->irq_happened &= ~PACA_IRQ_DEC; 153 - regs.trap = 0x900; 229 + regs.trap = INTERRUPT_DECREMENTER; 154 230 timer_interrupt(&regs); 155 231 if (!(local_paca->irq_happened & PACA_IRQ_HARD_DIS)) 156 232 hard_irq_disable(); ··· 158 234 159 235 if (local_paca->irq_happened & PACA_IRQ_EE) { 160 236 local_paca->irq_happened &= ~PACA_IRQ_EE; 161 - regs.trap = 0x500; 237 + regs.trap = INTERRUPT_EXTERNAL; 162 238 do_IRQ(&regs); 163 239 if (!(local_paca->irq_happened & PACA_IRQ_HARD_DIS)) 164 240 hard_irq_disable(); ··· 166 242 167 243 if (IS_ENABLED(CONFIG_PPC_DOORBELL) && (local_paca->irq_happened & PACA_IRQ_DBELL)) { 168 244 local_paca->irq_happened &= ~PACA_IRQ_DBELL; 169 - if (IS_ENABLED(CONFIG_PPC_BOOK3E)) 170 - regs.trap = 0x280; 171 - else 172 - regs.trap = 0xa00; 245 + regs.trap = INTERRUPT_DOORBELL; 173 246 doorbell_exception(&regs); 174 247 if (!(local_paca->irq_happened & PACA_IRQ_HARD_DIS)) 175 248 hard_irq_disable(); ··· 175 254 /* Book3E does not support soft-masking PMI interrupts */ 176 255 if (IS_ENABLED(CONFIG_PPC_BOOK3S) && (local_paca->irq_happened & PACA_IRQ_PMI)) { 177 256 local_paca->irq_happened &= ~PACA_IRQ_PMI; 178 - regs.trap = 0xf00; 257 + regs.trap = INTERRUPT_PERFMON; 179 258 performance_monitor_exception(&regs); 180 259 if (!(local_paca->irq_happened & PACA_IRQ_HARD_DIS)) 181 260 hard_irq_disable(); ··· 203 282 * and re-locking AMR but we shouldn't get here in the first place, 204 283 * hence the warning. 205 284 */ 206 - kuap_check_amr(); 285 + kuap_assert_locked(); 207 286 208 287 if (kuap_state != AMR_KUAP_BLOCKED) 209 288 set_kuap(AMR_KUAP_BLOCKED); ··· 586 665 pr_err("do_IRQ: stack overflow: %ld\n", sp); 587 666 dump_stack(); 588 667 } 668 + } 669 + 670 + static __always_inline void call_do_softirq(const void *sp) 671 + { 672 + /* Temporarily switch r1 to sp, call __do_softirq() then restore r1. */ 673 + asm volatile ( 674 + PPC_STLU " %%r1, %[offset](%[sp]) ;" 675 + "mr %%r1, %[sp] ;" 676 + "bl %[callee] ;" 677 + PPC_LL " %%r1, 0(%%r1) ;" 678 + : // Outputs 679 + : // Inputs 680 + [sp] "b" (sp), [offset] "i" (THREAD_SIZE - STACK_FRAME_OVERHEAD), 681 + [callee] "i" (__do_softirq) 682 + : // Clobbers 683 + "lr", "xer", "ctr", "memory", "cr0", "cr1", "cr5", "cr6", 684 + "cr7", "r0", "r3", "r4", "r5", "r6", "r7", "r8", "r9", "r10", 685 + "r11", "r12" 686 + ); 687 + } 688 + 689 + static __always_inline void call_do_irq(struct pt_regs *regs, void *sp) 690 + { 691 + register unsigned long r3 asm("r3") = (unsigned long)regs; 692 + 693 + /* Temporarily switch r1 to sp, call __do_irq() then restore r1. */ 694 + asm volatile ( 695 + PPC_STLU " %%r1, %[offset](%[sp]) ;" 696 + "mr %%r1, %[sp] ;" 697 + "bl %[callee] ;" 698 + PPC_LL " %%r1, 0(%%r1) ;" 699 + : // Outputs 700 + "+r" (r3) 701 + : // Inputs 702 + [sp] "b" (sp), [offset] "i" (THREAD_SIZE - STACK_FRAME_OVERHEAD), 703 + [callee] "i" (__do_irq) 704 + : // Clobbers 705 + "lr", "xer", "ctr", "memory", "cr0", "cr1", "cr5", "cr6", 706 + "cr7", "r0", "r4", "r5", "r6", "r7", "r8", "r9", "r10", 707 + "r11", "r12" 708 + ); 589 709 } 590 710 591 711 void __do_irq(struct pt_regs *regs)

+2 -2

arch/powerpc/kernel/jump_label.c

··· 11 11 void arch_jump_label_transform(struct jump_entry *entry, 12 12 enum jump_label_type type) 13 13 { 14 - struct ppc_inst *addr = (struct ppc_inst *)(unsigned long)entry->code; 14 + struct ppc_inst *addr = (struct ppc_inst *)jump_entry_code(entry); 15 15 16 16 if (type == JUMP_LABEL_JMP) 17 - patch_branch(addr, entry->target, 0); 17 + patch_branch(addr, jump_entry_target(entry), 0); 18 18 else 19 19 patch_instruction(addr, ppc_inst(PPC_INST_NOP)); 20 20 }

+1 -1

arch/powerpc/kernel/kgdb.c

··· 376 376 } 377 377 378 378 /* 379 - * This function does PowerPC specific procesing for interfacing to gdb. 379 + * This function does PowerPC specific processing for interfacing to gdb. 380 380 */ 381 381 int kgdb_arch_handle_exception(int vector, int signo, int err_code, 382 382 char *remcom_in_buffer, char *remcom_out_buffer,

+29 -4

arch/powerpc/kernel/legacy_serial.c

··· 15 15 #include <asm/udbg.h> 16 16 #include <asm/pci-bridge.h> 17 17 #include <asm/ppc-pci.h> 18 + #include <asm/early_ioremap.h> 18 19 19 20 #undef DEBUG 20 21 ··· 35 34 unsigned int clock; 36 35 int irq_check_parent; 37 36 phys_addr_t taddr; 37 + void __iomem *early_addr; 38 38 } legacy_serial_infos[MAX_LEGACY_SERIAL_PORTS]; 39 39 40 40 static const struct of_device_id legacy_serial_parents[] __initconst = { ··· 327 325 { 328 326 struct legacy_serial_info *info = &legacy_serial_infos[console]; 329 327 struct plat_serial8250_port *port = &legacy_serial_ports[console]; 330 - void __iomem *addr; 331 328 unsigned int stride; 332 329 333 330 stride = 1 << port->regshift; 334 331 335 332 /* Check if a translated MMIO address has been found */ 336 333 if (info->taddr) { 337 - addr = ioremap(info->taddr, 0x1000); 338 - if (addr == NULL) 334 + info->early_addr = early_ioremap(info->taddr, 0x1000); 335 + if (info->early_addr == NULL) 339 336 return; 340 - udbg_uart_init_mmio(addr, stride); 337 + udbg_uart_init_mmio(info->early_addr, stride); 341 338 } else { 342 339 /* Check if it's PIO and we support untranslated PIO */ 343 340 if (port->iotype == UPIO_PORT && isa_io_special) ··· 353 352 DBG("default console speed = %d\n", info->speed); 354 353 udbg_uart_setup(info->speed, info->clock); 355 354 } 355 + 356 + static int __init ioremap_legacy_serial_console(void) 357 + { 358 + struct legacy_serial_info *info = &legacy_serial_infos[legacy_serial_console]; 359 + struct plat_serial8250_port *port = &legacy_serial_ports[legacy_serial_console]; 360 + void __iomem *vaddr; 361 + 362 + if (legacy_serial_console < 0) 363 + return 0; 364 + 365 + if (!info->early_addr) 366 + return 0; 367 + 368 + vaddr = ioremap(info->taddr, 0x1000); 369 + if (WARN_ON(!vaddr)) 370 + return -ENOMEM; 371 + 372 + udbg_uart_init_mmio(vaddr, 1 << port->regshift); 373 + early_iounmap(info->early_addr, 0x1000); 374 + info->early_addr = NULL; 375 + 376 + return 0; 377 + } 378 + early_initcall(ioremap_legacy_serial_console); 356 379 357 380 /* 358 381 * This is called very early, as part of setup_system() or eventually

+3 -2

arch/powerpc/kernel/mce.c

··· 40 40 .func = machine_check_ue_irq_work, 41 41 }; 42 42 43 - DECLARE_WORK(mce_ue_event_work, machine_process_ue_event); 43 + static DECLARE_WORK(mce_ue_event_work, machine_process_ue_event); 44 44 45 45 static BLOCKING_NOTIFIER_HEAD(mce_notifier_list); 46 46 ··· 131 131 * Populate the mce error_type and type-specific error_type. 132 132 */ 133 133 mce_set_error_info(mce, mce_err); 134 + if (mce->error_type == MCE_ERROR_TYPE_UE) 135 + mce->u.ue_error.ignore_event = mce_err->ignore_event; 134 136 135 137 if (!addr) 136 138 return; ··· 161 159 if (phys_addr != ULONG_MAX) { 162 160 mce->u.ue_error.physical_address_provided = true; 163 161 mce->u.ue_error.physical_address = phys_addr; 164 - mce->u.ue_error.ignore_event = mce_err->ignore_event; 165 162 machine_check_ue_event(mce); 166 163 } 167 164 }

-39

arch/powerpc/kernel/misc_32.S

··· 28 28 .text 29 29 30 30 /* 31 - * We store the saved ksp_limit in the unused part 32 - * of the STACK_FRAME_OVERHEAD 33 - */ 34 - _GLOBAL(call_do_softirq) 35 - mflr r0 36 - stw r0,4(r1) 37 - lwz r10,THREAD+KSP_LIMIT(r2) 38 - stw r3, THREAD+KSP_LIMIT(r2) 39 - stwu r1,THREAD_SIZE-STACK_FRAME_OVERHEAD(r3) 40 - mr r1,r3 41 - stw r10,8(r1) 42 - bl __do_softirq 43 - lwz r10,8(r1) 44 - lwz r1,0(r1) 45 - lwz r0,4(r1) 46 - stw r10,THREAD+KSP_LIMIT(r2) 47 - mtlr r0 48 - blr 49 - 50 - /* 51 - * void call_do_irq(struct pt_regs *regs, void *sp); 52 - */ 53 - _GLOBAL(call_do_irq) 54 - mflr r0 55 - stw r0,4(r1) 56 - lwz r10,THREAD+KSP_LIMIT(r2) 57 - stw r4, THREAD+KSP_LIMIT(r2) 58 - stwu r1,THREAD_SIZE-STACK_FRAME_OVERHEAD(r4) 59 - mr r1,r4 60 - stw r10,8(r1) 61 - bl __do_irq 62 - lwz r10,8(r1) 63 - lwz r1,0(r1) 64 - lwz r0,4(r1) 65 - stw r10,THREAD+KSP_LIMIT(r2) 66 - mtlr r0 67 - blr 68 - 69 - /* 70 31 * This returns the high 64 bits of the product of two 64-bit numbers. 71 32 */ 72 33 _GLOBAL(mulhdu)

-22

arch/powerpc/kernel/misc_64.S

··· 27 27 28 28 .text 29 29 30 - _GLOBAL(call_do_softirq) 31 - mflr r0 32 - std r0,16(r1) 33 - stdu r1,THREAD_SIZE-STACK_FRAME_OVERHEAD(r3) 34 - mr r1,r3 35 - bl __do_softirq 36 - ld r1,0(r1) 37 - ld r0,16(r1) 38 - mtlr r0 39 - blr 40 - 41 - _GLOBAL(call_do_irq) 42 - mflr r0 43 - std r0,16(r1) 44 - stdu r1,THREAD_SIZE-STACK_FRAME_OVERHEAD(r4) 45 - mr r1,r4 46 - bl __do_irq 47 - ld r1,0(r1) 48 - ld r0,16(r1) 49 - mtlr r0 50 - blr 51 - 52 30 _GLOBAL(__bswapdi2) 53 31 EXPORT_SYMBOL(__bswapdi2) 54 32 srdi r8,r3,32

+21 -4

arch/powerpc/kernel/module.c

··· 14 14 #include <asm/firmware.h> 15 15 #include <linux/sort.h> 16 16 #include <asm/setup.h> 17 + #include <asm/sections.h> 17 18 18 19 static LIST_HEAD(module_bug_list); 19 20 ··· 89 88 } 90 89 91 90 #ifdef MODULES_VADDR 92 - void *module_alloc(unsigned long size) 91 + static __always_inline void * 92 + __module_alloc(unsigned long size, unsigned long start, unsigned long end) 93 93 { 94 - BUILD_BUG_ON(TASK_SIZE > MODULES_VADDR); 95 - 96 - return __vmalloc_node_range(size, 1, MODULES_VADDR, MODULES_END, GFP_KERNEL, 94 + return __vmalloc_node_range(size, 1, start, end, GFP_KERNEL, 97 95 PAGE_KERNEL_EXEC, VM_FLUSH_RESET_PERMS, NUMA_NO_NODE, 98 96 __builtin_return_address(0)); 97 + } 98 + 99 + void *module_alloc(unsigned long size) 100 + { 101 + unsigned long limit = (unsigned long)_etext - SZ_32M; 102 + void *ptr = NULL; 103 + 104 + BUILD_BUG_ON(TASK_SIZE > MODULES_VADDR); 105 + 106 + /* First try within 32M limit from _etext to avoid branch trampolines */ 107 + if (MODULES_VADDR < PAGE_OFFSET && MODULES_END > limit) 108 + ptr = __module_alloc(size, limit, MODULES_END); 109 + 110 + if (!ptr) 111 + ptr = __module_alloc(size, MODULES_VADDR, MODULES_END); 112 + 113 + return ptr; 99 114 } 100 115 #endif

+21 -3

arch/powerpc/kernel/optprobes.c

··· 141 141 } 142 142 } 143 143 144 + static void patch_imm32_load_insns(unsigned long val, int reg, kprobe_opcode_t *addr) 145 + { 146 + patch_instruction((struct ppc_inst *)addr, 147 + ppc_inst(PPC_RAW_LIS(reg, IMM_H(val)))); 148 + addr++; 149 + 150 + patch_instruction((struct ppc_inst *)addr, 151 + ppc_inst(PPC_RAW_ORI(reg, reg, IMM_L(val)))); 152 + } 153 + 144 154 /* 145 155 * Generate instructions to load provided immediate 64-bit value 146 156 * to register 'reg' and patch these instructions at 'addr'. 147 157 */ 148 - static void patch_imm64_load_insns(unsigned long val, int reg, kprobe_opcode_t *addr) 158 + static void patch_imm64_load_insns(unsigned long long val, int reg, kprobe_opcode_t *addr) 149 159 { 150 160 /* lis reg,(op)@highest */ 151 161 patch_instruction((struct ppc_inst *)addr, ··· 185 175 patch_instruction((struct ppc_inst *)addr, 186 176 ppc_inst(PPC_INST_ORI | ___PPC_RA(reg) | 187 177 ___PPC_RS(reg) | (val & 0xffff))); 178 + } 179 + 180 + static void patch_imm_load_insns(unsigned long val, int reg, kprobe_opcode_t *addr) 181 + { 182 + if (IS_ENABLED(CONFIG_PPC64)) 183 + patch_imm64_load_insns(val, reg, addr); 184 + else 185 + patch_imm32_load_insns(val, reg, addr); 188 186 } 189 187 190 188 int arch_prepare_optimized_kprobe(struct optimized_kprobe *op, struct kprobe *p) ··· 248 230 * Fixup the template with instructions to: 249 231 * 1. load the address of the actual probepoint 250 232 */ 251 - patch_imm64_load_insns((unsigned long)op, 3, buff + TMPL_OP_IDX); 233 + patch_imm_load_insns((unsigned long)op, 3, buff + TMPL_OP_IDX); 252 234 253 235 /* 254 236 * 2. branch to optimized_callback() and emulate_step() ··· 282 264 * 3. load instruction to be emulated into relevant register, and 283 265 */ 284 266 temp = ppc_inst_read((struct ppc_inst *)p->ainsn.insn); 285 - patch_imm64_load_insns(ppc_inst_as_u64(temp), 4, buff + TMPL_INSN_IDX); 267 + patch_imm_load_insns(ppc_inst_as_ulong(temp), 4, buff + TMPL_INSN_IDX); 286 268 287 269 /* 288 270 * 4. branch back from trampoline

+34 -31

arch/powerpc/kernel/optprobes_head.S

··· 9 9 #include <asm/ptrace.h> 10 10 #include <asm/asm-offsets.h> 11 11 12 + #ifdef CONFIG_PPC64 13 + #define SAVE_30GPRS(base) SAVE_10GPRS(2,base); SAVE_10GPRS(12,base); SAVE_10GPRS(22,base) 14 + #define REST_30GPRS(base) REST_10GPRS(2,base); REST_10GPRS(12,base); REST_10GPRS(22,base) 15 + #define TEMPLATE_FOR_IMM_LOAD_INSNS nop; nop; nop; nop; nop 16 + #else 17 + #define SAVE_30GPRS(base) stmw r2, GPR2(base) 18 + #define REST_30GPRS(base) lmw r2, GPR2(base) 19 + #define TEMPLATE_FOR_IMM_LOAD_INSNS nop; nop; nop 20 + #endif 21 + 12 22 #define OPT_SLOT_SIZE 65536 13 23 14 24 .balign 4 ··· 40 30 .global optprobe_template_entry 41 31 optprobe_template_entry: 42 32 /* Create an in-memory pt_regs */ 43 - stdu r1,-INT_FRAME_SIZE(r1) 33 + PPC_STLU r1,-INT_FRAME_SIZE(r1) 44 34 SAVE_GPR(0,r1) 45 35 /* Save the previous SP into stack */ 46 36 addi r0,r1,INT_FRAME_SIZE 47 - std r0,GPR1(r1) 48 - SAVE_10GPRS(2,r1) 49 - SAVE_10GPRS(12,r1) 50 - SAVE_10GPRS(22,r1) 37 + PPC_STL r0,GPR1(r1) 38 + SAVE_30GPRS(r1) 51 39 /* Save SPRS */ 52 40 mfmsr r5 53 - std r5,_MSR(r1) 41 + PPC_STL r5,_MSR(r1) 54 42 li r5,0x700 55 - std r5,_TRAP(r1) 43 + PPC_STL r5,_TRAP(r1) 56 44 li r5,0 57 - std r5,ORIG_GPR3(r1) 58 - std r5,RESULT(r1) 45 + PPC_STL r5,ORIG_GPR3(r1) 46 + PPC_STL r5,RESULT(r1) 59 47 mfctr r5 60 - std r5,_CTR(r1) 48 + PPC_STL r5,_CTR(r1) 61 49 mflr r5 62 - std r5,_LINK(r1) 50 + PPC_STL r5,_LINK(r1) 63 51 mfspr r5,SPRN_XER 64 - std r5,_XER(r1) 52 + PPC_STL r5,_XER(r1) 65 53 mfcr r5 66 - std r5,_CCR(r1) 54 + PPC_STL r5,_CCR(r1) 55 + #ifdef CONFIG_PPC64 67 56 lbz r5,PACAIRQSOFTMASK(r13) 68 57 std r5,SOFTE(r1) 58 + #endif 69 59 70 60 /* 71 61 * We may get here from a module, so load the kernel TOC in r2. 72 62 * The original TOC gets restored when pt_regs is restored 73 63 * further below. 74 64 */ 65 + #ifdef CONFIG_PPC64 75 66 ld r2,PACATOC(r13) 67 + #endif 76 68 77 69 .global optprobe_template_op_address 78 70 optprobe_template_op_address: ··· 82 70 * Parameters to optimized_callback(): 83 71 * 1. optimized_kprobe structure in r3 84 72 */ 85 - nop 86 - nop 87 - nop 88 - nop 89 - nop 73 + TEMPLATE_FOR_IMM_LOAD_INSNS 74 + 90 75 /* 2. pt_regs pointer in r4 */ 91 76 addi r4,r1,STACK_FRAME_OVERHEAD 92 77 ··· 101 92 .global optprobe_template_insn 102 93 optprobe_template_insn: 103 94 /* 2, Pass instruction to be emulated in r4 */ 104 - nop 105 - nop 106 - nop 107 - nop 108 - nop 95 + TEMPLATE_FOR_IMM_LOAD_INSNS 109 96 110 97 .global optprobe_template_call_emulate 111 98 optprobe_template_call_emulate: ··· 112 107 * All done. 113 108 * Now, restore the registers... 114 109 */ 115 - ld r5,_MSR(r1) 110 + PPC_LL r5,_MSR(r1) 116 111 mtmsr r5 117 - ld r5,_CTR(r1) 112 + PPC_LL r5,_CTR(r1) 118 113 mtctr r5 119 - ld r5,_LINK(r1) 114 + PPC_LL r5,_LINK(r1) 120 115 mtlr r5 121 - ld r5,_XER(r1) 116 + PPC_LL r5,_XER(r1) 122 117 mtxer r5 123 - ld r5,_CCR(r1) 118 + PPC_LL r5,_CCR(r1) 124 119 mtcr r5 125 120 REST_GPR(0,r1) 126 - REST_10GPRS(2,r1) 127 - REST_10GPRS(12,r1) 128 - REST_10GPRS(22,r1) 121 + REST_30GPRS(r1) 129 122 /* Restore the previous SP */ 130 123 addi r1,r1,INT_FRAME_SIZE 131 124

+8 -17

arch/powerpc/kernel/process.c

··· 1117 1117 regs->msr |= msr_diff; 1118 1118 } 1119 1119 1120 - #else 1120 + #else /* !CONFIG_PPC_TRANSACTIONAL_MEM */ 1121 1121 #define tm_recheckpoint_new_task(new) 1122 1122 #define __switch_to_tm(prev, new) 1123 + void tm_reclaim_current(uint8_t cause) {} 1123 1124 #endif /* CONFIG_PPC_TRANSACTIONAL_MEM */ 1124 1125 1125 1126 static inline void save_sprs(struct thread_struct *t) ··· 1256 1255 */ 1257 1256 restore_sprs(old_thread, new_thread); 1258 1257 1258 + #ifdef CONFIG_PPC32 1259 + kuap_assert_locked(); 1260 + #endif 1259 1261 last = _switch(old_thread, new_thread); 1260 1262 1261 1263 #ifdef CONFIG_PPC_BOOK3S_64 ··· 1448 1444 #ifdef CONFIG_PPC64 1449 1445 #define REG "%016lx" 1450 1446 #define REGS_PER_LINE 4 1451 - #define LAST_VOLATILE 13 1452 1447 #else 1453 1448 #define REG "%08lx" 1454 1449 #define REGS_PER_LINE 8 1455 - #define LAST_VOLATILE 12 1456 1450 #endif 1457 1451 1458 1452 static void __show_regs(struct pt_regs *regs) ··· 1467 1465 trap = TRAP(regs); 1468 1466 if (!trap_is_syscall(regs) && cpu_has_feature(CPU_FTR_CFAR)) 1469 1467 pr_cont("CFAR: "REG" ", regs->orig_gpr3); 1470 - if (trap == 0x200 || trap == 0x300 || trap == 0x600) { 1468 + if (trap == INTERRUPT_MACHINE_CHECK || 1469 + trap == INTERRUPT_DATA_STORAGE || 1470 + trap == INTERRUPT_ALIGNMENT) { 1471 1471 if (IS_ENABLED(CONFIG_4xx) || IS_ENABLED(CONFIG_BOOKE)) 1472 1472 pr_cont("DEAR: "REG" ESR: "REG" ", regs->dar, regs->dsisr); 1473 1473 else ··· 1488 1484 if ((i % REGS_PER_LINE) == 0) 1489 1485 pr_cont("\nGPR%02d: ", i); 1490 1486 pr_cont(REG " ", regs->gpr[i]); 1491 - if (i == LAST_VOLATILE && !FULL_REGS(regs)) 1492 - break; 1493 1487 } 1494 1488 pr_cont("\n"); 1495 1489 /* ··· 1690 1688 } else { 1691 1689 /* user thread */ 1692 1690 struct pt_regs *regs = current_pt_regs(); 1693 - CHECK_FULL_REGS(regs); 1694 1691 *childregs = *regs; 1695 1692 if (usp) 1696 1693 childregs->gpr[1] = usp; ··· 1725 1724 kregs = (struct pt_regs *) sp; 1726 1725 sp -= STACK_FRAME_OVERHEAD; 1727 1726 p->thread.ksp = sp; 1728 - #ifdef CONFIG_PPC32 1729 - p->thread.ksp_limit = (unsigned long)end_of_stack(p); 1730 - #endif 1731 1727 #ifdef CONFIG_HAVE_HW_BREAKPOINT 1732 1728 for (i = 0; i < nr_wp_slots(); i++) 1733 1729 p->thread.ptrace_bps[i] = NULL; ··· 1793 1795 regs->xer = 0; 1794 1796 regs->ccr = 0; 1795 1797 regs->gpr[1] = sp; 1796 - 1797 - /* 1798 - * We have just cleared all the nonvolatile GPRs, so make 1799 - * FULL_REGS(regs) return true. This is necessary to allow 1800 - * ptrace to examine the thread immediately after exec. 1801 - */ 1802 - SET_FULL_REGS(regs); 1803 1798 1804 1799 #ifdef CONFIG_PPC32 1805 1800 regs->mq = 0;

+16 -5

arch/powerpc/kernel/prom.c

··· 65 65 #define DBG(fmt...) 66 66 #endif 67 67 68 + int *chip_id_lookup_table; 69 + 68 70 #ifdef CONFIG_PPC64 69 71 int __initdata iommu_is_off; 70 72 int __initdata iommu_force_on; ··· 269 267 }; 270 268 271 269 #if defined(CONFIG_44x) && defined(CONFIG_PPC_FPU) 272 - static inline void identical_pvr_fixup(unsigned long node) 270 + static __init void identical_pvr_fixup(unsigned long node) 273 271 { 274 272 unsigned int pvr; 275 273 const char *model = of_get_flat_dt_prop(node, "model", NULL); ··· 916 914 int cpu_to_chip_id(int cpu) 917 915 { 918 916 struct device_node *np; 917 + int ret = -1, idx; 918 + 919 + idx = cpu / threads_per_core; 920 + if (chip_id_lookup_table && chip_id_lookup_table[idx] != -1) 921 + return chip_id_lookup_table[idx]; 919 922 920 923 np = of_get_cpu_node(cpu, NULL); 921 - if (!np) 922 - return -1; 924 + if (np) { 925 + ret = of_get_ibm_chip_id(np); 926 + of_node_put(np); 923 927 924 - of_node_put(np); 925 - return of_get_ibm_chip_id(np); 928 + if (chip_id_lookup_table) 929 + chip_id_lookup_table[idx] = ret; 930 + } 931 + 932 + return ret; 926 933 } 927 934 EXPORT_SYMBOL(cpu_to_chip_id); 928 935

+1 -1

arch/powerpc/kernel/prom_init.c

··· 2983 2983 " 0x3 encode-int encode+" 2984 2984 " s\" interrupts\" property" 2985 2985 " finish-device"); 2986 - }; 2986 + } 2987 2987 2988 2988 /* Check for a PHY device node - if missing then create one and 2989 2989 * give it's phandle to the ethernet node */

+18 -33

arch/powerpc/kernel/ptrace/ptrace-view.c

··· 111 111 return task->thread.regs->msr | task->thread.fpexc_mode; 112 112 } 113 113 114 - static int set_user_msr(struct task_struct *task, unsigned long msr) 114 + static __always_inline int set_user_msr(struct task_struct *task, unsigned long msr) 115 115 { 116 116 task->thread.regs->msr &= ~MSR_DEBUGCHANGE; 117 117 task->thread.regs->msr |= msr & MSR_DEBUGCHANGE; ··· 147 147 * We prevent mucking around with the reserved area of trap 148 148 * which are used internally by the kernel. 149 149 */ 150 - static int set_user_trap(struct task_struct *task, unsigned long trap) 150 + static __always_inline int set_user_trap(struct task_struct *task, unsigned long trap) 151 151 { 152 152 set_trap(task->thread.regs, trap); 153 153 return 0; ··· 221 221 #ifdef CONFIG_PPC64 222 222 struct membuf to_softe = membuf_at(&to, offsetof(struct pt_regs, softe)); 223 223 #endif 224 - int i; 225 - 226 224 if (target->thread.regs == NULL) 227 225 return -EIO; 228 - 229 - if (!FULL_REGS(target->thread.regs)) { 230 - /* We have a partial register set. Fill 14-31 with bogus values */ 231 - for (i = 14; i < 32; i++) 232 - target->thread.regs->gpr[i] = NV_REG_POISON; 233 - } 234 226 235 227 membuf_write(&to, target->thread.regs, sizeof(struct user_pt_regs)); 236 228 ··· 243 251 244 252 if (target->thread.regs == NULL) 245 253 return -EIO; 246 - 247 - CHECK_FULL_REGS(target->thread.regs); 248 254 249 255 ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, 250 256 target->thread.regs, ··· 649 659 const compat_ulong_t __user *u = ubuf; 650 660 compat_ulong_t reg; 651 661 662 + if (!kbuf && !user_read_access_begin(u, count)) 663 + return -EFAULT; 664 + 652 665 pos /= sizeof(reg); 653 666 count /= sizeof(reg); 654 667 ··· 660 667 regs[pos++] = *k++; 661 668 else 662 669 for (; count > 0 && pos < PT_MSR; --count) { 663 - if (__get_user(reg, u++)) 664 - return -EFAULT; 670 + unsafe_get_user(reg, u++, Efault); 665 671 regs[pos++] = reg; 666 672 } 667 673 ··· 668 676 if (count > 0 && pos == PT_MSR) { 669 677 if (kbuf) 670 678 reg = *k++; 671 - else if (__get_user(reg, u++)) 672 - return -EFAULT; 679 + else 680 + unsafe_get_user(reg, u++, Efault); 673 681 set_user_msr(target, reg); 674 682 ++pos; 675 683 --count; ··· 682 690 ++k; 683 691 } else { 684 692 for (; count > 0 && pos <= PT_MAX_PUT_REG; --count) { 685 - if (__get_user(reg, u++)) 686 - return -EFAULT; 693 + unsafe_get_user(reg, u++, Efault); 687 694 regs[pos++] = reg; 688 695 } 689 696 for (; count > 0 && pos < PT_TRAP; --count, ++pos) 690 - if (__get_user(reg, u++)) 691 - return -EFAULT; 697 + unsafe_get_user(reg, u++, Efault); 692 698 } 693 699 694 700 if (count > 0 && pos == PT_TRAP) { 695 701 if (kbuf) 696 702 reg = *k++; 697 - else if (__get_user(reg, u++)) 698 - return -EFAULT; 703 + else 704 + unsafe_get_user(reg, u++, Efault); 699 705 set_user_trap(target, reg); 700 706 ++pos; 701 707 --count; 702 708 } 709 + if (!kbuf) 710 + user_read_access_end(); 703 711 704 712 kbuf = k; 705 713 ubuf = u; ··· 707 715 count *= sizeof(reg); 708 716 return user_regset_copyin_ignore(&pos, &count, &kbuf, &ubuf, 709 717 (PT_TRAP + 1) * sizeof(reg), -1); 718 + 719 + Efault: 720 + user_read_access_end(); 721 + return -EFAULT; 710 722 } 711 723 712 724 static int gpr32_get(struct task_struct *target, 713 725 const struct user_regset *regset, 714 726 struct membuf to) 715 727 { 716 - int i; 717 - 718 728 if (target->thread.regs == NULL) 719 729 return -EIO; 720 730 721 - if (!FULL_REGS(target->thread.regs)) { 722 - /* 723 - * We have a partial register set. 724 - * Fill 14-31 with bogus values. 725 - */ 726 - for (i = 14; i < 32; i++) 727 - target->thread.regs->gpr[i] = NV_REG_POISON; 728 - } 729 731 return gpr32_get_common(target, regset, to, 730 732 &target->thread.regs->gpr[0]); 731 733 } ··· 732 746 if (target->thread.regs == NULL) 733 747 return -EIO; 734 748 735 - CHECK_FULL_REGS(target->thread.regs); 736 749 return gpr32_set_common(target, regset, pos, count, kbuf, ubuf, 737 750 &target->thread.regs->gpr[0]); 738 751 }

-4

arch/powerpc/kernel/ptrace/ptrace.c

··· 59 59 if ((addr & (sizeof(long) - 1)) || !child->thread.regs) 60 60 break; 61 61 62 - CHECK_FULL_REGS(child->thread.regs); 63 62 if (index < PT_FPR0) 64 63 ret = ptrace_get_reg(child, (int) index, &tmp); 65 64 else ··· 80 81 if ((addr & (sizeof(long) - 1)) || !child->thread.regs) 81 82 break; 82 83 83 - CHECK_FULL_REGS(child->thread.regs); 84 84 if (index < PT_FPR0) 85 85 ret = ptrace_put_reg(child, index, data); 86 86 else ··· 350 352 offsetof(struct user_pt_regs, gpr)); 351 353 BUILD_BUG_ON(offsetof(struct pt_regs, nip) != 352 354 offsetof(struct user_pt_regs, nip)); 353 - BUILD_BUG_ON(offsetof(struct pt_regs, msr) != 354 - offsetof(struct user_pt_regs, msr)); 355 355 BUILD_BUG_ON(offsetof(struct pt_regs, msr) != 356 356 offsetof(struct user_pt_regs, msr)); 357 357 BUILD_BUG_ON(offsetof(struct pt_regs, orig_gpr3) !=

-4

arch/powerpc/kernel/ptrace/ptrace32.c

··· 83 83 if ((addr & 3) || (index > PT_FPSCR32)) 84 84 break; 85 85 86 - CHECK_FULL_REGS(child->thread.regs); 87 86 if (index < PT_FPR0) { 88 87 ret = ptrace_get_reg(child, index, &tmp); 89 88 if (ret) ··· 132 133 if ((addr & 3) || numReg > PT_FPSCR) 133 134 break; 134 135 135 - CHECK_FULL_REGS(child->thread.regs); 136 136 if (numReg >= PT_FPR0) { 137 137 flush_fp_to_thread(child); 138 138 /* get 64 bit FPR */ ··· 185 187 if ((addr & 3) || (index > PT_FPSCR32)) 186 188 break; 187 189 188 - CHECK_FULL_REGS(child->thread.regs); 189 190 if (index < PT_FPR0) { 190 191 ret = ptrace_put_reg(child, index, data); 191 192 } else { ··· 223 226 */ 224 227 if ((addr & 3) || (numReg > PT_FPSCR)) 225 228 break; 226 - CHECK_FULL_REGS(child->thread.regs); 227 229 if (numReg < PT_FPR0) { 228 230 unsigned long freg; 229 231 ret = ptrace_get_reg(child, numReg, &freg);

+11 -4

arch/powerpc/kernel/rtas-proc.c

··· 755 755 return 0; 756 756 } 757 757 758 - #define RMO_READ_BUF_MAX 30 759 - 760 - /* RTAS Userspace access */ 758 + /** 759 + * ppc_rtas_rmo_buf_show() - Describe RTAS-addressable region for user space. 760 + * 761 + * Base + size description of a range of RTAS-addressable memory set 762 + * aside for user space to use as work area(s) for certain RTAS 763 + * functions. User space accesses this region via /dev/mem. Apart from 764 + * security policies, the kernel does not arbitrate or serialize 765 + * access to this region, and user space must ensure that concurrent 766 + * users do not interfere with each other. 767 + */ 761 768 static int ppc_rtas_rmo_buf_show(struct seq_file *m, void *v) 762 769 { 763 - seq_printf(m, "%016lx %x\n", rtas_rmo_buf, RTAS_RMOBUF_MAX); 770 + seq_printf(m, "%016lx %x\n", rtas_rmo_buf, RTAS_USER_REGION_SIZE); 764 771 return 0; 765 772 }

+18 -16

arch/powerpc/kernel/rtas.c

··· 828 828 pr_err("ibm,activate-firmware failed (%i)\n", fwrc); 829 829 } 830 830 831 - static int ibm_suspend_me_token = RTAS_UNKNOWN_SERVICE; 832 831 #ifdef CONFIG_PPC_PSERIES 833 832 /** 834 833 * rtas_call_reentrant() - Used for reentrant rtas calls ··· 987 988 static bool in_rmo_buf(u32 base, u32 end) 988 989 { 989 990 return base >= rtas_rmo_buf && 990 - base < (rtas_rmo_buf + RTAS_RMOBUF_MAX) && 991 + base < (rtas_rmo_buf + RTAS_USER_REGION_SIZE) && 991 992 base <= end && 992 993 end >= rtas_rmo_buf && 993 - end < (rtas_rmo_buf + RTAS_RMOBUF_MAX); 994 + end < (rtas_rmo_buf + RTAS_USER_REGION_SIZE); 994 995 } 995 996 996 997 static bool block_rtas_call(int token, int nargs, ··· 1051 1052 return true; 1052 1053 } 1053 1054 1055 + static void __init rtas_syscall_filter_init(void) 1056 + { 1057 + unsigned int i; 1058 + 1059 + for (i = 0; i < ARRAY_SIZE(rtas_filters); i++) 1060 + rtas_filters[i].token = rtas_token(rtas_filters[i].name); 1061 + } 1062 + 1054 1063 #else 1055 1064 1056 1065 static bool block_rtas_call(int token, int nargs, 1057 1066 struct rtas_args *args) 1058 1067 { 1059 1068 return false; 1069 + } 1070 + 1071 + static void __init rtas_syscall_filter_init(void) 1072 + { 1060 1073 } 1061 1074 1062 1075 #endif /* CONFIG_PPC_RTAS_FILTER */ ··· 1114 1103 return -EINVAL; 1115 1104 1116 1105 /* Need to handle ibm,suspend_me call specially */ 1117 - if (token == ibm_suspend_me_token) { 1106 + if (token == rtas_token("ibm,suspend-me")) { 1118 1107 1119 1108 /* 1120 1109 * rtas_ibm_suspend_me assumes the streamid handle is in cpu ··· 1174 1163 unsigned long rtas_region = RTAS_INSTANTIATE_MAX; 1175 1164 u32 base, size, entry; 1176 1165 int no_base, no_size, no_entry; 1177 - #ifdef CONFIG_PPC_RTAS_FILTER 1178 - int i; 1179 - #endif 1180 1166 1181 1167 /* Get RTAS dev node and fill up our "rtas" structure with infos 1182 1168 * about it. ··· 1199 1191 * the stop-self token if any 1200 1192 */ 1201 1193 #ifdef CONFIG_PPC64 1202 - if (firmware_has_feature(FW_FEATURE_LPAR)) { 1194 + if (firmware_has_feature(FW_FEATURE_LPAR)) 1203 1195 rtas_region = min(ppc64_rma_size, RTAS_INSTANTIATE_MAX); 1204 - ibm_suspend_me_token = rtas_token("ibm,suspend-me"); 1205 - } 1206 1196 #endif 1207 - rtas_rmo_buf = memblock_phys_alloc_range(RTAS_RMOBUF_MAX, PAGE_SIZE, 1197 + rtas_rmo_buf = memblock_phys_alloc_range(RTAS_USER_REGION_SIZE, PAGE_SIZE, 1208 1198 0, rtas_region); 1209 1199 if (!rtas_rmo_buf) 1210 1200 panic("ERROR: RTAS: Failed to allocate %lx bytes below %pa\n", ··· 1212 1206 rtas_last_error_token = rtas_token("rtas-last-error"); 1213 1207 #endif 1214 1208 1215 - #ifdef CONFIG_PPC_RTAS_FILTER 1216 - for (i = 0; i < ARRAY_SIZE(rtas_filters); i++) { 1217 - rtas_filters[i].token = rtas_token(rtas_filters[i].name); 1218 - } 1219 - #endif 1209 + rtas_syscall_filter_init(); 1220 1210 } 1221 1211 1222 1212 int __init early_init_dt_scan_rtas(unsigned long node,

+262 -1

arch/powerpc/kernel/security.c

··· 7 7 #include <linux/cpu.h> 8 8 #include <linux/kernel.h> 9 9 #include <linux/device.h> 10 + #include <linux/memblock.h> 10 11 #include <linux/nospec.h> 11 12 #include <linux/prctl.h> 12 13 #include <linux/seq_buf.h> ··· 19 18 #include <asm/setup.h> 20 19 #include <asm/inst.h> 21 20 21 + #include "setup.h" 22 22 23 23 u64 powerpc_security_features __read_mostly = SEC_FTR_DEFAULT; 24 24 ··· 252 250 253 251 static enum stf_barrier_type stf_enabled_flush_types; 254 252 static bool no_stf_barrier; 255 - bool stf_barrier; 253 + static bool stf_barrier; 256 254 257 255 static int __init handle_no_stf_barrier(char *p) 258 256 { ··· 543 541 toggle_branch_cache_flush(enable); 544 542 } 545 543 544 + static enum l1d_flush_type enabled_flush_types; 545 + static void *l1d_flush_fallback_area; 546 + static bool no_rfi_flush; 547 + static bool no_entry_flush; 548 + static bool no_uaccess_flush; 549 + bool rfi_flush; 550 + static bool entry_flush; 551 + static bool uaccess_flush; 552 + DEFINE_STATIC_KEY_FALSE(uaccess_flush_key); 553 + EXPORT_SYMBOL(uaccess_flush_key); 554 + 555 + static int __init handle_no_rfi_flush(char *p) 556 + { 557 + pr_info("rfi-flush: disabled on command line."); 558 + no_rfi_flush = true; 559 + return 0; 560 + } 561 + early_param("no_rfi_flush", handle_no_rfi_flush); 562 + 563 + static int __init handle_no_entry_flush(char *p) 564 + { 565 + pr_info("entry-flush: disabled on command line."); 566 + no_entry_flush = true; 567 + return 0; 568 + } 569 + early_param("no_entry_flush", handle_no_entry_flush); 570 + 571 + static int __init handle_no_uaccess_flush(char *p) 572 + { 573 + pr_info("uaccess-flush: disabled on command line."); 574 + no_uaccess_flush = true; 575 + return 0; 576 + } 577 + early_param("no_uaccess_flush", handle_no_uaccess_flush); 578 + 579 + /* 580 + * The RFI flush is not KPTI, but because users will see doco that says to use 581 + * nopti we hijack that option here to also disable the RFI flush. 582 + */ 583 + static int __init handle_no_pti(char *p) 584 + { 585 + pr_info("rfi-flush: disabling due to 'nopti' on command line.\n"); 586 + handle_no_rfi_flush(NULL); 587 + return 0; 588 + } 589 + early_param("nopti", handle_no_pti); 590 + 591 + static void do_nothing(void *unused) 592 + { 593 + /* 594 + * We don't need to do the flush explicitly, just enter+exit kernel is 595 + * sufficient, the RFI exit handlers will do the right thing. 596 + */ 597 + } 598 + 599 + void rfi_flush_enable(bool enable) 600 + { 601 + if (enable) { 602 + do_rfi_flush_fixups(enabled_flush_types); 603 + on_each_cpu(do_nothing, NULL, 1); 604 + } else 605 + do_rfi_flush_fixups(L1D_FLUSH_NONE); 606 + 607 + rfi_flush = enable; 608 + } 609 + 610 + static void entry_flush_enable(bool enable) 611 + { 612 + if (enable) { 613 + do_entry_flush_fixups(enabled_flush_types); 614 + on_each_cpu(do_nothing, NULL, 1); 615 + } else { 616 + do_entry_flush_fixups(L1D_FLUSH_NONE); 617 + } 618 + 619 + entry_flush = enable; 620 + } 621 + 622 + static void uaccess_flush_enable(bool enable) 623 + { 624 + if (enable) { 625 + do_uaccess_flush_fixups(enabled_flush_types); 626 + static_branch_enable(&uaccess_flush_key); 627 + on_each_cpu(do_nothing, NULL, 1); 628 + } else { 629 + static_branch_disable(&uaccess_flush_key); 630 + do_uaccess_flush_fixups(L1D_FLUSH_NONE); 631 + } 632 + 633 + uaccess_flush = enable; 634 + } 635 + 636 + static void __ref init_fallback_flush(void) 637 + { 638 + u64 l1d_size, limit; 639 + int cpu; 640 + 641 + /* Only allocate the fallback flush area once (at boot time). */ 642 + if (l1d_flush_fallback_area) 643 + return; 644 + 645 + l1d_size = ppc64_caches.l1d.size; 646 + 647 + /* 648 + * If there is no d-cache-size property in the device tree, l1d_size 649 + * could be zero. That leads to the loop in the asm wrapping around to 650 + * 2^64-1, and then walking off the end of the fallback area and 651 + * eventually causing a page fault which is fatal. Just default to 652 + * something vaguely sane. 653 + */ 654 + if (!l1d_size) 655 + l1d_size = (64 * 1024); 656 + 657 + limit = min(ppc64_bolted_size(), ppc64_rma_size); 658 + 659 + /* 660 + * Align to L1d size, and size it at 2x L1d size, to catch possible 661 + * hardware prefetch runoff. We don't have a recipe for load patterns to 662 + * reliably avoid the prefetcher. 663 + */ 664 + l1d_flush_fallback_area = memblock_alloc_try_nid(l1d_size * 2, 665 + l1d_size, MEMBLOCK_LOW_LIMIT, 666 + limit, NUMA_NO_NODE); 667 + if (!l1d_flush_fallback_area) 668 + panic("%s: Failed to allocate %llu bytes align=0x%llx max_addr=%pa\n", 669 + __func__, l1d_size * 2, l1d_size, &limit); 670 + 671 + 672 + for_each_possible_cpu(cpu) { 673 + struct paca_struct *paca = paca_ptrs[cpu]; 674 + paca->rfi_flush_fallback_area = l1d_flush_fallback_area; 675 + paca->l1d_flush_size = l1d_size; 676 + } 677 + } 678 + 679 + void setup_rfi_flush(enum l1d_flush_type types, bool enable) 680 + { 681 + if (types & L1D_FLUSH_FALLBACK) { 682 + pr_info("rfi-flush: fallback displacement flush available\n"); 683 + init_fallback_flush(); 684 + } 685 + 686 + if (types & L1D_FLUSH_ORI) 687 + pr_info("rfi-flush: ori type flush available\n"); 688 + 689 + if (types & L1D_FLUSH_MTTRIG) 690 + pr_info("rfi-flush: mttrig type flush available\n"); 691 + 692 + enabled_flush_types = types; 693 + 694 + if (!cpu_mitigations_off() && !no_rfi_flush) 695 + rfi_flush_enable(enable); 696 + } 697 + 698 + void setup_entry_flush(bool enable) 699 + { 700 + if (cpu_mitigations_off()) 701 + return; 702 + 703 + if (!no_entry_flush) 704 + entry_flush_enable(enable); 705 + } 706 + 707 + void setup_uaccess_flush(bool enable) 708 + { 709 + if (cpu_mitigations_off()) 710 + return; 711 + 712 + if (!no_uaccess_flush) 713 + uaccess_flush_enable(enable); 714 + } 715 + 546 716 #ifdef CONFIG_DEBUG_FS 547 717 static int count_cache_flush_set(void *data, u64 val) 548 718 { ··· 753 579 return 0; 754 580 } 755 581 device_initcall(count_cache_flush_debugfs_init); 582 + 583 + static int rfi_flush_set(void *data, u64 val) 584 + { 585 + bool enable; 586 + 587 + if (val == 1) 588 + enable = true; 589 + else if (val == 0) 590 + enable = false; 591 + else 592 + return -EINVAL; 593 + 594 + /* Only do anything if we're changing state */ 595 + if (enable != rfi_flush) 596 + rfi_flush_enable(enable); 597 + 598 + return 0; 599 + } 600 + 601 + static int rfi_flush_get(void *data, u64 *val) 602 + { 603 + *val = rfi_flush ? 1 : 0; 604 + return 0; 605 + } 606 + 607 + DEFINE_SIMPLE_ATTRIBUTE(fops_rfi_flush, rfi_flush_get, rfi_flush_set, "%llu\n"); 608 + 609 + static int entry_flush_set(void *data, u64 val) 610 + { 611 + bool enable; 612 + 613 + if (val == 1) 614 + enable = true; 615 + else if (val == 0) 616 + enable = false; 617 + else 618 + return -EINVAL; 619 + 620 + /* Only do anything if we're changing state */ 621 + if (enable != entry_flush) 622 + entry_flush_enable(enable); 623 + 624 + return 0; 625 + } 626 + 627 + static int entry_flush_get(void *data, u64 *val) 628 + { 629 + *val = entry_flush ? 1 : 0; 630 + return 0; 631 + } 632 + 633 + DEFINE_SIMPLE_ATTRIBUTE(fops_entry_flush, entry_flush_get, entry_flush_set, "%llu\n"); 634 + 635 + static int uaccess_flush_set(void *data, u64 val) 636 + { 637 + bool enable; 638 + 639 + if (val == 1) 640 + enable = true; 641 + else if (val == 0) 642 + enable = false; 643 + else 644 + return -EINVAL; 645 + 646 + /* Only do anything if we're changing state */ 647 + if (enable != uaccess_flush) 648 + uaccess_flush_enable(enable); 649 + 650 + return 0; 651 + } 652 + 653 + static int uaccess_flush_get(void *data, u64 *val) 654 + { 655 + *val = uaccess_flush ? 1 : 0; 656 + return 0; 657 + } 658 + 659 + DEFINE_SIMPLE_ATTRIBUTE(fops_uaccess_flush, uaccess_flush_get, uaccess_flush_set, "%llu\n"); 660 + 661 + static __init int rfi_flush_debugfs_init(void) 662 + { 663 + debugfs_create_file("rfi_flush", 0600, powerpc_debugfs_root, NULL, &fops_rfi_flush); 664 + debugfs_create_file("entry_flush", 0600, powerpc_debugfs_root, NULL, &fops_entry_flush); 665 + debugfs_create_file("uaccess_flush", 0600, powerpc_debugfs_root, NULL, &fops_uaccess_flush); 666 + return 0; 667 + } 668 + device_initcall(rfi_flush_debugfs_init); 756 669 #endif /* CONFIG_DEBUG_FS */ 757 670 #endif /* CONFIG_PPC_BOOK3S_64 */

+1 -2

arch/powerpc/kernel/setup-common.c

··· 69 69 #include "setup.h" 70 70 71 71 #ifdef DEBUG 72 - #include <asm/udbg.h> 73 72 #define DBG(fmt...) udbg_printf(fmt) 74 73 #else 75 74 #define DBG(fmt...) ··· 828 829 } 829 830 830 831 #ifdef CONFIG_SMP 831 - static void smp_setup_pacas(void) 832 + static void __init smp_setup_pacas(void) 832 833 { 833 834 int cpu; 834 835

+1 -1

arch/powerpc/kernel/setup_32.c

··· 164 164 } 165 165 166 166 #ifdef CONFIG_VMAP_STACK 167 - void *emergency_ctx[NR_CPUS] __ro_after_init; 167 + void *emergency_ctx[NR_CPUS] __ro_after_init = {[0] = &init_stack}; 168 168 169 169 void __init emergency_stack_init(void) 170 170 {

+16 -266

arch/powerpc/kernel/setup_64.c

··· 232 232 * If we are not in hypervisor mode the job is done once for 233 233 * the whole partition in configure_exceptions(). 234 234 */ 235 - if (cpu_has_feature(CPU_FTR_HVMODE) && 236 - cpu_has_feature(CPU_FTR_ARCH_207S)) { 235 + if (cpu_has_feature(CPU_FTR_HVMODE)) { 237 236 unsigned long lpcr = mfspr(SPRN_LPCR); 238 - mtspr(SPRN_LPCR, lpcr | LPCR_AIL_3); 237 + unsigned long new_lpcr = lpcr; 238 + 239 + if (cpu_has_feature(CPU_FTR_ARCH_31)) { 240 + /* P10 DD1 does not have HAIL */ 241 + if (pvr_version_is(PVR_POWER10) && 242 + (mfspr(SPRN_PVR) & 0xf00) == 0x100) 243 + new_lpcr |= LPCR_AIL_3; 244 + else 245 + new_lpcr |= LPCR_HAIL; 246 + } else if (cpu_has_feature(CPU_FTR_ARCH_207S)) { 247 + new_lpcr |= LPCR_AIL_3; 248 + } 249 + 250 + if (new_lpcr != lpcr) 251 + mtspr(SPRN_LPCR, new_lpcr); 239 252 } 240 253 241 254 /* ··· 954 941 return 0; 955 942 } 956 943 early_initcall(disable_hardlockup_detector); 957 - 958 - #ifdef CONFIG_PPC_BOOK3S_64 959 - static enum l1d_flush_type enabled_flush_types; 960 - static void *l1d_flush_fallback_area; 961 - static bool no_rfi_flush; 962 - static bool no_entry_flush; 963 - static bool no_uaccess_flush; 964 - bool rfi_flush; 965 - bool entry_flush; 966 - bool uaccess_flush; 967 - DEFINE_STATIC_KEY_FALSE(uaccess_flush_key); 968 - EXPORT_SYMBOL(uaccess_flush_key); 969 - 970 - static int __init handle_no_rfi_flush(char *p) 971 - { 972 - pr_info("rfi-flush: disabled on command line."); 973 - no_rfi_flush = true; 974 - return 0; 975 - } 976 - early_param("no_rfi_flush", handle_no_rfi_flush); 977 - 978 - static int __init handle_no_entry_flush(char *p) 979 - { 980 - pr_info("entry-flush: disabled on command line."); 981 - no_entry_flush = true; 982 - return 0; 983 - } 984 - early_param("no_entry_flush", handle_no_entry_flush); 985 - 986 - static int __init handle_no_uaccess_flush(char *p) 987 - { 988 - pr_info("uaccess-flush: disabled on command line."); 989 - no_uaccess_flush = true; 990 - return 0; 991 - } 992 - early_param("no_uaccess_flush", handle_no_uaccess_flush); 993 - 994 - /* 995 - * The RFI flush is not KPTI, but because users will see doco that says to use 996 - * nopti we hijack that option here to also disable the RFI flush. 997 - */ 998 - static int __init handle_no_pti(char *p) 999 - { 1000 - pr_info("rfi-flush: disabling due to 'nopti' on command line.\n"); 1001 - handle_no_rfi_flush(NULL); 1002 - return 0; 1003 - } 1004 - early_param("nopti", handle_no_pti); 1005 - 1006 - static void do_nothing(void *unused) 1007 - { 1008 - /* 1009 - * We don't need to do the flush explicitly, just enter+exit kernel is 1010 - * sufficient, the RFI exit handlers will do the right thing. 1011 - */ 1012 - } 1013 - 1014 - void rfi_flush_enable(bool enable) 1015 - { 1016 - if (enable) { 1017 - do_rfi_flush_fixups(enabled_flush_types); 1018 - on_each_cpu(do_nothing, NULL, 1); 1019 - } else 1020 - do_rfi_flush_fixups(L1D_FLUSH_NONE); 1021 - 1022 - rfi_flush = enable; 1023 - } 1024 - 1025 - static void entry_flush_enable(bool enable) 1026 - { 1027 - if (enable) { 1028 - do_entry_flush_fixups(enabled_flush_types); 1029 - on_each_cpu(do_nothing, NULL, 1); 1030 - } else { 1031 - do_entry_flush_fixups(L1D_FLUSH_NONE); 1032 - } 1033 - 1034 - entry_flush = enable; 1035 - } 1036 - 1037 - static void uaccess_flush_enable(bool enable) 1038 - { 1039 - if (enable) { 1040 - do_uaccess_flush_fixups(enabled_flush_types); 1041 - static_branch_enable(&uaccess_flush_key); 1042 - on_each_cpu(do_nothing, NULL, 1); 1043 - } else { 1044 - static_branch_disable(&uaccess_flush_key); 1045 - do_uaccess_flush_fixups(L1D_FLUSH_NONE); 1046 - } 1047 - 1048 - uaccess_flush = enable; 1049 - } 1050 - 1051 - static void __ref init_fallback_flush(void) 1052 - { 1053 - u64 l1d_size, limit; 1054 - int cpu; 1055 - 1056 - /* Only allocate the fallback flush area once (at boot time). */ 1057 - if (l1d_flush_fallback_area) 1058 - return; 1059 - 1060 - l1d_size = ppc64_caches.l1d.size; 1061 - 1062 - /* 1063 - * If there is no d-cache-size property in the device tree, l1d_size 1064 - * could be zero. That leads to the loop in the asm wrapping around to 1065 - * 2^64-1, and then walking off the end of the fallback area and 1066 - * eventually causing a page fault which is fatal. Just default to 1067 - * something vaguely sane. 1068 - */ 1069 - if (!l1d_size) 1070 - l1d_size = (64 * 1024); 1071 - 1072 - limit = min(ppc64_bolted_size(), ppc64_rma_size); 1073 - 1074 - /* 1075 - * Align to L1d size, and size it at 2x L1d size, to catch possible 1076 - * hardware prefetch runoff. We don't have a recipe for load patterns to 1077 - * reliably avoid the prefetcher. 1078 - */ 1079 - l1d_flush_fallback_area = memblock_alloc_try_nid(l1d_size * 2, 1080 - l1d_size, MEMBLOCK_LOW_LIMIT, 1081 - limit, NUMA_NO_NODE); 1082 - if (!l1d_flush_fallback_area) 1083 - panic("%s: Failed to allocate %llu bytes align=0x%llx max_addr=%pa\n", 1084 - __func__, l1d_size * 2, l1d_size, &limit); 1085 - 1086 - 1087 - for_each_possible_cpu(cpu) { 1088 - struct paca_struct *paca = paca_ptrs[cpu]; 1089 - paca->rfi_flush_fallback_area = l1d_flush_fallback_area; 1090 - paca->l1d_flush_size = l1d_size; 1091 - } 1092 - } 1093 - 1094 - void setup_rfi_flush(enum l1d_flush_type types, bool enable) 1095 - { 1096 - if (types & L1D_FLUSH_FALLBACK) { 1097 - pr_info("rfi-flush: fallback displacement flush available\n"); 1098 - init_fallback_flush(); 1099 - } 1100 - 1101 - if (types & L1D_FLUSH_ORI) 1102 - pr_info("rfi-flush: ori type flush available\n"); 1103 - 1104 - if (types & L1D_FLUSH_MTTRIG) 1105 - pr_info("rfi-flush: mttrig type flush available\n"); 1106 - 1107 - enabled_flush_types = types; 1108 - 1109 - if (!cpu_mitigations_off() && !no_rfi_flush) 1110 - rfi_flush_enable(enable); 1111 - } 1112 - 1113 - void setup_entry_flush(bool enable) 1114 - { 1115 - if (cpu_mitigations_off()) 1116 - return; 1117 - 1118 - if (!no_entry_flush) 1119 - entry_flush_enable(enable); 1120 - } 1121 - 1122 - void setup_uaccess_flush(bool enable) 1123 - { 1124 - if (cpu_mitigations_off()) 1125 - return; 1126 - 1127 - if (!no_uaccess_flush) 1128 - uaccess_flush_enable(enable); 1129 - } 1130 - 1131 - #ifdef CONFIG_DEBUG_FS 1132 - static int rfi_flush_set(void *data, u64 val) 1133 - { 1134 - bool enable; 1135 - 1136 - if (val == 1) 1137 - enable = true; 1138 - else if (val == 0) 1139 - enable = false; 1140 - else 1141 - return -EINVAL; 1142 - 1143 - /* Only do anything if we're changing state */ 1144 - if (enable != rfi_flush) 1145 - rfi_flush_enable(enable); 1146 - 1147 - return 0; 1148 - } 1149 - 1150 - static int rfi_flush_get(void *data, u64 *val) 1151 - { 1152 - *val = rfi_flush ? 1 : 0; 1153 - return 0; 1154 - } 1155 - 1156 - DEFINE_SIMPLE_ATTRIBUTE(fops_rfi_flush, rfi_flush_get, rfi_flush_set, "%llu\n"); 1157 - 1158 - static int entry_flush_set(void *data, u64 val) 1159 - { 1160 - bool enable; 1161 - 1162 - if (val == 1) 1163 - enable = true; 1164 - else if (val == 0) 1165 - enable = false; 1166 - else 1167 - return -EINVAL; 1168 - 1169 - /* Only do anything if we're changing state */ 1170 - if (enable != entry_flush) 1171 - entry_flush_enable(enable); 1172 - 1173 - return 0; 1174 - } 1175 - 1176 - static int entry_flush_get(void *data, u64 *val) 1177 - { 1178 - *val = entry_flush ? 1 : 0; 1179 - return 0; 1180 - } 1181 - 1182 - DEFINE_SIMPLE_ATTRIBUTE(fops_entry_flush, entry_flush_get, entry_flush_set, "%llu\n"); 1183 - 1184 - static int uaccess_flush_set(void *data, u64 val) 1185 - { 1186 - bool enable; 1187 - 1188 - if (val == 1) 1189 - enable = true; 1190 - else if (val == 0) 1191 - enable = false; 1192 - else 1193 - return -EINVAL; 1194 - 1195 - /* Only do anything if we're changing state */ 1196 - if (enable != uaccess_flush) 1197 - uaccess_flush_enable(enable); 1198 - 1199 - return 0; 1200 - } 1201 - 1202 - static int uaccess_flush_get(void *data, u64 *val) 1203 - { 1204 - *val = uaccess_flush ? 1 : 0; 1205 - return 0; 1206 - } 1207 - 1208 - DEFINE_SIMPLE_ATTRIBUTE(fops_uaccess_flush, uaccess_flush_get, uaccess_flush_set, "%llu\n"); 1209 - 1210 - static __init int rfi_flush_debugfs_init(void) 1211 - { 1212 - debugfs_create_file("rfi_flush", 0600, powerpc_debugfs_root, NULL, &fops_rfi_flush); 1213 - debugfs_create_file("entry_flush", 0600, powerpc_debugfs_root, NULL, &fops_entry_flush); 1214 - debugfs_create_file("uaccess_flush", 0600, powerpc_debugfs_root, NULL, &fops_uaccess_flush); 1215 - return 0; 1216 - } 1217 - device_initcall(rfi_flush_debugfs_init); 1218 - #endif 1219 - #endif /* CONFIG_PPC_BOOK3S_64 */

+55

arch/powerpc/kernel/signal.h

··· 19 19 extern int handle_rt_signal32(struct ksignal *ksig, sigset_t *oldset, 20 20 struct task_struct *tsk); 21 21 22 + static inline int __get_user_sigset(sigset_t *dst, const sigset_t __user *src) 23 + { 24 + BUILD_BUG_ON(sizeof(sigset_t) != sizeof(u64)); 25 + 26 + return __get_user(dst->sig[0], (u64 __user *)&src->sig[0]); 27 + } 28 + #define unsafe_get_user_sigset(dst, src, label) \ 29 + unsafe_get_user((dst)->sig[0], (u64 __user *)&(src)->sig[0], label) 30 + 22 31 #ifdef CONFIG_VSX 23 32 extern unsigned long copy_vsx_to_user(void __user *to, 24 33 struct task_struct *task); ··· 62 53 &buf[i], label);\ 63 54 } while (0) 64 55 56 + #define unsafe_copy_fpr_from_user(task, from, label) do { \ 57 + struct task_struct *__t = task; \ 58 + u64 __user *buf = (u64 __user *)from; \ 59 + int i; \ 60 + \ 61 + for (i = 0; i < ELF_NFPREG - 1; i++) \ 62 + unsafe_get_user(__t->thread.TS_FPR(i), &buf[i], label); \ 63 + unsafe_get_user(__t->thread.fp_state.fpscr, &buf[i], label); \ 64 + } while (0) 65 + 66 + #define unsafe_copy_vsx_from_user(task, from, label) do { \ 67 + struct task_struct *__t = task; \ 68 + u64 __user *buf = (u64 __user *)from; \ 69 + int i; \ 70 + \ 71 + for (i = 0; i < ELF_NVSRHALFREG ; i++) \ 72 + unsafe_get_user(__t->thread.fp_state.fpr[i][TS_VSRLOWOFFSET], \ 73 + &buf[i], label); \ 74 + } while (0) 75 + 65 76 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM 66 77 #define unsafe_copy_ckfpr_to_user(to, task, label) do { \ 67 78 struct task_struct *__t = task; \ ··· 102 73 unsafe_put_user(__t->thread.ckfp_state.fpr[i][TS_VSRLOWOFFSET], \ 103 74 &buf[i], label);\ 104 75 } while (0) 76 + 77 + #define unsafe_copy_ckfpr_from_user(task, from, label) do { \ 78 + struct task_struct *__t = task; \ 79 + u64 __user *buf = (u64 __user *)from; \ 80 + int i; \ 81 + \ 82 + for (i = 0; i < ELF_NFPREG - 1 ; i++) \ 83 + unsafe_get_user(__t->thread.TS_CKFPR(i), &buf[i], label);\ 84 + unsafe_get_user(__t->thread.ckfp_state.fpscr, &buf[i], failed); \ 85 + } while (0) 86 + 87 + #define unsafe_copy_ckvsx_from_user(task, from, label) do { \ 88 + struct task_struct *__t = task; \ 89 + u64 __user *buf = (u64 __user *)from; \ 90 + int i; \ 91 + \ 92 + for (i = 0; i < ELF_NVSRHALFREG ; i++) \ 93 + unsafe_get_user(__t->thread.ckfp_state.fpr[i][TS_VSRLOWOFFSET], \ 94 + &buf[i], label); \ 95 + } while (0) 105 96 #endif 106 97 #elif defined(CONFIG_PPC_FPU_REGS) 107 98 108 99 #define unsafe_copy_fpr_to_user(to, task, label) \ 109 100 unsafe_copy_to_user(to, (task)->thread.fp_state.fpr, \ 101 + ELF_NFPREG * sizeof(double), label) 102 + 103 + #define unsafe_copy_fpr_from_user(task, from, label) \ 104 + unsafe_copy_from_user((task)->thread.fp_state.fpr, from, \ 110 105 ELF_NFPREG * sizeof(double), label) 111 106 112 107 static inline unsigned long ··· 167 114 #endif /* CONFIG_PPC_TRANSACTIONAL_MEM */ 168 115 #else 169 116 #define unsafe_copy_fpr_to_user(to, task, label) do { } while (0) 117 + 118 + #define unsafe_copy_fpr_from_user(task, from, label) do { } while (0) 170 119 171 120 static inline unsigned long 172 121 copy_fpr_to_user(void __user *to, struct task_struct *task)

+143 -111

arch/powerpc/kernel/signal_32.c

··· 83 83 * implementation that makes things simple for little endian only) 84 84 */ 85 85 #define unsafe_put_sigset_t unsafe_put_compat_sigset 86 - 87 - static inline int get_sigset_t(sigset_t *set, 88 - const compat_sigset_t __user *uset) 89 - { 90 - return get_compat_sigset(set, uset); 91 - } 86 + #define unsafe_get_sigset_t unsafe_get_compat_sigset 92 87 93 88 #define to_user_ptr(p) ptr_to_compat(p) 94 89 #define from_user_ptr(p) compat_ptr(p) 95 90 96 91 static __always_inline int 97 - save_general_regs_unsafe(struct pt_regs *regs, struct mcontext __user *frame) 92 + __unsafe_save_general_regs(struct pt_regs *regs, struct mcontext __user *frame) 98 93 { 99 94 elf_greg_t64 *gregs = (elf_greg_t64 *)regs; 100 95 int val, i; 101 - 102 - WARN_ON(!FULL_REGS(regs)); 103 96 104 97 for (i = 0; i <= PT_RESULT; i ++) { 105 98 /* Force usr to alway see softe as 1 (interrupts enabled) */ ··· 109 116 return 1; 110 117 } 111 118 112 - static inline int restore_general_regs(struct pt_regs *regs, 113 - struct mcontext __user *sr) 119 + static __always_inline int 120 + __unsafe_restore_general_regs(struct pt_regs *regs, struct mcontext __user *sr) 114 121 { 115 122 elf_greg_t64 *gregs = (elf_greg_t64 *)regs; 116 123 int i; ··· 118 125 for (i = 0; i <= PT_RESULT; i++) { 119 126 if ((i == PT_MSR) || (i == PT_SOFTE)) 120 127 continue; 121 - if (__get_user(gregs[i], &sr->mc_gregs[i])) 122 - return -EFAULT; 128 + unsafe_get_user(gregs[i], &sr->mc_gregs[i], failed); 123 129 } 124 130 return 0; 131 + 132 + failed: 133 + return 1; 125 134 } 126 135 127 136 #else /* CONFIG_PPC64 */ ··· 137 142 unsafe_copy_to_user(__us, __s, sizeof(*__us), label); \ 138 143 } while (0) 139 144 140 - static inline int get_sigset_t(sigset_t *set, const sigset_t __user *uset) 141 - { 142 - return copy_from_user(set, uset, sizeof(*uset)); 143 - } 145 + #define unsafe_get_sigset_t unsafe_get_user_sigset 144 146 145 147 #define to_user_ptr(p) ((unsigned long)(p)) 146 148 #define from_user_ptr(p) ((void __user *)(p)) 147 149 148 150 static __always_inline int 149 - save_general_regs_unsafe(struct pt_regs *regs, struct mcontext __user *frame) 151 + __unsafe_save_general_regs(struct pt_regs *regs, struct mcontext __user *frame) 150 152 { 151 - WARN_ON(!FULL_REGS(regs)); 152 153 unsafe_copy_to_user(&frame->mc_gregs, regs, GP_REGS_SIZE, failed); 153 154 return 0; 154 155 ··· 152 161 return 1; 153 162 } 154 163 155 - static inline int restore_general_regs(struct pt_regs *regs, 156 - struct mcontext __user *sr) 164 + static __always_inline 165 + int __unsafe_restore_general_regs(struct pt_regs *regs, struct mcontext __user *sr) 157 166 { 158 167 /* copy up to but not including MSR */ 159 - if (__copy_from_user(regs, &sr->mc_gregs, 160 - PT_MSR * sizeof(elf_greg_t))) 161 - return -EFAULT; 168 + unsafe_copy_from_user(regs, &sr->mc_gregs, PT_MSR * sizeof(elf_greg_t), failed); 169 + 162 170 /* copy from orig_r3 (the word after the MSR) up to the end */ 163 - if (__copy_from_user(&regs->orig_gpr3, &sr->mc_gregs[PT_ORIG_R3], 164 - GP_REGS_SIZE - PT_ORIG_R3 * sizeof(elf_greg_t))) 165 - return -EFAULT; 171 + unsafe_copy_from_user(&regs->orig_gpr3, &sr->mc_gregs[PT_ORIG_R3], 172 + GP_REGS_SIZE - PT_ORIG_R3 * sizeof(elf_greg_t), failed); 173 + 166 174 return 0; 175 + 176 + failed: 177 + return 1; 167 178 } 168 179 #endif 169 180 170 181 #define unsafe_save_general_regs(regs, frame, label) do { \ 171 - if (save_general_regs_unsafe(regs, frame)) \ 182 + if (__unsafe_save_general_regs(regs, frame)) \ 183 + goto label; \ 184 + } while (0) 185 + 186 + #define unsafe_restore_general_regs(regs, frame, label) do { \ 187 + if (__unsafe_restore_general_regs(regs, frame)) \ 172 188 goto label; \ 173 189 } while (0) 174 190 ··· 258 260 #endif 259 261 } 260 262 261 - static int save_user_regs_unsafe(struct pt_regs *regs, struct mcontext __user *frame, 262 - struct mcontext __user *tm_frame, int ctx_has_vsx_region) 263 + static int __unsafe_save_user_regs(struct pt_regs *regs, struct mcontext __user *frame, 264 + struct mcontext __user *tm_frame, int ctx_has_vsx_region) 263 265 { 264 266 unsigned long msr = regs->msr; 265 267 ··· 336 338 } 337 339 338 340 #define unsafe_save_user_regs(regs, frame, tm_frame, has_vsx, label) do { \ 339 - if (save_user_regs_unsafe(regs, frame, tm_frame, has_vsx)) \ 341 + if (__unsafe_save_user_regs(regs, frame, tm_frame, has_vsx)) \ 340 342 goto label; \ 341 343 } while (0) 342 344 ··· 348 350 * We also save the transactional registers to a second ucontext in the 349 351 * frame. 350 352 * 351 - * See save_user_regs_unsafe() and signal_64.c:setup_tm_sigcontexts(). 353 + * See __unsafe_save_user_regs() and signal_64.c:setup_tm_sigcontexts(). 352 354 */ 353 355 static void prepare_save_tm_user_regs(void) 354 356 { ··· 439 441 #endif /* CONFIG_VSX */ 440 442 #ifdef CONFIG_SPE 441 443 /* SPE regs are not checkpointed with TM, so this section is 442 - * simply the same as in save_user_regs_unsafe(). 444 + * simply the same as in __unsafe_save_user_regs(). 443 445 */ 444 446 if (current->thread.used_spe) { 445 447 unsafe_copy_to_user(&frame->mc_vregs, current->thread.evr, ··· 483 485 static long restore_user_regs(struct pt_regs *regs, 484 486 struct mcontext __user *sr, int sig) 485 487 { 486 - long err; 487 488 unsigned int save_r2 = 0; 488 489 unsigned long msr; 489 490 #ifdef CONFIG_VSX 490 491 int i; 491 492 #endif 492 493 494 + if (!user_read_access_begin(sr, sizeof(*sr))) 495 + return 1; 493 496 /* 494 497 * restore general registers but not including MSR or SOFTE. Also 495 498 * take care of keeping r2 (TLS) intact if not a signal 496 499 */ 497 500 if (!sig) 498 501 save_r2 = (unsigned int)regs->gpr[2]; 499 - err = restore_general_regs(regs, sr); 502 + unsafe_restore_general_regs(regs, sr, failed); 500 503 set_trap_norestart(regs); 501 - err |= __get_user(msr, &sr->mc_gregs[PT_MSR]); 504 + unsafe_get_user(msr, &sr->mc_gregs[PT_MSR], failed); 502 505 if (!sig) 503 506 regs->gpr[2] = (unsigned long) save_r2; 504 - if (err) 505 - return 1; 506 507 507 508 /* if doing signal return, restore the previous little-endian mode */ 508 509 if (sig) ··· 515 518 regs->msr &= ~MSR_VEC; 516 519 if (msr & MSR_VEC) { 517 520 /* restore altivec registers from the stack */ 518 - if (__copy_from_user(&current->thread.vr_state, &sr->mc_vregs, 519 - sizeof(sr->mc_vregs))) 520 - return 1; 521 + unsafe_copy_from_user(&current->thread.vr_state, &sr->mc_vregs, 522 + sizeof(sr->mc_vregs), failed); 521 523 current->thread.used_vr = true; 522 524 } else if (current->thread.used_vr) 523 525 memset(&current->thread.vr_state, 0, 524 526 ELF_NVRREG * sizeof(vector128)); 525 527 526 528 /* Always get VRSAVE back */ 527 - if (__get_user(current->thread.vrsave, (u32 __user *)&sr->mc_vregs[32])) 528 - return 1; 529 + unsafe_get_user(current->thread.vrsave, (u32 __user *)&sr->mc_vregs[32], failed); 529 530 if (cpu_has_feature(CPU_FTR_ALTIVEC)) 530 531 mtspr(SPRN_VRSAVE, current->thread.vrsave); 531 532 #endif /* CONFIG_ALTIVEC */ 532 - if (copy_fpr_from_user(current, &sr->mc_fregs)) 533 - return 1; 533 + unsafe_copy_fpr_from_user(current, &sr->mc_fregs, failed); 534 534 535 535 #ifdef CONFIG_VSX 536 536 /* ··· 540 546 * Restore altivec registers from the stack to a local 541 547 * buffer, then write this out to the thread_struct 542 548 */ 543 - if (copy_vsx_from_user(current, &sr->mc_vsregs)) 544 - return 1; 549 + unsafe_copy_vsx_from_user(current, &sr->mc_vsregs, failed); 545 550 current->thread.used_vsr = true; 546 551 } else if (current->thread.used_vsr) 547 552 for (i = 0; i < 32 ; i++) ··· 558 565 regs->msr &= ~MSR_SPE; 559 566 if (msr & MSR_SPE) { 560 567 /* restore spe registers from the stack */ 561 - if (__copy_from_user(current->thread.evr, &sr->mc_vregs, 562 - ELF_NEVRREG * sizeof(u32))) 563 - return 1; 568 + unsafe_copy_from_user(current->thread.evr, &sr->mc_vregs, 569 + ELF_NEVRREG * sizeof(u32), failed); 564 570 current->thread.used_spe = true; 565 571 } else if (current->thread.used_spe) 566 572 memset(current->thread.evr, 0, ELF_NEVRREG * sizeof(u32)); 567 573 568 574 /* Always get SPEFSCR back */ 569 - if (__get_user(current->thread.spefscr, (u32 __user *)&sr->mc_vregs + ELF_NEVRREG)) 570 - return 1; 575 + unsafe_get_user(current->thread.spefscr, (u32 __user *)&sr->mc_vregs + ELF_NEVRREG, failed); 571 576 #endif /* CONFIG_SPE */ 572 577 578 + user_read_access_end(); 573 579 return 0; 580 + 581 + failed: 582 + user_read_access_end(); 583 + return 1; 574 584 } 575 585 576 586 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM ··· 586 590 struct mcontext __user *sr, 587 591 struct mcontext __user *tm_sr) 588 592 { 589 - long err; 590 593 unsigned long msr, msr_hi; 591 594 #ifdef CONFIG_VSX 592 595 int i; ··· 600 605 * TFHAR is restored from the checkpointed NIP; TEXASR and TFIAR 601 606 * were set by the signal delivery. 602 607 */ 603 - err = restore_general_regs(regs, tm_sr); 604 - err |= restore_general_regs(&current->thread.ckpt_regs, sr); 605 - 606 - err |= __get_user(current->thread.tm_tfhar, &sr->mc_gregs[PT_NIP]); 607 - 608 - err |= __get_user(msr, &sr->mc_gregs[PT_MSR]); 609 - if (err) 608 + if (!user_read_access_begin(sr, sizeof(*sr))) 610 609 return 1; 610 + 611 + unsafe_restore_general_regs(&current->thread.ckpt_regs, sr, failed); 612 + unsafe_get_user(current->thread.tm_tfhar, &sr->mc_gregs[PT_NIP], failed); 613 + unsafe_get_user(msr, &sr->mc_gregs[PT_MSR], failed); 611 614 612 615 /* Restore the previous little-endian mode */ 613 616 regs->msr = (regs->msr & ~MSR_LE) | (msr & MSR_LE); ··· 614 621 regs->msr &= ~MSR_VEC; 615 622 if (msr & MSR_VEC) { 616 623 /* restore altivec registers from the stack */ 617 - if (__copy_from_user(&current->thread.ckvr_state, &sr->mc_vregs, 618 - sizeof(sr->mc_vregs)) || 619 - __copy_from_user(&current->thread.vr_state, 620 - &tm_sr->mc_vregs, 621 - sizeof(sr->mc_vregs))) 622 - return 1; 624 + unsafe_copy_from_user(&current->thread.ckvr_state, &sr->mc_vregs, 625 + sizeof(sr->mc_vregs), failed); 623 626 current->thread.used_vr = true; 624 627 } else if (current->thread.used_vr) { 625 628 memset(&current->thread.vr_state, 0, ··· 625 636 } 626 637 627 638 /* Always get VRSAVE back */ 628 - if (__get_user(current->thread.ckvrsave, 629 - (u32 __user *)&sr->mc_vregs[32]) || 630 - __get_user(current->thread.vrsave, 631 - (u32 __user *)&tm_sr->mc_vregs[32])) 632 - return 1; 639 + unsafe_get_user(current->thread.ckvrsave, 640 + (u32 __user *)&sr->mc_vregs[32], failed); 633 641 if (cpu_has_feature(CPU_FTR_ALTIVEC)) 634 642 mtspr(SPRN_VRSAVE, current->thread.ckvrsave); 635 643 #endif /* CONFIG_ALTIVEC */ 636 644 637 645 regs->msr &= ~(MSR_FP | MSR_FE0 | MSR_FE1); 638 646 639 - if (copy_fpr_from_user(current, &sr->mc_fregs) || 640 - copy_ckfpr_from_user(current, &tm_sr->mc_fregs)) 641 - return 1; 647 + unsafe_copy_fpr_from_user(current, &sr->mc_fregs, failed); 642 648 643 649 #ifdef CONFIG_VSX 644 650 regs->msr &= ~MSR_VSX; ··· 642 658 * Restore altivec registers from the stack to a local 643 659 * buffer, then write this out to the thread_struct 644 660 */ 645 - if (copy_vsx_from_user(current, &tm_sr->mc_vsregs) || 646 - copy_ckvsx_from_user(current, &sr->mc_vsregs)) 647 - return 1; 661 + unsafe_copy_ckvsx_from_user(current, &sr->mc_vsregs, failed); 648 662 current->thread.used_vsr = true; 649 663 } else if (current->thread.used_vsr) 650 664 for (i = 0; i < 32 ; i++) { ··· 657 675 */ 658 676 regs->msr &= ~MSR_SPE; 659 677 if (msr & MSR_SPE) { 660 - if (__copy_from_user(current->thread.evr, &sr->mc_vregs, 661 - ELF_NEVRREG * sizeof(u32))) 662 - return 1; 678 + unsafe_copy_from_user(current->thread.evr, &sr->mc_vregs, 679 + ELF_NEVRREG * sizeof(u32), failed); 663 680 current->thread.used_spe = true; 664 681 } else if (current->thread.used_spe) 665 682 memset(current->thread.evr, 0, ELF_NEVRREG * sizeof(u32)); 666 683 667 684 /* Always get SPEFSCR back */ 668 - if (__get_user(current->thread.spefscr, (u32 __user *)&sr->mc_vregs 669 - + ELF_NEVRREG)) 670 - return 1; 685 + unsafe_get_user(current->thread.spefscr, 686 + (u32 __user *)&sr->mc_vregs + ELF_NEVRREG, failed); 671 687 #endif /* CONFIG_SPE */ 672 688 673 - /* Get the top half of the MSR from the user context */ 674 - if (__get_user(msr_hi, &tm_sr->mc_gregs[PT_MSR])) 689 + user_read_access_end(); 690 + 691 + if (!user_read_access_begin(tm_sr, sizeof(*tm_sr))) 675 692 return 1; 693 + 694 + unsafe_restore_general_regs(regs, tm_sr, failed); 695 + 696 + #ifdef CONFIG_ALTIVEC 697 + /* restore altivec registers from the stack */ 698 + if (msr & MSR_VEC) 699 + unsafe_copy_from_user(&current->thread.vr_state, &tm_sr->mc_vregs, 700 + sizeof(sr->mc_vregs), failed); 701 + 702 + /* Always get VRSAVE back */ 703 + unsafe_get_user(current->thread.vrsave, 704 + (u32 __user *)&tm_sr->mc_vregs[32], failed); 705 + #endif /* CONFIG_ALTIVEC */ 706 + 707 + unsafe_copy_ckfpr_from_user(current, &tm_sr->mc_fregs, failed); 708 + 709 + #ifdef CONFIG_VSX 710 + if (msr & MSR_VSX) { 711 + /* 712 + * Restore altivec registers from the stack to a local 713 + * buffer, then write this out to the thread_struct 714 + */ 715 + unsafe_copy_vsx_from_user(current, &tm_sr->mc_vsregs, failed); 716 + current->thread.used_vsr = true; 717 + } 718 + #endif /* CONFIG_VSX */ 719 + 720 + /* Get the top half of the MSR from the user context */ 721 + unsafe_get_user(msr_hi, &tm_sr->mc_gregs[PT_MSR], failed); 676 722 msr_hi <<= 32; 723 + 724 + user_read_access_end(); 725 + 677 726 /* If TM bits are set to the reserved value, it's an invalid context */ 678 727 if (MSR_TM_RESV(msr_hi)) 679 728 return 1; ··· 751 738 752 739 preempt_enable(); 753 740 741 + return 0; 742 + 743 + failed: 744 + user_read_access_end(); 745 + return 1; 746 + } 747 + #else 748 + static long restore_tm_user_regs(struct pt_regs *regs, struct mcontext __user *sr, 749 + struct mcontext __user *tm_sr) 750 + { 754 751 return 0; 755 752 } 756 753 #endif ··· 967 944 sigset_t set; 968 945 struct mcontext __user *mcp; 969 946 970 - if (get_sigset_t(&set, &ucp->uc_sigmask)) 947 + if (!user_read_access_begin(ucp, sizeof(*ucp))) 971 948 return -EFAULT; 949 + 950 + unsafe_get_sigset_t(&set, &ucp->uc_sigmask, failed); 972 951 #ifdef CONFIG_PPC64 973 952 { 974 953 u32 cmcp; 975 954 976 - if (__get_user(cmcp, &ucp->uc_regs)) 977 - return -EFAULT; 955 + unsafe_get_user(cmcp, &ucp->uc_regs, failed); 978 956 mcp = (struct mcontext __user *)(u64)cmcp; 979 - /* no need to check access_ok(mcp), since mcp < 4GB */ 980 957 } 981 958 #else 982 - if (__get_user(mcp, &ucp->uc_regs)) 983 - return -EFAULT; 984 - if (!access_ok(mcp, sizeof(*mcp))) 985 - return -EFAULT; 959 + unsafe_get_user(mcp, &ucp->uc_regs, failed); 986 960 #endif 961 + user_read_access_end(); 962 + 987 963 set_current_blocked(&set); 988 964 if (restore_user_regs(regs, mcp, sig)) 989 965 return -EFAULT; 990 966 991 967 return 0; 968 + 969 + failed: 970 + user_read_access_end(); 971 + return -EFAULT; 992 972 } 993 973 994 974 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM ··· 1005 979 u32 cmcp; 1006 980 u32 tm_cmcp; 1007 981 1008 - if (get_sigset_t(&set, &ucp->uc_sigmask)) 982 + if (!user_read_access_begin(ucp, sizeof(*ucp))) 1009 983 return -EFAULT; 1010 984 1011 - if (__get_user(cmcp, &ucp->uc_regs) || 1012 - __get_user(tm_cmcp, &tm_ucp->uc_regs)) 985 + unsafe_get_sigset_t(&set, &ucp->uc_sigmask, failed); 986 + unsafe_get_user(cmcp, &ucp->uc_regs, failed); 987 + 988 + user_read_access_end(); 989 + 990 + if (__get_user(tm_cmcp, &tm_ucp->uc_regs)) 1013 991 return -EFAULT; 1014 992 mcp = (struct mcontext __user *)(u64)cmcp; 1015 993 tm_mcp = (struct mcontext __user *)(u64)tm_cmcp; ··· 1024 994 return -EFAULT; 1025 995 1026 996 return 0; 997 + 998 + failed: 999 + user_read_access_end(); 1000 + return -EFAULT; 1027 1001 } 1028 1002 #endif 1029 1003 ··· 1345 1311 struct sigcontext __user *sc; 1346 1312 struct sigcontext sigctx; 1347 1313 struct mcontext __user *sr; 1348 - void __user *addr; 1349 1314 sigset_t set; 1350 - #ifdef CONFIG_PPC_TRANSACTIONAL_MEM 1351 - struct mcontext __user *mcp, *tm_mcp; 1352 - unsigned long msr_hi; 1353 - #endif 1315 + struct mcontext __user *mcp; 1316 + struct mcontext __user *tm_mcp = NULL; 1317 + unsigned long long msr_hi = 0; 1354 1318 1355 1319 /* Always make any pending restarted system calls return -EINTR */ 1356 1320 current->restart_block.fn = do_no_restart_syscall; 1357 1321 1358 1322 sf = (struct sigframe __user *)(regs->gpr[1] + __SIGNAL_FRAMESIZE); 1359 1323 sc = &sf->sctx; 1360 - addr = sc; 1361 1324 if (copy_from_user(&sigctx, sc, sizeof(sigctx))) 1362 1325 goto badframe; 1363 1326 ··· 1370 1339 #endif 1371 1340 set_current_blocked(&set); 1372 1341 1373 - #ifdef CONFIG_PPC_TRANSACTIONAL_MEM 1374 1342 mcp = (struct mcontext __user *)&sf->mctx; 1343 + #ifdef CONFIG_PPC_TRANSACTIONAL_MEM 1375 1344 tm_mcp = (struct mcontext __user *)&sf->mctx_transact; 1376 1345 if (__get_user(msr_hi, &tm_mcp->mc_gregs[PT_MSR])) 1377 1346 goto badframe; 1347 + #endif 1378 1348 if (MSR_TM_ACTIVE(msr_hi<<32)) { 1379 1349 if (!cpu_has_feature(CPU_FTR_TM)) 1380 1350 goto badframe; 1381 1351 if (restore_tm_user_regs(regs, mcp, tm_mcp)) 1382 1352 goto badframe; 1383 - } else 1384 - #endif 1385 - { 1353 + } else { 1386 1354 sr = (struct mcontext __user *)from_user_ptr(sigctx.regs); 1387 - addr = sr; 1388 - if (!access_ok(sr, sizeof(*sr)) 1389 - || restore_user_regs(regs, sr, 1)) 1390 - goto badframe; 1355 + if (restore_user_regs(regs, sr, 1)) { 1356 + signal_fault(current, regs, "sys_sigreturn", sr); 1357 + 1358 + force_sig(SIGSEGV); 1359 + return 0; 1360 + } 1391 1361 } 1392 1362 1393 1363 set_thread_flag(TIF_RESTOREALL); 1394 1364 return 0; 1395 1365 1396 1366 badframe: 1397 - signal_fault(current, regs, "sys_sigreturn", addr); 1367 + signal_fault(current, regs, "sys_sigreturn", sc); 1398 1368 1399 1369 force_sig(SIGSEGV); 1400 1370 return 0;

+189 -130

arch/powerpc/kernel/signal_64.c

··· 79 79 } 80 80 #endif 81 81 82 + static void prepare_setup_sigcontext(struct task_struct *tsk) 83 + { 84 + #ifdef CONFIG_ALTIVEC 85 + /* save altivec registers */ 86 + if (tsk->thread.used_vr) 87 + flush_altivec_to_thread(tsk); 88 + if (cpu_has_feature(CPU_FTR_ALTIVEC)) 89 + tsk->thread.vrsave = mfspr(SPRN_VRSAVE); 90 + #endif /* CONFIG_ALTIVEC */ 91 + 92 + flush_fp_to_thread(tsk); 93 + 94 + #ifdef CONFIG_VSX 95 + if (tsk->thread.used_vsr) 96 + flush_vsx_to_thread(tsk); 97 + #endif /* CONFIG_VSX */ 98 + } 99 + 82 100 /* 83 101 * Set up the sigcontext for the signal frame. 84 102 */ 85 103 86 - static long setup_sigcontext(struct sigcontext __user *sc, 87 - struct task_struct *tsk, int signr, sigset_t *set, 88 - unsigned long handler, int ctx_has_vsx_region) 104 + #define unsafe_setup_sigcontext(sc, tsk, signr, set, handler, ctx_has_vsx_region, label)\ 105 + do { \ 106 + if (__unsafe_setup_sigcontext(sc, tsk, signr, set, handler, ctx_has_vsx_region))\ 107 + goto label; \ 108 + } while (0) 109 + static long notrace __unsafe_setup_sigcontext(struct sigcontext __user *sc, 110 + struct task_struct *tsk, int signr, sigset_t *set, 111 + unsigned long handler, int ctx_has_vsx_region) 89 112 { 90 113 /* When CONFIG_ALTIVEC is set, we _always_ setup v_regs even if the 91 114 * process never used altivec yet (MSR_VEC is zero in pt_regs of ··· 120 97 */ 121 98 #ifdef CONFIG_ALTIVEC 122 99 elf_vrreg_t __user *v_regs = sigcontext_vmx_regs(sc); 123 - unsigned long vrsave; 124 100 #endif 125 101 struct pt_regs *regs = tsk->thread.regs; 126 102 unsigned long msr = regs->msr; 127 - long err = 0; 128 103 /* Force usr to alway see softe as 1 (interrupts enabled) */ 129 104 unsigned long softe = 0x1; 130 105 131 106 BUG_ON(tsk != current); 132 107 133 108 #ifdef CONFIG_ALTIVEC 134 - err |= __put_user(v_regs, &sc->v_regs); 109 + unsafe_put_user(v_regs, &sc->v_regs, efault_out); 135 110 136 111 /* save altivec registers */ 137 112 if (tsk->thread.used_vr) { 138 - flush_altivec_to_thread(tsk); 139 113 /* Copy 33 vec registers (vr0..31 and vscr) to the stack */ 140 - err |= __copy_to_user(v_regs, &tsk->thread.vr_state, 141 - 33 * sizeof(vector128)); 114 + unsafe_copy_to_user(v_regs, &tsk->thread.vr_state, 115 + 33 * sizeof(vector128), efault_out); 142 116 /* set MSR_VEC in the MSR value in the frame to indicate that sc->v_reg) 143 117 * contains valid data. 144 118 */ ··· 144 124 /* We always copy to/from vrsave, it's 0 if we don't have or don't 145 125 * use altivec. 146 126 */ 147 - vrsave = 0; 148 - if (cpu_has_feature(CPU_FTR_ALTIVEC)) { 149 - vrsave = mfspr(SPRN_VRSAVE); 150 - tsk->thread.vrsave = vrsave; 151 - } 152 - 153 - err |= __put_user(vrsave, (u32 __user *)&v_regs[33]); 127 + unsafe_put_user(tsk->thread.vrsave, (u32 __user *)&v_regs[33], efault_out); 154 128 #else /* CONFIG_ALTIVEC */ 155 - err |= __put_user(0, &sc->v_regs); 129 + unsafe_put_user(0, &sc->v_regs, efault_out); 156 130 #endif /* CONFIG_ALTIVEC */ 157 - flush_fp_to_thread(tsk); 158 131 /* copy fpr regs and fpscr */ 159 - err |= copy_fpr_to_user(&sc->fp_regs, tsk); 132 + unsafe_copy_fpr_to_user(&sc->fp_regs, tsk, efault_out); 160 133 161 134 /* 162 135 * Clear the MSR VSX bit to indicate there is no valid state attached ··· 163 150 * VMX data. 164 151 */ 165 152 if (tsk->thread.used_vsr && ctx_has_vsx_region) { 166 - flush_vsx_to_thread(tsk); 167 153 v_regs += ELF_NVRREG; 168 - err |= copy_vsx_to_user(v_regs, tsk); 154 + unsafe_copy_vsx_to_user(v_regs, tsk, efault_out); 169 155 /* set MSR_VSX in the MSR value in the frame to 170 156 * indicate that sc->vs_reg) contains valid data. 171 157 */ 172 158 msr |= MSR_VSX; 173 159 } 174 160 #endif /* CONFIG_VSX */ 175 - err |= __put_user(&sc->gp_regs, &sc->regs); 176 - WARN_ON(!FULL_REGS(regs)); 177 - err |= __copy_to_user(&sc->gp_regs, regs, GP_REGS_SIZE); 178 - err |= __put_user(msr, &sc->gp_regs[PT_MSR]); 179 - err |= __put_user(softe, &sc->gp_regs[PT_SOFTE]); 180 - err |= __put_user(signr, &sc->signal); 181 - err |= __put_user(handler, &sc->handler); 161 + unsafe_put_user(&sc->gp_regs, &sc->regs, efault_out); 162 + unsafe_copy_to_user(&sc->gp_regs, regs, GP_REGS_SIZE, efault_out); 163 + unsafe_put_user(msr, &sc->gp_regs[PT_MSR], efault_out); 164 + unsafe_put_user(softe, &sc->gp_regs[PT_SOFTE], efault_out); 165 + unsafe_put_user(signr, &sc->signal, efault_out); 166 + unsafe_put_user(handler, &sc->handler, efault_out); 182 167 if (set != NULL) 183 - err |= __put_user(set->sig[0], &sc->oldmask); 168 + unsafe_put_user(set->sig[0], &sc->oldmask, efault_out); 184 169 185 - return err; 170 + return 0; 171 + 172 + efault_out: 173 + return -EFAULT; 186 174 } 187 175 188 176 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM ··· 308 294 309 295 err |= __put_user(&sc->gp_regs, &sc->regs); 310 296 err |= __put_user(&tm_sc->gp_regs, &tm_sc->regs); 311 - WARN_ON(!FULL_REGS(regs)); 312 297 err |= __copy_to_user(&tm_sc->gp_regs, regs, GP_REGS_SIZE); 313 298 err |= __copy_to_user(&sc->gp_regs, 314 299 &tsk->thread.ckpt_regs, GP_REGS_SIZE); ··· 325 312 /* 326 313 * Restore the sigcontext from the signal frame. 327 314 */ 328 - 329 - static long restore_sigcontext(struct task_struct *tsk, sigset_t *set, int sig, 330 - struct sigcontext __user *sc) 315 + #define unsafe_restore_sigcontext(tsk, set, sig, sc, label) do { \ 316 + if (__unsafe_restore_sigcontext(tsk, set, sig, sc)) \ 317 + goto label; \ 318 + } while (0) 319 + static long notrace __unsafe_restore_sigcontext(struct task_struct *tsk, sigset_t *set, 320 + int sig, struct sigcontext __user *sc) 331 321 { 332 322 #ifdef CONFIG_ALTIVEC 333 323 elf_vrreg_t __user *v_regs; 334 324 #endif 335 - unsigned long err = 0; 336 325 unsigned long save_r13 = 0; 337 326 unsigned long msr; 338 327 struct pt_regs *regs = tsk->thread.regs; ··· 349 334 save_r13 = regs->gpr[13]; 350 335 351 336 /* copy the GPRs */ 352 - err |= __copy_from_user(regs->gpr, sc->gp_regs, sizeof(regs->gpr)); 353 - err |= __get_user(regs->nip, &sc->gp_regs[PT_NIP]); 337 + unsafe_copy_from_user(regs->gpr, sc->gp_regs, sizeof(regs->gpr), efault_out); 338 + unsafe_get_user(regs->nip, &sc->gp_regs[PT_NIP], efault_out); 354 339 /* get MSR separately, transfer the LE bit if doing signal return */ 355 - err |= __get_user(msr, &sc->gp_regs[PT_MSR]); 340 + unsafe_get_user(msr, &sc->gp_regs[PT_MSR], efault_out); 356 341 if (sig) 357 342 regs->msr = (regs->msr & ~MSR_LE) | (msr & MSR_LE); 358 - err |= __get_user(regs->orig_gpr3, &sc->gp_regs[PT_ORIG_R3]); 359 - err |= __get_user(regs->ctr, &sc->gp_regs[PT_CTR]); 360 - err |= __get_user(regs->link, &sc->gp_regs[PT_LNK]); 361 - err |= __get_user(regs->xer, &sc->gp_regs[PT_XER]); 362 - err |= __get_user(regs->ccr, &sc->gp_regs[PT_CCR]); 343 + unsafe_get_user(regs->orig_gpr3, &sc->gp_regs[PT_ORIG_R3], efault_out); 344 + unsafe_get_user(regs->ctr, &sc->gp_regs[PT_CTR], efault_out); 345 + unsafe_get_user(regs->link, &sc->gp_regs[PT_LNK], efault_out); 346 + unsafe_get_user(regs->xer, &sc->gp_regs[PT_XER], efault_out); 347 + unsafe_get_user(regs->ccr, &sc->gp_regs[PT_CCR], efault_out); 363 348 /* Don't allow userspace to set SOFTE */ 364 349 set_trap_norestart(regs); 365 - err |= __get_user(regs->dar, &sc->gp_regs[PT_DAR]); 366 - err |= __get_user(regs->dsisr, &sc->gp_regs[PT_DSISR]); 367 - err |= __get_user(regs->result, &sc->gp_regs[PT_RESULT]); 350 + unsafe_get_user(regs->dar, &sc->gp_regs[PT_DAR], efault_out); 351 + unsafe_get_user(regs->dsisr, &sc->gp_regs[PT_DSISR], efault_out); 352 + unsafe_get_user(regs->result, &sc->gp_regs[PT_RESULT], efault_out); 368 353 369 354 if (!sig) 370 355 regs->gpr[13] = save_r13; 371 356 if (set != NULL) 372 - err |= __get_user(set->sig[0], &sc->oldmask); 357 + unsafe_get_user(set->sig[0], &sc->oldmask, efault_out); 373 358 374 359 /* 375 360 * Force reload of FP/VEC. ··· 379 364 regs->msr &= ~(MSR_FP | MSR_FE0 | MSR_FE1 | MSR_VEC | MSR_VSX); 380 365 381 366 #ifdef CONFIG_ALTIVEC 382 - err |= __get_user(v_regs, &sc->v_regs); 383 - if (err) 384 - return err; 367 + unsafe_get_user(v_regs, &sc->v_regs, efault_out); 385 368 if (v_regs && !access_ok(v_regs, 34 * sizeof(vector128))) 386 369 return -EFAULT; 387 370 /* Copy 33 vec registers (vr0..31 and vscr) from the stack */ 388 371 if (v_regs != NULL && (msr & MSR_VEC) != 0) { 389 - err |= __copy_from_user(&tsk->thread.vr_state, v_regs, 390 - 33 * sizeof(vector128)); 372 + unsafe_copy_from_user(&tsk->thread.vr_state, v_regs, 373 + 33 * sizeof(vector128), efault_out); 391 374 tsk->thread.used_vr = true; 392 375 } else if (tsk->thread.used_vr) { 393 376 memset(&tsk->thread.vr_state, 0, 33 * sizeof(vector128)); 394 377 } 395 378 /* Always get VRSAVE back */ 396 379 if (v_regs != NULL) 397 - err |= __get_user(tsk->thread.vrsave, (u32 __user *)&v_regs[33]); 380 + unsafe_get_user(tsk->thread.vrsave, (u32 __user *)&v_regs[33], efault_out); 398 381 else 399 382 tsk->thread.vrsave = 0; 400 383 if (cpu_has_feature(CPU_FTR_ALTIVEC)) 401 384 mtspr(SPRN_VRSAVE, tsk->thread.vrsave); 402 385 #endif /* CONFIG_ALTIVEC */ 403 386 /* restore floating point */ 404 - err |= copy_fpr_from_user(tsk, &sc->fp_regs); 387 + unsafe_copy_fpr_from_user(tsk, &sc->fp_regs, efault_out); 405 388 #ifdef CONFIG_VSX 406 389 /* 407 390 * Get additional VSX data. Update v_regs to point after the ··· 408 395 */ 409 396 v_regs += ELF_NVRREG; 410 397 if ((msr & MSR_VSX) != 0) { 411 - err |= copy_vsx_from_user(tsk, v_regs); 398 + unsafe_copy_vsx_from_user(tsk, v_regs, efault_out); 412 399 tsk->thread.used_vsr = true; 413 400 } else { 414 401 for (i = 0; i < 32 ; i++) 415 402 tsk->thread.fp_state.fpr[i][TS_VSRLOWOFFSET] = 0; 416 403 } 417 404 #endif 418 - return err; 405 + return 0; 406 + 407 + efault_out: 408 + return -EFAULT; 419 409 } 420 410 421 411 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM ··· 602 586 603 587 return err; 604 588 } 589 + #else /* !CONFIG_PPC_TRANSACTIONAL_MEM */ 590 + static long restore_tm_sigcontexts(struct task_struct *tsk, struct sigcontext __user *sc, 591 + struct sigcontext __user *tm_sc) 592 + { 593 + return -EINVAL; 594 + } 605 595 #endif 606 596 607 597 /* ··· 677 655 ctx_has_vsx_region = 1; 678 656 679 657 if (old_ctx != NULL) { 680 - if (!access_ok(old_ctx, ctx_size) 681 - || setup_sigcontext(&old_ctx->uc_mcontext, current, 0, NULL, 0, 682 - ctx_has_vsx_region) 683 - || __copy_to_user(&old_ctx->uc_sigmask, 684 - &current->blocked, sizeof(sigset_t))) 658 + prepare_setup_sigcontext(current); 659 + if (!user_write_access_begin(old_ctx, ctx_size)) 685 660 return -EFAULT; 661 + 662 + unsafe_setup_sigcontext(&old_ctx->uc_mcontext, current, 0, NULL, 663 + 0, ctx_has_vsx_region, efault_out); 664 + unsafe_copy_to_user(&old_ctx->uc_sigmask, &current->blocked, 665 + sizeof(sigset_t), efault_out); 666 + 667 + user_write_access_end(); 686 668 } 687 669 if (new_ctx == NULL) 688 670 return 0; ··· 706 680 * We kill the task with a SIGSEGV in this situation. 707 681 */ 708 682 709 - if (__copy_from_user(&set, &new_ctx->uc_sigmask, sizeof(set))) 683 + if (__get_user_sigset(&set, &new_ctx->uc_sigmask)) 710 684 do_exit(SIGSEGV); 711 685 set_current_blocked(&set); 712 - if (restore_sigcontext(current, NULL, 0, &new_ctx->uc_mcontext)) 686 + 687 + if (!user_read_access_begin(new_ctx, ctx_size)) 688 + return -EFAULT; 689 + if (__unsafe_restore_sigcontext(current, NULL, 0, &new_ctx->uc_mcontext)) { 690 + user_read_access_end(); 713 691 do_exit(SIGSEGV); 692 + } 693 + user_read_access_end(); 714 694 715 695 /* This returns like rt_sigreturn */ 716 696 set_thread_flag(TIF_RESTOREALL); 717 697 return 0; 698 + 699 + efault_out: 700 + user_write_access_end(); 701 + return -EFAULT; 718 702 } 719 703 720 704 ··· 737 701 struct pt_regs *regs = current_pt_regs(); 738 702 struct ucontext __user *uc = (struct ucontext __user *)regs->gpr[1]; 739 703 sigset_t set; 740 - #ifdef CONFIG_PPC_TRANSACTIONAL_MEM 741 704 unsigned long msr; 742 - #endif 743 705 744 706 /* Always make any pending restarted system calls return -EINTR */ 745 707 current->restart_block.fn = do_no_restart_syscall; ··· 745 711 if (!access_ok(uc, sizeof(*uc))) 746 712 goto badframe; 747 713 748 - if (__copy_from_user(&set, &uc->uc_sigmask, sizeof(set))) 714 + if (__get_user_sigset(&set, &uc->uc_sigmask)) 749 715 goto badframe; 750 716 set_current_blocked(&set); 751 717 752 - #ifdef CONFIG_PPC_TRANSACTIONAL_MEM 753 - /* 754 - * If there is a transactional state then throw it away. 755 - * The purpose of a sigreturn is to destroy all traces of the 756 - * signal frame, this includes any transactional state created 757 - * within in. We only check for suspended as we can never be 758 - * active in the kernel, we are active, there is nothing better to 759 - * do than go ahead and Bad Thing later. 760 - * The cause is not important as there will never be a 761 - * recheckpoint so it's not user visible. 762 - */ 763 - if (MSR_TM_SUSPENDED(mfmsr())) 764 - tm_reclaim_current(0); 718 + if (IS_ENABLED(CONFIG_PPC_TRANSACTIONAL_MEM)) { 719 + /* 720 + * If there is a transactional state then throw it away. 721 + * The purpose of a sigreturn is to destroy all traces of the 722 + * signal frame, this includes any transactional state created 723 + * within in. We only check for suspended as we can never be 724 + * active in the kernel, we are active, there is nothing better to 725 + * do than go ahead and Bad Thing later. 726 + * The cause is not important as there will never be a 727 + * recheckpoint so it's not user visible. 728 + */ 729 + if (MSR_TM_SUSPENDED(mfmsr())) 730 + tm_reclaim_current(0); 765 731 766 - /* 767 - * Disable MSR[TS] bit also, so, if there is an exception in the 768 - * code below (as a page fault in copy_ckvsx_to_user()), it does 769 - * not recheckpoint this task if there was a context switch inside 770 - * the exception. 771 - * 772 - * A major page fault can indirectly call schedule(). A reschedule 773 - * process in the middle of an exception can have a side effect 774 - * (Changing the CPU MSR[TS] state), since schedule() is called 775 - * with the CPU MSR[TS] disable and returns with MSR[TS]=Suspended 776 - * (switch_to() calls tm_recheckpoint() for the 'new' process). In 777 - * this case, the process continues to be the same in the CPU, but 778 - * the CPU state just changed. 779 - * 780 - * This can cause a TM Bad Thing, since the MSR in the stack will 781 - * have the MSR[TS]=0, and this is what will be used to RFID. 782 - * 783 - * Clearing MSR[TS] state here will avoid a recheckpoint if there 784 - * is any process reschedule in kernel space. The MSR[TS] state 785 - * does not need to be saved also, since it will be replaced with 786 - * the MSR[TS] that came from user context later, at 787 - * restore_tm_sigcontexts. 788 - */ 789 - regs->msr &= ~MSR_TS_MASK; 732 + /* 733 + * Disable MSR[TS] bit also, so, if there is an exception in the 734 + * code below (as a page fault in copy_ckvsx_to_user()), it does 735 + * not recheckpoint this task if there was a context switch inside 736 + * the exception. 737 + * 738 + * A major page fault can indirectly call schedule(). A reschedule 739 + * process in the middle of an exception can have a side effect 740 + * (Changing the CPU MSR[TS] state), since schedule() is called 741 + * with the CPU MSR[TS] disable and returns with MSR[TS]=Suspended 742 + * (switch_to() calls tm_recheckpoint() for the 'new' process). In 743 + * this case, the process continues to be the same in the CPU, but 744 + * the CPU state just changed. 745 + * 746 + * This can cause a TM Bad Thing, since the MSR in the stack will 747 + * have the MSR[TS]=0, and this is what will be used to RFID. 748 + * 749 + * Clearing MSR[TS] state here will avoid a recheckpoint if there 750 + * is any process reschedule in kernel space. The MSR[TS] state 751 + * does not need to be saved also, since it will be replaced with 752 + * the MSR[TS] that came from user context later, at 753 + * restore_tm_sigcontexts. 754 + */ 755 + regs->msr &= ~MSR_TS_MASK; 790 756 791 - if (__get_user(msr, &uc->uc_mcontext.gp_regs[PT_MSR])) 792 - goto badframe; 793 - if (MSR_TM_ACTIVE(msr)) { 757 + if (__get_user(msr, &uc->uc_mcontext.gp_regs[PT_MSR])) 758 + goto badframe; 759 + } 760 + 761 + if (IS_ENABLED(CONFIG_PPC_TRANSACTIONAL_MEM) && MSR_TM_ACTIVE(msr)) { 794 762 /* We recheckpoint on return. */ 795 763 struct ucontext __user *uc_transact; 796 764 ··· 805 769 if (restore_tm_sigcontexts(current, &uc->uc_mcontext, 806 770 &uc_transact->uc_mcontext)) 807 771 goto badframe; 808 - } else 809 - #endif 810 - { 772 + } else { 811 773 /* 812 774 * Fall through, for non-TM restore 813 775 * ··· 819 785 * causing a TM bad thing. 820 786 */ 821 787 current->thread.regs->msr &= ~MSR_TS_MASK; 822 - if (restore_sigcontext(current, NULL, 1, &uc->uc_mcontext)) 788 + if (!user_read_access_begin(&uc->uc_mcontext, sizeof(uc->uc_mcontext))) 823 789 goto badframe; 790 + 791 + unsafe_restore_sigcontext(current, NULL, 1, &uc->uc_mcontext, 792 + badframe_block); 793 + 794 + user_read_access_end(); 824 795 } 825 796 826 797 if (restore_altstack(&uc->uc_stack)) ··· 834 795 set_thread_flag(TIF_RESTOREALL); 835 796 return 0; 836 797 798 + badframe_block: 799 + user_read_access_end(); 837 800 badframe: 838 801 signal_fault(current, regs, "rt_sigreturn", uc); 839 802 ··· 850 809 unsigned long newsp = 0; 851 810 long err = 0; 852 811 struct pt_regs *regs = tsk->thread.regs; 853 - #ifdef CONFIG_PPC_TRANSACTIONAL_MEM 854 812 /* Save the thread's msr before get_tm_stackpointer() changes it */ 855 813 unsigned long msr = regs->msr; 856 - #endif 857 814 858 815 frame = get_sigframe(ksig, tsk, sizeof(*frame), 0); 859 - if (!access_ok(frame, sizeof(*frame))) 816 + 817 + /* 818 + * This only applies when calling unsafe_setup_sigcontext() and must be 819 + * called before opening the uaccess window. 820 + */ 821 + if (!MSR_TM_ACTIVE(msr)) 822 + prepare_setup_sigcontext(tsk); 823 + 824 + if (!user_write_access_begin(frame, sizeof(*frame))) 860 825 goto badframe; 861 826 862 - err |= __put_user(&frame->info, &frame->pinfo); 863 - err |= __put_user(&frame->uc, &frame->puc); 864 - err |= copy_siginfo_to_user(&frame->info, &ksig->info); 865 - if (err) 866 - goto badframe; 827 + unsafe_put_user(&frame->info, &frame->pinfo, badframe_block); 828 + unsafe_put_user(&frame->uc, &frame->puc, badframe_block); 867 829 868 830 /* Create the ucontext. */ 869 - err |= __put_user(0, &frame->uc.uc_flags); 870 - err |= __save_altstack(&frame->uc.uc_stack, regs->gpr[1]); 871 - #ifdef CONFIG_PPC_TRANSACTIONAL_MEM 831 + unsafe_put_user(0, &frame->uc.uc_flags, badframe_block); 832 + unsafe_save_altstack(&frame->uc.uc_stack, regs->gpr[1], badframe_block); 833 + 872 834 if (MSR_TM_ACTIVE(msr)) { 835 + #ifdef CONFIG_PPC_TRANSACTIONAL_MEM 873 836 /* The ucontext_t passed to userland points to the second 874 837 * ucontext_t (for transactional state) with its uc_link ptr. 875 838 */ 876 - err |= __put_user(&frame->uc_transact, &frame->uc.uc_link); 839 + unsafe_put_user(&frame->uc_transact, &frame->uc.uc_link, badframe_block); 840 + 841 + user_write_access_end(); 842 + 877 843 err |= setup_tm_sigcontexts(&frame->uc.uc_mcontext, 878 844 &frame->uc_transact.uc_mcontext, 879 845 tsk, ksig->sig, NULL, 880 846 (unsigned long)ksig->ka.sa.sa_handler, 881 847 msr); 882 - } else 848 + 849 + if (!user_write_access_begin(&frame->uc.uc_sigmask, 850 + sizeof(frame->uc.uc_sigmask))) 851 + goto badframe; 852 + 883 853 #endif 884 - { 885 - err |= __put_user(0, &frame->uc.uc_link); 886 - err |= setup_sigcontext(&frame->uc.uc_mcontext, tsk, ksig->sig, 854 + } else { 855 + unsafe_put_user(0, &frame->uc.uc_link, badframe_block); 856 + unsafe_setup_sigcontext(&frame->uc.uc_mcontext, tsk, ksig->sig, 887 857 NULL, (unsigned long)ksig->ka.sa.sa_handler, 888 - 1); 858 + 1, badframe_block); 889 859 } 890 - err |= __copy_to_user(&frame->uc.uc_sigmask, set, sizeof(*set)); 891 - if (err) 892 - goto badframe; 860 + 861 + unsafe_copy_to_user(&frame->uc.uc_sigmask, set, sizeof(*set), badframe_block); 862 + user_write_access_end(); 893 863 894 864 /* Make sure signal handler doesn't get spurious FP exceptions */ 895 865 tsk->thread.fp_state.fpscr = 0; ··· 914 862 goto badframe; 915 863 regs->nip = (unsigned long) &frame->tramp[0]; 916 864 } 865 + 866 + 867 + /* Save the siginfo outside of the unsafe block. */ 868 + if (copy_siginfo_to_user(&frame->info, &ksig->info)) 869 + goto badframe; 917 870 918 871 /* Allocate a dummy caller frame for the signal handler. */ 919 872 newsp = ((unsigned long)frame) - __SIGNAL_FRAMESIZE; ··· 959 902 960 903 return 0; 961 904 905 + badframe_block: 906 + user_write_access_end(); 962 907 badframe: 963 908 signal_fault(current, regs, "handle_rt_signal64", frame); 964 909

+55 -13

arch/powerpc/kernel/smp.c

··· 83 83 DEFINE_PER_CPU(cpumask_var_t, cpu_smallcore_map); 84 84 DEFINE_PER_CPU(cpumask_var_t, cpu_l2_cache_map); 85 85 DEFINE_PER_CPU(cpumask_var_t, cpu_core_map); 86 - DEFINE_PER_CPU(cpumask_var_t, cpu_coregroup_map); 86 + static DEFINE_PER_CPU(cpumask_var_t, cpu_coregroup_map); 87 87 88 88 EXPORT_PER_CPU_SYMBOL(cpu_sibling_map); 89 89 EXPORT_PER_CPU_SYMBOL(cpu_l2_cache_map); ··· 122 122 * On big-cores system, thread_group_l1_cache_map for each CPU corresponds to 123 123 * the set its siblings that share the L1-cache. 124 124 */ 125 - DEFINE_PER_CPU(cpumask_var_t, thread_group_l1_cache_map); 125 + static DEFINE_PER_CPU(cpumask_var_t, thread_group_l1_cache_map); 126 126 127 127 /* 128 128 * On some big-cores system, thread_group_l2_cache_map for each CPU 129 129 * corresponds to the set its siblings within the core that share the 130 130 * L2-cache. 131 131 */ 132 - DEFINE_PER_CPU(cpumask_var_t, thread_group_l2_cache_map); 132 + static DEFINE_PER_CPU(cpumask_var_t, thread_group_l2_cache_map); 133 133 134 134 /* SMP operations for this machine */ 135 135 struct smp_ops_t *smp_ops; ··· 1057 1057 local_memory_node(numa_cpu_lookup_table[cpu])); 1058 1058 } 1059 1059 #endif 1060 - /* 1061 - * cpu_core_map is now more updated and exists only since 1062 - * its been exported for long. It only will have a snapshot 1063 - * of cpu_cpu_mask. 1064 - */ 1065 - cpumask_copy(per_cpu(cpu_core_map, cpu), cpu_cpu_mask(cpu)); 1066 1060 } 1067 1061 1068 1062 /* Init the cpumasks so the boot CPU is related to itself */ 1069 1063 cpumask_set_cpu(boot_cpuid, cpu_sibling_mask(boot_cpuid)); 1070 1064 cpumask_set_cpu(boot_cpuid, cpu_l2_cache_mask(boot_cpuid)); 1065 + cpumask_set_cpu(boot_cpuid, cpu_core_mask(boot_cpuid)); 1071 1066 1072 1067 if (has_coregroup_support()) 1073 1068 cpumask_set_cpu(boot_cpuid, cpu_coregroup_mask(boot_cpuid)); ··· 1071 1076 if (has_big_cores) { 1072 1077 cpumask_set_cpu(boot_cpuid, 1073 1078 cpu_smallcore_mask(boot_cpuid)); 1079 + } 1080 + 1081 + if (cpu_to_chip_id(boot_cpuid) != -1) { 1082 + int idx = num_possible_cpus() / threads_per_core; 1083 + 1084 + /* 1085 + * All threads of a core will all belong to the same core, 1086 + * chip_id_lookup_table will have one entry per core. 1087 + * Assumption: if boot_cpuid doesn't have a chip-id, then no 1088 + * other CPUs, will also not have chip-id. 1089 + */ 1090 + chip_id_lookup_table = kcalloc(idx, sizeof(int), GFP_KERNEL); 1091 + if (chip_id_lookup_table) 1092 + memset(chip_id_lookup_table, -1, sizeof(int) * idx); 1074 1093 } 1075 1094 1076 1095 if (smp_ops && smp_ops->probe) ··· 1417 1408 set_cpus_unrelated(cpu, i, cpu_smallcore_mask); 1418 1409 } 1419 1410 1411 + for_each_cpu(i, cpu_core_mask(cpu)) 1412 + set_cpus_unrelated(cpu, i, cpu_core_mask); 1413 + 1420 1414 if (has_coregroup_support()) { 1421 1415 for_each_cpu(i, cpu_coregroup_mask(cpu)) 1422 1416 set_cpus_unrelated(cpu, i, cpu_coregroup_mask); ··· 1480 1468 1481 1469 static void add_cpu_to_masks(int cpu) 1482 1470 { 1471 + struct cpumask *(*submask_fn)(int) = cpu_sibling_mask; 1483 1472 int first_thread = cpu_first_thread_sibling(cpu); 1484 1473 cpumask_var_t mask; 1474 + int chip_id = -1; 1475 + bool ret; 1485 1476 int i; 1486 1477 1487 1478 /* ··· 1500 1485 add_cpu_to_smallcore_masks(cpu); 1501 1486 1502 1487 /* In CPU-hotplug path, hence use GFP_ATOMIC */ 1503 - alloc_cpumask_var_node(&mask, GFP_ATOMIC, cpu_to_node(cpu)); 1488 + ret = alloc_cpumask_var_node(&mask, GFP_ATOMIC, cpu_to_node(cpu)); 1504 1489 update_mask_by_l2(cpu, &mask); 1505 1490 1506 1491 if (has_coregroup_support()) 1507 1492 update_coregroup_mask(cpu, &mask); 1508 1493 1494 + if (chip_id_lookup_table && ret) 1495 + chip_id = cpu_to_chip_id(cpu); 1496 + 1497 + if (chip_id == -1) { 1498 + cpumask_copy(per_cpu(cpu_core_map, cpu), cpu_cpu_mask(cpu)); 1499 + goto out; 1500 + } 1501 + 1502 + if (shared_caches) 1503 + submask_fn = cpu_l2_cache_mask; 1504 + 1505 + /* Update core_mask with all the CPUs that are part of submask */ 1506 + or_cpumasks_related(cpu, cpu, submask_fn, cpu_core_mask); 1507 + 1508 + /* Skip all CPUs already part of current CPU core mask */ 1509 + cpumask_andnot(mask, cpu_online_mask, cpu_core_mask(cpu)); 1510 + 1511 + for_each_cpu(i, mask) { 1512 + if (chip_id == cpu_to_chip_id(i)) { 1513 + or_cpumasks_related(cpu, i, submask_fn, cpu_core_mask); 1514 + cpumask_andnot(mask, mask, submask_fn(i)); 1515 + } else { 1516 + cpumask_andnot(mask, mask, cpu_core_mask(i)); 1517 + } 1518 + } 1519 + 1520 + out: 1509 1521 free_cpumask_var(mask); 1510 1522 } 1511 1523 ··· 1563 1521 1564 1522 vdso_getcpu_init(); 1565 1523 #endif 1524 + set_numa_node(numa_cpu_lookup_table[cpu]); 1525 + set_numa_mem(local_memory_node(numa_cpu_lookup_table[cpu])); 1526 + 1566 1527 /* Update topology CPU masks */ 1567 1528 add_cpu_to_masks(cpu); 1568 1529 ··· 1583 1538 if (cpumask_weight(mask) > cpumask_weight(sibling_mask(cpu))) 1584 1539 shared_caches = true; 1585 1540 } 1586 - 1587 - set_numa_node(numa_cpu_lookup_table[cpu]); 1588 - set_numa_mem(local_memory_node(numa_cpu_lookup_table[cpu])); 1589 1541 1590 1542 smp_wmb(); 1591 1543 notify_cpu_starting(cpu);

+25 -83

arch/powerpc/kernel/stacktrace.c

··· 23 23 24 24 #include <asm/paca.h> 25 25 26 - /* 27 - * Save stack-backtrace addresses into a stack_trace buffer. 28 - */ 29 - static void save_context_stack(struct stack_trace *trace, unsigned long sp, 30 - struct task_struct *tsk, int savesched) 26 + void arch_stack_walk(stack_trace_consume_fn consume_entry, void *cookie, 27 + struct task_struct *task, struct pt_regs *regs) 31 28 { 29 + unsigned long sp; 30 + 31 + if (regs && !consume_entry(cookie, regs->nip)) 32 + return; 33 + 34 + if (regs) 35 + sp = regs->gpr[1]; 36 + else if (task == current) 37 + sp = current_stack_frame(); 38 + else 39 + sp = task->thread.ksp; 40 + 32 41 for (;;) { 33 42 unsigned long *stack = (unsigned long *) sp; 34 43 unsigned long newsp, ip; 35 44 36 - if (!validate_sp(sp, tsk, STACK_FRAME_OVERHEAD)) 45 + if (!validate_sp(sp, task, STACK_FRAME_OVERHEAD)) 37 46 return; 38 47 39 48 newsp = stack[0]; 40 49 ip = stack[STACK_FRAME_LR_SAVE]; 41 50 42 - if (savesched || !in_sched_functions(ip)) { 43 - if (!trace->skip) 44 - trace->entries[trace->nr_entries++] = ip; 45 - else 46 - trace->skip--; 47 - } 48 - 49 - if (trace->nr_entries >= trace->max_entries) 51 + if (!consume_entry(cookie, ip)) 50 52 return; 51 53 52 54 sp = newsp; 53 55 } 54 56 } 55 57 56 - void save_stack_trace(struct stack_trace *trace) 57 - { 58 - unsigned long sp; 59 - 60 - sp = current_stack_frame(); 61 - 62 - save_context_stack(trace, sp, current, 1); 63 - } 64 - EXPORT_SYMBOL_GPL(save_stack_trace); 65 - 66 - void save_stack_trace_tsk(struct task_struct *tsk, struct stack_trace *trace) 67 - { 68 - unsigned long sp; 69 - 70 - if (!try_get_task_stack(tsk)) 71 - return; 72 - 73 - if (tsk == current) 74 - sp = current_stack_frame(); 75 - else 76 - sp = tsk->thread.ksp; 77 - 78 - save_context_stack(trace, sp, tsk, 0); 79 - 80 - put_task_stack(tsk); 81 - } 82 - EXPORT_SYMBOL_GPL(save_stack_trace_tsk); 83 - 84 - void 85 - save_stack_trace_regs(struct pt_regs *regs, struct stack_trace *trace) 86 - { 87 - save_context_stack(trace, regs->gpr[1], current, 0); 88 - } 89 - EXPORT_SYMBOL_GPL(save_stack_trace_regs); 90 - 91 - #ifdef CONFIG_HAVE_RELIABLE_STACKTRACE 92 58 /* 93 59 * This function returns an error if it detects any unreliable features of the 94 60 * stack. Otherwise it guarantees that the stack trace is reliable. 95 61 * 96 62 * If the task is not 'current', the caller *must* ensure the task is inactive. 97 63 */ 98 - static int __save_stack_trace_tsk_reliable(struct task_struct *tsk, 99 - struct stack_trace *trace) 64 + int arch_stack_walk_reliable(stack_trace_consume_fn consume_entry, 65 + void *cookie, struct task_struct *task) 100 66 { 101 67 unsigned long sp; 102 68 unsigned long newsp; 103 - unsigned long stack_page = (unsigned long)task_stack_page(tsk); 69 + unsigned long stack_page = (unsigned long)task_stack_page(task); 104 70 unsigned long stack_end; 105 71 int graph_idx = 0; 106 72 bool firstframe; 107 73 108 74 stack_end = stack_page + THREAD_SIZE; 109 - if (!is_idle_task(tsk)) { 75 + if (!is_idle_task(task)) { 110 76 /* 111 77 * For user tasks, this is the SP value loaded on 112 78 * kernel entry, see "PACAKSAVE(r13)" in _switch() and ··· 96 130 stack_end -= STACK_FRAME_OVERHEAD; 97 131 } 98 132 99 - if (tsk == current) 133 + if (task == current) 100 134 sp = current_stack_frame(); 101 135 else 102 - sp = tsk->thread.ksp; 136 + sp = task->thread.ksp; 103 137 104 138 if (sp < stack_page + sizeof(struct thread_struct) || 105 139 sp > stack_end - STACK_FRAME_MIN_SIZE) { ··· 148 182 * FIXME: IMHO these tests do not belong in 149 183 * arch-dependent code, they are generic. 150 184 */ 151 - ip = ftrace_graph_ret_addr(tsk, &graph_idx, ip, stack); 185 + ip = ftrace_graph_ret_addr(task, &graph_idx, ip, stack); 152 186 #ifdef CONFIG_KPROBES 153 187 /* 154 188 * Mark stacktraces with kretprobed functions on them ··· 158 192 return -EINVAL; 159 193 #endif 160 194 161 - if (trace->nr_entries >= trace->max_entries) 162 - return -E2BIG; 163 - if (!trace->skip) 164 - trace->entries[trace->nr_entries++] = ip; 165 - else 166 - trace->skip--; 195 + if (!consume_entry(cookie, ip)) 196 + return -EINVAL; 167 197 } 168 198 return 0; 169 199 } 170 - 171 - int save_stack_trace_tsk_reliable(struct task_struct *tsk, 172 - struct stack_trace *trace) 173 - { 174 - int ret; 175 - 176 - /* 177 - * If the task doesn't have a stack (e.g., a zombie), the stack is 178 - * "reliably" empty. 179 - */ 180 - if (!try_get_task_stack(tsk)) 181 - return 0; 182 - 183 - ret = __save_stack_trace_tsk_reliable(tsk, trace); 184 - 185 - put_task_stack(tsk); 186 - 187 - return ret; 188 - } 189 - #endif /* CONFIG_HAVE_RELIABLE_STACKTRACE */ 190 200 191 201 #if defined(CONFIG_PPC_BOOK3S_64) && defined(CONFIG_NMI_IPI) 192 202 static void handle_backtrace_ipi(struct pt_regs *regs)

+2 -10

arch/powerpc/kernel/syscalls.c

··· 82 82 ppc_select(int n, fd_set __user *inp, fd_set __user *outp, fd_set __user *exp, struct __kernel_old_timeval __user *tvp) 83 83 { 84 84 if ( (unsigned long)n >= 4096 ) 85 - { 86 - unsigned long __user *buffer = (unsigned long __user *)n; 87 - if (!access_ok(buffer, 5*sizeof(unsigned long)) 88 - || __get_user(n, buffer) 89 - || __get_user(inp, ((fd_set __user * __user *)(buffer+1))) 90 - || __get_user(outp, ((fd_set __user * __user *)(buffer+2))) 91 - || __get_user(exp, ((fd_set __user * __user *)(buffer+3))) 92 - || __get_user(tvp, ((struct __kernel_old_timeval __user * __user *)(buffer+4)))) 93 - return -EFAULT; 94 - } 85 + return sys_old_select((void __user *)n); 86 + 95 87 return sys_select(n, inp, outp, exp, tvp); 96 88 } 97 89 #endif

+9 -24

arch/powerpc/kernel/syscalls/Makefile

··· 6 6 $(shell [ -d '$(kapi)' ] || mkdir -p '$(kapi)') 7 7 8 8 syscall := $(src)/syscall.tbl 9 - syshdr := $(srctree)/$(src)/syscallhdr.sh 10 - systbl := $(srctree)/$(src)/syscalltbl.sh 9 + syshdr := $(srctree)/scripts/syscallhdr.sh 10 + systbl := $(srctree)/scripts/syscalltbl.sh 11 11 12 12 quiet_cmd_syshdr = SYSHDR $@ 13 - cmd_syshdr = $(CONFIG_SHELL) '$(syshdr)' '$<' '$@' \ 14 - '$(syshdr_abis_$(basetarget))' \ 15 - '$(syshdr_pfx_$(basetarget))' \ 16 - '$(syshdr_offset_$(basetarget))' 13 + cmd_syshdr = $(CONFIG_SHELL) $(syshdr) --emit-nr --abis $(abis) $< $@ 17 14 18 15 quiet_cmd_systbl = SYSTBL $@ 19 - cmd_systbl = $(CONFIG_SHELL) '$(systbl)' '$<' '$@' \ 20 - '$(systbl_abis_$(basetarget))' \ 21 - '$(systbl_abi_$(basetarget))' \ 22 - '$(systbl_offset_$(basetarget))' 16 + cmd_systbl = $(CONFIG_SHELL) $(systbl) --abis $(abis) $< $@ 23 17 24 - syshdr_abis_unistd_32 := common,nospu,32 18 + $(uapi)/unistd_32.h: abis := common,nospu,32 25 19 $(uapi)/unistd_32.h: $(syscall) $(syshdr) FORCE 26 20 $(call if_changed,syshdr) 27 21 28 - syshdr_abis_unistd_64 := common,nospu,64 22 + $(uapi)/unistd_64.h: abis := common,nospu,64 29 23 $(uapi)/unistd_64.h: $(syscall) $(syshdr) FORCE 30 24 $(call if_changed,syshdr) 31 25 32 - systbl_abis_syscall_table_32 := common,nospu,32 33 - systbl_abi_syscall_table_32 := 32 26 + $(kapi)/syscall_table_32.h: abis := common,nospu,32 34 27 $(kapi)/syscall_table_32.h: $(syscall) $(systbl) FORCE 35 28 $(call if_changed,systbl) 36 29 37 - systbl_abis_syscall_table_64 := common,nospu,64 38 - systbl_abi_syscall_table_64 := 64 30 + $(kapi)/syscall_table_64.h: abis := common,nospu,64 39 31 $(kapi)/syscall_table_64.h: $(syscall) $(systbl) FORCE 40 32 $(call if_changed,systbl) 41 33 42 - systbl_abis_syscall_table_c32 := common,nospu,32 43 - systbl_abi_syscall_table_c32 := c32 44 - $(kapi)/syscall_table_c32.h: $(syscall) $(systbl) FORCE 45 - $(call if_changed,systbl) 46 - 47 - systbl_abis_syscall_table_spu := common,spu 48 - systbl_abi_syscall_table_spu := spu 34 + $(kapi)/syscall_table_spu.h: abis := common,spu 49 35 $(kapi)/syscall_table_spu.h: $(syscall) $(systbl) FORCE 50 36 $(call if_changed,systbl) 51 37 52 38 uapisyshdr-y += unistd_32.h unistd_64.h 53 39 kapisyshdr-y += syscall_table_32.h \ 54 40 syscall_table_64.h \ 55 - syscall_table_c32.h \ 56 41 syscall_table_spu.h 57 42 58 43 uapisyshdr-y := $(addprefix $(uapi)/, $(uapisyshdr-y))

-36

arch/powerpc/kernel/syscalls/syscallhdr.sh

··· 1 - #!/bin/sh 2 - # SPDX-License-Identifier: GPL-2.0 3 - 4 - in="$1" 5 - out="$2" 6 - my_abis=`echo "($3)" | tr ',' '|'` 7 - prefix="$4" 8 - offset="$5" 9 - 10 - fileguard=_UAPI_ASM_POWERPC_`basename "$out" | sed \ 11 - -e 'y/abcdefghijklmnopqrstuvwxyz/ABCDEFGHIJKLMNOPQRSTUVWXYZ/' \ 12 - -e 's/[^A-Z0-9_]/_/g' -e 's/__/_/g'` 13 - grep -E "^[0-9A-Fa-fXx]+[[:space:]]+${my_abis}" "$in" | sort -n | ( 14 - printf "#ifndef %s\n" "${fileguard}" 15 - printf "#define %s\n" "${fileguard}" 16 - printf "\n" 17 - 18 - nxt=0 19 - while read nr abi name entry compat ; do 20 - if [ -z "$offset" ]; then 21 - printf "#define __NR_%s%s\t%s\n" \ 22 - "${prefix}" "${name}" "${nr}" 23 - else 24 - printf "#define __NR_%s%s\t(%s + %s)\n" \ 25 - "${prefix}" "${name}" "${offset}" "${nr}" 26 - fi 27 - nxt=$((nr+1)) 28 - done 29 - 30 - printf "\n" 31 - printf "#ifdef __KERNEL__\n" 32 - printf "#define __NR_syscalls\t%s\n" "${nxt}" 33 - printf "#endif\n" 34 - printf "\n" 35 - printf "#endif /* %s */\n" "${fileguard}" 36 - ) > "$out"

-36

arch/powerpc/kernel/syscalls/syscalltbl.sh

··· 1 - #!/bin/sh 2 - # SPDX-License-Identifier: GPL-2.0 3 - 4 - in="$1" 5 - out="$2" 6 - my_abis=`echo "($3)" | tr ',' '|'` 7 - my_abi="$4" 8 - offset="$5" 9 - 10 - emit() { 11 - t_nxt="$1" 12 - t_nr="$2" 13 - t_entry="$3" 14 - 15 - while [ $t_nxt -lt $t_nr ]; do 16 - printf "__SYSCALL(%s,sys_ni_syscall)\n" "${t_nxt}" 17 - t_nxt=$((t_nxt+1)) 18 - done 19 - printf "__SYSCALL(%s,%s)\n" "${t_nxt}" "${t_entry}" 20 - } 21 - 22 - grep -E "^[0-9A-Fa-fXx]+[[:space:]]+${my_abis}" "$in" | sort -n | ( 23 - nxt=0 24 - if [ -z "$offset" ]; then 25 - offset=0 26 - fi 27 - 28 - while read nr abi name entry compat ; do 29 - if [ "$my_abi" = "c32" ] && [ ! -z "$compat" ]; then 30 - emit $((nxt+offset)) $((nr+offset)) $compat 31 - else 32 - emit $((nxt+offset)) $((nr+offset)) $entry 33 - fi 34 - nxt=$((nr+1)) 35 - done 36 - ) > "$out"

+4 -1

arch/powerpc/kernel/systbl.S

··· 21 21 #define __SYSCALL(nr, entry) .long entry 22 22 #endif 23 23 24 + #define __SYSCALL_WITH_COMPAT(nr, native, compat) __SYSCALL(nr, native) 24 25 .globl sys_call_table 25 26 sys_call_table: 26 27 #ifdef CONFIG_PPC64 ··· 31 30 #endif 32 31 33 32 #ifdef CONFIG_COMPAT 33 + #undef __SYSCALL_WITH_COMPAT 34 + #define __SYSCALL_WITH_COMPAT(nr, native, compat) __SYSCALL(nr, compat) 34 35 .globl compat_sys_call_table 35 36 compat_sys_call_table: 36 37 #define compat_sys_sigsuspend sys_sigsuspend 37 - #include <asm/syscall_table_c32.h> 38 + #include <asm/syscall_table_32.h> 38 39 #endif

+11 -11

arch/powerpc/kernel/trace/ftrace.c

··· 68 68 */ 69 69 70 70 /* read the text we want to modify */ 71 - if (probe_kernel_read_inst(&replaced, (void *)ip)) 71 + if (copy_inst_from_kernel_nofault(&replaced, (void *)ip)) 72 72 return -EFAULT; 73 73 74 74 /* Make sure it is what we expect it to be */ ··· 130 130 struct ppc_inst op, pop; 131 131 132 132 /* read where this goes */ 133 - if (probe_kernel_read_inst(&op, (void *)ip)) { 133 + if (copy_inst_from_kernel_nofault(&op, (void *)ip)) { 134 134 pr_err("Fetching opcode failed.\n"); 135 135 return -EFAULT; 136 136 } ··· 164 164 /* When using -mkernel_profile there is no load to jump over */ 165 165 pop = ppc_inst(PPC_INST_NOP); 166 166 167 - if (probe_kernel_read_inst(&op, (void *)(ip - 4))) { 167 + if (copy_inst_from_kernel_nofault(&op, (void *)(ip - 4))) { 168 168 pr_err("Fetching instruction at %lx failed.\n", ip - 4); 169 169 return -EFAULT; 170 170 } ··· 197 197 * Check what is in the next instruction. We can see ld r2,40(r1), but 198 198 * on first pass after boot we will see mflr r0. 199 199 */ 200 - if (probe_kernel_read_inst(&op, (void *)(ip + 4))) { 200 + if (copy_inst_from_kernel_nofault(&op, (void *)(ip + 4))) { 201 201 pr_err("Fetching op failed.\n"); 202 202 return -EFAULT; 203 203 } ··· 349 349 return -1; 350 350 351 351 /* New trampoline -- read where this goes */ 352 - if (probe_kernel_read_inst(&op, (void *)tramp)) { 352 + if (copy_inst_from_kernel_nofault(&op, (void *)tramp)) { 353 353 pr_debug("Fetching opcode failed.\n"); 354 354 return -1; 355 355 } ··· 399 399 struct ppc_inst op; 400 400 401 401 /* Read where this goes */ 402 - if (probe_kernel_read_inst(&op, (void *)ip)) { 402 + if (copy_inst_from_kernel_nofault(&op, (void *)ip)) { 403 403 pr_err("Fetching opcode failed.\n"); 404 404 return -EFAULT; 405 405 } ··· 526 526 struct module *mod = rec->arch.mod; 527 527 528 528 /* read where this goes */ 529 - if (probe_kernel_read_inst(op, ip)) 529 + if (copy_inst_from_kernel_nofault(op, ip)) 530 530 return -EFAULT; 531 531 532 - if (probe_kernel_read_inst(op + 1, ip + 4)) 532 + if (copy_inst_from_kernel_nofault(op + 1, ip + 4)) 533 533 return -EFAULT; 534 534 535 535 if (!expected_nop_sequence(ip, op[0], op[1])) { ··· 592 592 unsigned long ip = rec->ip; 593 593 594 594 /* read where this goes */ 595 - if (probe_kernel_read_inst(&op, (void *)ip)) 595 + if (copy_inst_from_kernel_nofault(&op, (void *)ip)) 596 596 return -EFAULT; 597 597 598 598 /* It should be pointing to a nop */ ··· 648 648 } 649 649 650 650 /* Make sure we have a nop */ 651 - if (probe_kernel_read_inst(&op, ip)) { 651 + if (copy_inst_from_kernel_nofault(&op, ip)) { 652 652 pr_err("Unable to read ftrace location %p\n", ip); 653 653 return -EFAULT; 654 654 } ··· 726 726 } 727 727 728 728 /* read where this goes */ 729 - if (probe_kernel_read_inst(&op, (void *)ip)) { 729 + if (copy_inst_from_kernel_nofault(&op, (void *)ip)) { 730 730 pr_err("Fetching opcode failed.\n"); 731 731 return -EFAULT; 732 732 }

+21 -19

arch/powerpc/kernel/traps.c

··· 53 53 #ifdef CONFIG_PPC64 54 54 #include <asm/firmware.h> 55 55 #include <asm/processor.h> 56 - #include <asm/tm.h> 57 56 #endif 58 57 #include <asm/kexec.h> 59 58 #include <asm/ppc-opcode.h> ··· 221 222 /* 222 223 * system_reset_excption handles debugger, crash dump, panic, for 0x100 223 224 */ 224 - if (TRAP(regs) == 0x100) 225 + if (TRAP(regs) == INTERRUPT_SYSTEM_RESET) 225 226 return; 226 227 227 228 crash_fadump(regs, "die oops"); ··· 289 290 /* 290 291 * system_reset_excption handles debugger, crash dump, panic, for 0x100 291 292 */ 292 - if (TRAP(regs) != 0x100) { 293 + if (TRAP(regs) != INTERRUPT_SYSTEM_RESET) { 293 294 if (debugger(regs)) 294 295 return; 295 296 } ··· 404 405 * Now test if the interrupt has hit a range that may be using 405 406 * HSPRG1 without having RI=0 (i.e., an HSRR interrupt). The 406 407 * problem ranges all run un-relocated. Test real and virt modes 407 - * at the same time by droping the high bit of the nip (virt mode 408 + * at the same time by dropping the high bit of the nip (virt mode 408 409 * entry points still have the +0x4000 offset). 409 410 */ 410 411 nip &= ~0xc000000000000000ULL; ··· 863 864 unsigned long ea, msr, msr_mask; 864 865 bool swap; 865 866 866 - if (__get_user_inatomic(instr, (unsigned int __user *)regs->nip)) 867 + if (__get_user(instr, (unsigned int __user *)regs->nip)) 867 868 return; 868 869 869 870 /* ··· 1076 1077 regs->nip, regs->msr, regs->trap); 1077 1078 1078 1079 _exception(SIGTRAP, regs, TRAP_UNK, 0); 1080 + } 1081 + 1082 + DEFINE_INTERRUPT_HANDLER_NMI(unknown_nmi_exception) 1083 + { 1084 + printk("Bad trap at PC: %lx, SR: %lx, vector=%lx\n", 1085 + regs->nip, regs->msr, regs->trap); 1086 + 1087 + _exception(SIGTRAP, regs, TRAP_UNK, 0); 1088 + 1089 + return 0; 1079 1090 } 1080 1091 1081 1092 DEFINE_INTERRUPT_HANDLER(instruction_breakpoint_exception) ··· 1318 1309 1319 1310 if (!user_mode(regs)) 1320 1311 return -EINVAL; 1321 - CHECK_FULL_REGS(regs); 1322 1312 1323 1313 if (get_user(instword, (u32 __user *)(regs->nip))) 1324 1314 return -EFAULT; ··· 1414 1406 static int emulate_math(struct pt_regs *regs) 1415 1407 { 1416 1408 int ret; 1417 - extern int do_mathemu(struct pt_regs *regs); 1418 1409 1419 1410 ret = do_mathemu(regs); 1420 1411 if (ret >= 0) ··· 1613 1606 bad_page_fault(regs, sig); 1614 1607 } 1615 1608 1616 - DEFINE_INTERRUPT_HANDLER(StackOverflow) 1617 - { 1618 - pr_crit("Kernel stack overflow in process %s[%d], r1=%lx\n", 1619 - current->comm, task_pid_nr(current), regs->gpr[1]); 1620 - debugger(regs); 1621 - show_regs(regs); 1622 - panic("kernel stack overflow"); 1623 - } 1624 - 1625 1609 DEFINE_INTERRUPT_HANDLER(stack_overflow_exception) 1626 1610 { 1627 1611 die("Kernel stack overflow", regs, SIGSEGV); ··· 1691 1693 u8 status; 1692 1694 bool hv; 1693 1695 1694 - hv = (TRAP(regs) == 0xf80); 1696 + hv = (TRAP(regs) == INTERRUPT_H_FAC_UNAVAIL); 1695 1697 if (hv) 1696 1698 value = mfspr(SPRN_HFSCR); 1697 1699 else ··· 2168 2170 * in the MSR is 0. This indicates that SRR0/1 are live, and that 2169 2171 * we therefore lost state by taking this exception. 2170 2172 */ 2171 - void unrecoverable_exception(struct pt_regs *regs) 2173 + void __noreturn unrecoverable_exception(struct pt_regs *regs) 2172 2174 { 2173 2175 pr_emerg("Unrecoverable exception %lx at %lx (msr=%lx)\n", 2174 2176 regs->trap, regs->nip, regs->msr); 2175 2177 die("Unrecoverable exception", regs, SIGABRT); 2178 + /* die() should not return */ 2179 + for (;;) 2180 + ; 2176 2181 } 2177 2182 2178 2183 #if defined(CONFIG_BOOKE_WDT) || defined(CONFIG_40x) ··· 2190 2189 return; 2191 2190 } 2192 2191 2193 - DEFINE_INTERRUPT_HANDLER(WatchdogException) /* XXX NMI? async? */ 2192 + DEFINE_INTERRUPT_HANDLER_NMI(WatchdogException) 2194 2193 { 2195 2194 printk (KERN_EMERG "PowerPC Book-E Watchdog Exception\n"); 2196 2195 WatchdogHandler(regs); 2196 + return 0; 2197 2197 } 2198 2198 #endif 2199 2199

+7

arch/powerpc/kernel/uprobes.c

··· 41 41 if (addr & 0x03) 42 42 return -EINVAL; 43 43 44 + if (cpu_has_feature(CPU_FTR_ARCH_31) && 45 + ppc_inst_prefixed(auprobe->insn) && 46 + (addr & 0x3f) == 60) { 47 + pr_info_ratelimited("Cannot register a uprobe on 64 byte unaligned prefixed instruction\n"); 48 + return -EINVAL; 49 + } 50 + 44 51 return 0; 45 52 } 46 53

+123 -15

arch/powerpc/kernel/vdso.c

··· 18 18 #include <linux/security.h> 19 19 #include <linux/memblock.h> 20 20 #include <linux/syscalls.h> 21 + #include <linux/time_namespace.h> 21 22 #include <vdso/datapage.h> 22 23 23 24 #include <asm/syscall.h> ··· 51 50 } vdso_data_store __page_aligned_data; 52 51 struct vdso_arch_data *vdso_data = &vdso_data_store.data; 53 52 53 + enum vvar_pages { 54 + VVAR_DATA_PAGE_OFFSET, 55 + VVAR_TIMENS_PAGE_OFFSET, 56 + VVAR_NR_PAGES, 57 + }; 58 + 54 59 static int vdso_mremap(const struct vm_special_mapping *sm, struct vm_area_struct *new_vma, 55 60 unsigned long text_size) 56 61 { 57 62 unsigned long new_size = new_vma->vm_end - new_vma->vm_start; 58 63 59 - if (new_size != text_size + PAGE_SIZE) 64 + if (new_size != text_size) 60 65 return -EINVAL; 61 66 62 - current->mm->context.vdso = (void __user *)new_vma->vm_start + PAGE_SIZE; 67 + current->mm->context.vdso = (void __user *)new_vma->vm_start; 63 68 64 69 return 0; 65 70 } ··· 80 73 return vdso_mremap(sm, new_vma, &vdso64_end - &vdso64_start); 81 74 } 82 75 76 + static vm_fault_t vvar_fault(const struct vm_special_mapping *sm, 77 + struct vm_area_struct *vma, struct vm_fault *vmf); 78 + 79 + static struct vm_special_mapping vvar_spec __ro_after_init = { 80 + .name = "[vvar]", 81 + .fault = vvar_fault, 82 + }; 83 + 83 84 static struct vm_special_mapping vdso32_spec __ro_after_init = { 84 85 .name = "[vdso]", 85 86 .mremap = vdso32_mremap, ··· 98 83 .mremap = vdso64_mremap, 99 84 }; 100 85 86 + #ifdef CONFIG_TIME_NS 87 + struct vdso_data *arch_get_vdso_data(void *vvar_page) 88 + { 89 + return ((struct vdso_arch_data *)vvar_page)->data; 90 + } 91 + 92 + /* 93 + * The vvar mapping contains data for a specific time namespace, so when a task 94 + * changes namespace we must unmap its vvar data for the old namespace. 95 + * Subsequent faults will map in data for the new namespace. 96 + * 97 + * For more details see timens_setup_vdso_data(). 98 + */ 99 + int vdso_join_timens(struct task_struct *task, struct time_namespace *ns) 100 + { 101 + struct mm_struct *mm = task->mm; 102 + struct vm_area_struct *vma; 103 + 104 + mmap_read_lock(mm); 105 + 106 + for (vma = mm->mmap; vma; vma = vma->vm_next) { 107 + unsigned long size = vma->vm_end - vma->vm_start; 108 + 109 + if (vma_is_special_mapping(vma, &vvar_spec)) 110 + zap_page_range(vma, vma->vm_start, size); 111 + } 112 + 113 + mmap_read_unlock(mm); 114 + return 0; 115 + } 116 + 117 + static struct page *find_timens_vvar_page(struct vm_area_struct *vma) 118 + { 119 + if (likely(vma->vm_mm == current->mm)) 120 + return current->nsproxy->time_ns->vvar_page; 121 + 122 + /* 123 + * VM_PFNMAP | VM_IO protect .fault() handler from being called 124 + * through interfaces like /proc/$pid/mem or 125 + * process_vm_{readv,writev}() as long as there's no .access() 126 + * in special_mapping_vmops. 127 + * For more details check_vma_flags() and __access_remote_vm() 128 + */ 129 + WARN(1, "vvar_page accessed remotely"); 130 + 131 + return NULL; 132 + } 133 + #else 134 + static struct page *find_timens_vvar_page(struct vm_area_struct *vma) 135 + { 136 + return NULL; 137 + } 138 + #endif 139 + 140 + static vm_fault_t vvar_fault(const struct vm_special_mapping *sm, 141 + struct vm_area_struct *vma, struct vm_fault *vmf) 142 + { 143 + struct page *timens_page = find_timens_vvar_page(vma); 144 + unsigned long pfn; 145 + 146 + switch (vmf->pgoff) { 147 + case VVAR_DATA_PAGE_OFFSET: 148 + if (timens_page) 149 + pfn = page_to_pfn(timens_page); 150 + else 151 + pfn = virt_to_pfn(vdso_data); 152 + break; 153 + #ifdef CONFIG_TIME_NS 154 + case VVAR_TIMENS_PAGE_OFFSET: 155 + /* 156 + * If a task belongs to a time namespace then a namespace 157 + * specific VVAR is mapped with the VVAR_DATA_PAGE_OFFSET and 158 + * the real VVAR page is mapped with the VVAR_TIMENS_PAGE_OFFSET 159 + * offset. 160 + * See also the comment near timens_setup_vdso_data(). 161 + */ 162 + if (!timens_page) 163 + return VM_FAULT_SIGBUS; 164 + pfn = virt_to_pfn(vdso_data); 165 + break; 166 + #endif /* CONFIG_TIME_NS */ 167 + default: 168 + return VM_FAULT_SIGBUS; 169 + } 170 + 171 + return vmf_insert_pfn(vma, vmf->address, pfn); 172 + } 173 + 101 174 /* 102 175 * This is called from binfmt_elf, we create the special vma for the 103 176 * vDSO and insert it into the mm struct tree 104 177 */ 105 178 static int __arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp) 106 179 { 107 - struct mm_struct *mm = current->mm; 180 + unsigned long vdso_size, vdso_base, mappings_size; 108 181 struct vm_special_mapping *vdso_spec; 182 + unsigned long vvar_size = VVAR_NR_PAGES * PAGE_SIZE; 183 + struct mm_struct *mm = current->mm; 109 184 struct vm_area_struct *vma; 110 - unsigned long vdso_size; 111 - unsigned long vdso_base; 112 185 113 186 if (is_32bit_task()) { 114 187 vdso_spec = &vdso32_spec; ··· 213 110 vdso_base = 0; 214 111 } 215 112 216 - /* Add a page to the vdso size for the data page */ 217 - vdso_size += PAGE_SIZE; 113 + mappings_size = vdso_size + vvar_size; 114 + mappings_size += (VDSO_ALIGNMENT - 1) & PAGE_MASK; 218 115 219 116 /* 220 117 * pick a base address for the vDSO in process space. We try to put it ··· 222 119 * and end up putting it elsewhere. 223 120 * Add enough to the size so that the result can be aligned. 224 121 */ 225 - vdso_base = get_unmapped_area(NULL, vdso_base, 226 - vdso_size + ((VDSO_ALIGNMENT - 1) & PAGE_MASK), 227 - 0, 0); 122 + vdso_base = get_unmapped_area(NULL, vdso_base, mappings_size, 0, 0); 228 123 if (IS_ERR_VALUE(vdso_base)) 229 124 return vdso_base; 230 125 ··· 234 133 * install_special_mapping or the perf counter mmap tracking code 235 134 * will fail to recognise it as a vDSO. 236 135 */ 237 - mm->context.vdso = (void __user *)vdso_base + PAGE_SIZE; 136 + mm->context.vdso = (void __user *)vdso_base + vvar_size; 137 + 138 + vma = _install_special_mapping(mm, vdso_base, vvar_size, 139 + VM_READ | VM_MAYREAD | VM_IO | 140 + VM_DONTDUMP | VM_PFNMAP, &vvar_spec); 141 + if (IS_ERR(vma)) 142 + return PTR_ERR(vma); 238 143 239 144 /* 240 145 * our vma flags don't have VM_WRITE so by default, the process isn't ··· 252 145 * It's fine to use that for setting breakpoints in the vDSO code 253 146 * pages though. 254 147 */ 255 - vma = _install_special_mapping(mm, vdso_base, vdso_size, 148 + vma = _install_special_mapping(mm, vdso_base + vvar_size, vdso_size, 256 149 VM_READ | VM_EXEC | VM_MAYREAD | 257 150 VM_MAYWRITE | VM_MAYEXEC, vdso_spec); 151 + if (IS_ERR(vma)) 152 + do_munmap(mm, vdso_base, vvar_size, NULL); 153 + 258 154 return PTR_ERR_OR_ZERO(vma); 259 155 } 260 156 ··· 359 249 if (!pagelist) 360 250 panic("%s: Cannot allocate page list for VDSO", __func__); 361 251 362 - pagelist[0] = virt_to_page(vdso_data); 363 - 364 252 for (i = 0; i < pages; i++) 365 - pagelist[i + 1] = virt_to_page(start + i * PAGE_SIZE); 253 + pagelist[i] = virt_to_page(start + i * PAGE_SIZE); 366 254 367 255 return pagelist; 368 256 }

+1 -1

arch/powerpc/kernel/vdso32/vdso32.lds.S

··· 17 17 18 18 SECTIONS 19 19 { 20 - PROVIDE(_vdso_datapage = . - PAGE_SIZE); 20 + PROVIDE(_vdso_datapage = . - 2 * PAGE_SIZE); 21 21 . = SIZEOF_HEADERS; 22 22 23 23 .hash : { *(.hash) } :text

+1 -1

arch/powerpc/kernel/vdso64/vdso64.lds.S

··· 17 17 18 18 SECTIONS 19 19 { 20 - PROVIDE(_vdso_datapage = . - PAGE_SIZE); 20 + PROVIDE(_vdso_datapage = . - 2 * PAGE_SIZE); 21 21 . = SIZEOF_HEADERS; 22 22 23 23 .hash : { *(.hash) } :text

-2

arch/powerpc/kernel/vector.S

··· 67 67 #ifdef CONFIG_PPC32 68 68 mfspr r5,SPRN_SPRG_THREAD /* current task's THREAD (phys) */ 69 69 oris r9,r9,MSR_VEC@h 70 - #ifdef CONFIG_VMAP_STACK 71 70 tovirt(r5, r5) 72 - #endif 73 71 #else 74 72 ld r4,PACACURRENT(r13) 75 73 addi r5,r4,THREAD /* Get THREAD */

+2 -1

arch/powerpc/kexec/crash.c

··· 24 24 #include <asm/smp.h> 25 25 #include <asm/setjmp.h> 26 26 #include <asm/debug.h> 27 + #include <asm/interrupt.h> 27 28 28 29 /* 29 30 * The primary CPU waits a while for all secondary CPUs to enter. This is to ··· 337 336 * If we came in via system reset, wait a while for the secondary 338 337 * CPUs to enter. 339 338 */ 340 - if (TRAP(regs) == 0x100) 339 + if (TRAP(regs) == INTERRUPT_SYSTEM_RESET) 341 340 mdelay(PRIMARY_TIMEOUT); 342 341 343 342 crash_kexec_prepare_cpus(crashing_cpu);

+2

arch/powerpc/kvm/book3s_64_mmu_host.c

··· 8 8 */ 9 9 10 10 #include <linux/kvm_host.h> 11 + #include <linux/pkeys.h> 11 12 12 13 #include <asm/kvm_ppc.h> 13 14 #include <asm/kvm_book3s.h> ··· 134 133 else 135 134 kvmppc_mmu_flush_icache(pfn); 136 135 136 + rflags |= pte_to_hpte_pkey_bits(0, HPTE_USE_KERNEL_KEY); 137 137 rflags = (rflags & ~HPTE_R_WIMG) | orig_pte->wimg; 138 138 139 139 /*

+65 -20

arch/powerpc/kvm/book3s_hv.c

··· 803 803 vcpu->arch.dawrx1 = value2; 804 804 return H_SUCCESS; 805 805 case H_SET_MODE_RESOURCE_ADDR_TRANS_MODE: 806 - /* KVM does not support mflags=2 (AIL=2) */ 806 + /* 807 + * KVM does not support mflags=2 (AIL=2) and AIL=1 is reserved. 808 + * Keep this in synch with kvmppc_filter_guest_lpcr_hv. 809 + */ 807 810 if (mflags != 0 && mflags != 3) 808 811 return H_UNSUPPORTED_FLAG_START; 809 812 return H_TOO_HARD; ··· 1638 1635 return 0; 1639 1636 } 1640 1637 1638 + /* 1639 + * Enforce limits on guest LPCR values based on hardware availability, 1640 + * guest configuration, and possibly hypervisor support and security 1641 + * concerns. 1642 + */ 1643 + unsigned long kvmppc_filter_lpcr_hv(struct kvm *kvm, unsigned long lpcr) 1644 + { 1645 + /* LPCR_TC only applies to HPT guests */ 1646 + if (kvm_is_radix(kvm)) 1647 + lpcr &= ~LPCR_TC; 1648 + 1649 + /* On POWER8 and above, userspace can modify AIL */ 1650 + if (!cpu_has_feature(CPU_FTR_ARCH_207S)) 1651 + lpcr &= ~LPCR_AIL; 1652 + if ((lpcr & LPCR_AIL) != LPCR_AIL_3) 1653 + lpcr &= ~LPCR_AIL; /* LPCR[AIL]=1/2 is disallowed */ 1654 + 1655 + /* 1656 + * On POWER9, allow userspace to enable large decrementer for the 1657 + * guest, whether or not the host has it enabled. 1658 + */ 1659 + if (!cpu_has_feature(CPU_FTR_ARCH_300)) 1660 + lpcr &= ~LPCR_LD; 1661 + 1662 + return lpcr; 1663 + } 1664 + 1665 + static void verify_lpcr(struct kvm *kvm, unsigned long lpcr) 1666 + { 1667 + if (lpcr != kvmppc_filter_lpcr_hv(kvm, lpcr)) { 1668 + WARN_ONCE(1, "lpcr 0x%lx differs from filtered 0x%lx\n", 1669 + lpcr, kvmppc_filter_lpcr_hv(kvm, lpcr)); 1670 + } 1671 + } 1672 + 1641 1673 static void kvmppc_set_lpcr(struct kvm_vcpu *vcpu, u64 new_lpcr, 1642 1674 bool preserve_top32) 1643 1675 { ··· 1681 1643 u64 mask; 1682 1644 1683 1645 spin_lock(&vc->lock); 1646 + 1647 + /* 1648 + * Userspace can only modify 1649 + * DPFD (default prefetch depth), ILE (interrupt little-endian), 1650 + * TC (translation control), AIL (alternate interrupt location), 1651 + * LD (large decrementer). 1652 + * These are subject to restrictions from kvmppc_filter_lcpr_hv(). 1653 + */ 1654 + mask = LPCR_DPFD | LPCR_ILE | LPCR_TC | LPCR_AIL | LPCR_LD; 1655 + 1656 + /* Broken 32-bit version of LPCR must not clear top bits */ 1657 + if (preserve_top32) 1658 + mask &= 0xFFFFFFFF; 1659 + 1660 + new_lpcr = kvmppc_filter_lpcr_hv(kvm, 1661 + (vc->lpcr & ~mask) | (new_lpcr & mask)); 1662 + 1684 1663 /* 1685 1664 * If ILE (interrupt little-endian) has changed, update the 1686 1665 * MSR_LE bit in the intr_msr for each vcpu in this vcore. ··· 1716 1661 } 1717 1662 } 1718 1663 1719 - /* 1720 - * Userspace can only modify DPFD (default prefetch depth), 1721 - * ILE (interrupt little-endian) and TC (translation control). 1722 - * On POWER8 and POWER9 userspace can also modify AIL (alt. interrupt loc.). 1723 - */ 1724 - mask = LPCR_DPFD | LPCR_ILE | LPCR_TC; 1725 - if (cpu_has_feature(CPU_FTR_ARCH_207S)) 1726 - mask |= LPCR_AIL; 1727 - /* 1728 - * On POWER9, allow userspace to enable large decrementer for the 1729 - * guest, whether or not the host has it enabled. 1730 - */ 1731 - if (cpu_has_feature(CPU_FTR_ARCH_300)) 1732 - mask |= LPCR_LD; 1664 + vc->lpcr = new_lpcr; 1733 1665 1734 - /* Broken 32-bit version of LPCR must not clear top bits */ 1735 - if (preserve_top32) 1736 - mask &= 0xFFFFFFFF; 1737 - vc->lpcr = (vc->lpcr & ~mask) | (new_lpcr & mask); 1738 1666 spin_unlock(&vc->lock); 1739 1667 } 1740 1668 ··· 3766 3728 vcpu->arch.dec_expires = dec + tb; 3767 3729 vcpu->cpu = -1; 3768 3730 vcpu->arch.thread_cpu = -1; 3731 + /* Save guest CTRL register, set runlatch to 1 */ 3769 3732 vcpu->arch.ctrl = mfspr(SPRN_CTRLF); 3733 + if (!(vcpu->arch.ctrl & 1)) 3734 + mtspr(SPRN_CTRLT, vcpu->arch.ctrl | 1); 3770 3735 3771 3736 vcpu->arch.iamr = mfspr(SPRN_IAMR); 3772 3737 vcpu->arch.pspb = mfspr(SPRN_PSPB); ··· 3790 3749 mtspr(SPRN_DSCR, host_dscr); 3791 3750 mtspr(SPRN_TIDR, host_tidr); 3792 3751 mtspr(SPRN_IAMR, host_iamr); 3793 - mtspr(SPRN_PSPB, 0); 3794 3752 3795 3753 if (host_amr != vcpu->arch.amr) 3796 3754 mtspr(SPRN_AMR, host_amr); ··· 4681 4641 struct kvmppc_vcore *vc = kvm->arch.vcores[i]; 4682 4642 if (!vc) 4683 4643 continue; 4644 + 4684 4645 spin_lock(&vc->lock); 4685 4646 vc->lpcr = (vc->lpcr & ~mask) | lpcr; 4647 + verify_lpcr(kvm, vc->lpcr); 4686 4648 spin_unlock(&vc->lock); 4687 4649 if (++cores_done >= kvm->arch.online_vcores) 4688 4650 break; ··· 5012 4970 kvmppc_setup_partition_table(kvm); 5013 4971 } 5014 4972 4973 + verify_lpcr(kvm, lpcr); 5015 4974 kvm->arch.lpcr = lpcr; 5016 4975 5017 4976 /* Initialization for future HPT resizes */ ··· 5412 5369 H_READ, 5413 5370 H_PROTECT, 5414 5371 H_BULK_REMOVE, 5372 + #ifdef CONFIG_SPAPR_TCE_IOMMU 5415 5373 H_GET_TCE, 5416 5374 H_PUT_TCE, 5375 + #endif 5417 5376 H_SET_DABR, 5418 5377 H_SET_XDABR, 5419 5378 H_CEDE,

+3

arch/powerpc/kvm/book3s_hv_builtin.c

··· 662 662 663 663 void kvmppc_set_msr_hv(struct kvm_vcpu *vcpu, u64 msr) 664 664 { 665 + /* Guest must always run with ME enabled, HV disabled. */ 666 + msr = (msr | MSR_ME) & ~MSR_HV; 667 + 665 668 /* 666 669 * Check for illegal transactional state bit combination 667 670 * and if we find it, force the TS field to a safe state.

+30 -7

arch/powerpc/kvm/book3s_hv_nested.c

··· 132 132 } 133 133 } 134 134 135 + /* 136 + * This can result in some L0 HV register state being leaked to an L1 137 + * hypervisor when the hv_guest_state is copied back to the guest after 138 + * being modified here. 139 + * 140 + * There is no known problem with such a leak, and in many cases these 141 + * register settings could be derived by the guest by observing behaviour 142 + * and timing, interrupts, etc., but it is an issue to consider. 143 + */ 135 144 static void sanitise_hv_regs(struct kvm_vcpu *vcpu, struct hv_guest_state *hr) 136 145 { 146 + struct kvmppc_vcore *vc = vcpu->arch.vcore; 147 + u64 mask; 148 + 149 + /* 150 + * Don't let L1 change LPCR bits for the L2 except these: 151 + */ 152 + mask = LPCR_DPFD | LPCR_ILE | LPCR_TC | LPCR_AIL | LPCR_LD | 153 + LPCR_LPES | LPCR_MER; 154 + 155 + /* 156 + * Additional filtering is required depending on hardware 157 + * and configuration. 158 + */ 159 + hr->lpcr = kvmppc_filter_lpcr_hv(vcpu->kvm, 160 + (vc->lpcr & ~mask) | (hr->lpcr & mask)); 161 + 137 162 /* 138 163 * Don't let L1 enable features for L2 which we've disabled for L1, 139 164 * but preserve the interrupt cause field. ··· 296 271 u64 hv_ptr, regs_ptr; 297 272 u64 hdec_exp; 298 273 s64 delta_purr, delta_spurr, delta_ic, delta_vtb; 299 - u64 mask; 300 - unsigned long lpcr; 301 274 302 275 if (vcpu->kvm->arch.l1_ptcr == 0) 303 276 return H_NOT_AVAILABLE; ··· 343 320 vcpu->arch.nested = l2; 344 321 vcpu->arch.nested_vcpu_id = l2_hv.vcpu_token; 345 322 vcpu->arch.regs = l2_regs; 346 - vcpu->arch.shregs.msr = vcpu->arch.regs.msr; 347 - mask = LPCR_DPFD | LPCR_ILE | LPCR_TC | LPCR_AIL | LPCR_LD | 348 - LPCR_LPES | LPCR_MER; 349 - lpcr = (vc->lpcr & ~mask) | (l2_hv.lpcr & mask); 323 + 324 + /* Guest must always run with ME enabled, HV disabled. */ 325 + vcpu->arch.shregs.msr = (vcpu->arch.regs.msr | MSR_ME) & ~MSR_HV; 326 + 350 327 sanitise_hv_regs(vcpu, &l2_hv); 351 328 restore_hv_regs(vcpu, &l2_hv); 352 329 ··· 358 335 r = RESUME_HOST; 359 336 break; 360 337 } 361 - r = kvmhv_run_single_vcpu(vcpu, hdec_exp, lpcr); 338 + r = kvmhv_run_single_vcpu(vcpu, hdec_exp, l2_hv.lpcr); 362 339 } while (is_kvmppc_resume_guest(r)); 363 340 364 341 /* save L2 state for return */

+1 -2

arch/powerpc/kvm/book3s_hv_rm_mmu.c

··· 673 673 } 674 674 675 675 long kvmppc_h_protect(struct kvm_vcpu *vcpu, unsigned long flags, 676 - unsigned long pte_index, unsigned long avpn, 677 - unsigned long va) 676 + unsigned long pte_index, unsigned long avpn) 678 677 { 679 678 struct kvm *kvm = vcpu->kvm; 680 679 __be64 *hpte;

+1 -1

arch/powerpc/lib/Makefile

··· 16 16 CFLAGS_feature-fixups.o += -DDISABLE_BRANCH_PROFILING 17 17 endif 18 18 19 - obj-y += alloc.o code-patching.o feature-fixups.o pmem.o inst.o test_code-patching.o 19 + obj-y += alloc.o code-patching.o feature-fixups.o pmem.o test_code-patching.o 20 20 21 21 ifndef CONFIG_KASAN 22 22 obj-y += string.o memcmp_$(BITS).o

+4 -11

arch/powerpc/lib/checksum_wrappers.c

··· 16 16 { 17 17 __wsum csum; 18 18 19 - might_sleep(); 20 - 21 - if (unlikely(!access_ok(src, len))) 19 + if (unlikely(!user_read_access_begin(src, len))) 22 20 return 0; 23 - 24 - allow_read_from_user(src, len); 25 21 26 22 csum = csum_partial_copy_generic((void __force *)src, dst, len); 27 23 28 - prevent_read_from_user(src, len); 24 + user_read_access_end(); 29 25 return csum; 30 26 } 31 27 EXPORT_SYMBOL(csum_and_copy_from_user); ··· 30 34 { 31 35 __wsum csum; 32 36 33 - might_sleep(); 34 - if (unlikely(!access_ok(dst, len))) 37 + if (unlikely(!user_write_access_begin(dst, len))) 35 38 return 0; 36 - 37 - allow_write_to_user(dst, len); 38 39 39 40 csum = csum_partial_copy_generic(src, (void __force *)dst, len); 40 41 41 - prevent_write_to_user(dst, len); 42 + user_write_access_end(); 42 43 return csum; 43 44 } 44 45 EXPORT_SYMBOL(csum_and_copy_to_user);

+9 -4

arch/powerpc/lib/code-patching.c

··· 21 21 static int __patch_instruction(struct ppc_inst *exec_addr, struct ppc_inst instr, 22 22 struct ppc_inst *patch_addr) 23 23 { 24 - if (!ppc_inst_prefixed(instr)) 25 - __put_user_asm_goto(ppc_inst_val(instr), patch_addr, failed, "stw"); 26 - else 27 - __put_user_asm_goto(ppc_inst_as_u64(instr), patch_addr, failed, "std"); 24 + if (!ppc_inst_prefixed(instr)) { 25 + u32 val = ppc_inst_val(instr); 26 + 27 + __put_kernel_nofault(patch_addr, &val, u32, failed); 28 + } else { 29 + u64 val = ppc_inst_as_ulong(instr); 30 + 31 + __put_kernel_nofault(patch_addr, &val, u64, failed); 32 + } 28 33 29 34 asm ("dcbst 0, %0; sync; icbi 0,%1; sync; isync" :: "r" (patch_addr), 30 35 "r" (exec_addr));

-73

arch/powerpc/lib/inst.c

··· 1 - // SPDX-License-Identifier: GPL-2.0-or-later 2 - /* 3 - * Copyright 2020, IBM Corporation. 4 - */ 5 - 6 - #include <linux/uaccess.h> 7 - #include <asm/disassemble.h> 8 - #include <asm/inst.h> 9 - #include <asm/ppc-opcode.h> 10 - 11 - #ifdef CONFIG_PPC64 12 - int probe_user_read_inst(struct ppc_inst *inst, 13 - struct ppc_inst __user *nip) 14 - { 15 - unsigned int val, suffix; 16 - int err; 17 - 18 - err = copy_from_user_nofault(&val, nip, sizeof(val)); 19 - if (err) 20 - return err; 21 - if (get_op(val) == OP_PREFIX) { 22 - err = copy_from_user_nofault(&suffix, (void __user *)nip + 4, 4); 23 - *inst = ppc_inst_prefix(val, suffix); 24 - } else { 25 - *inst = ppc_inst(val); 26 - } 27 - return err; 28 - } 29 - 30 - int probe_kernel_read_inst(struct ppc_inst *inst, 31 - struct ppc_inst *src) 32 - { 33 - unsigned int val, suffix; 34 - int err; 35 - 36 - err = copy_from_kernel_nofault(&val, src, sizeof(val)); 37 - if (err) 38 - return err; 39 - if (get_op(val) == OP_PREFIX) { 40 - err = copy_from_kernel_nofault(&suffix, (void *)src + 4, 4); 41 - *inst = ppc_inst_prefix(val, suffix); 42 - } else { 43 - *inst = ppc_inst(val); 44 - } 45 - return err; 46 - } 47 - #else /* !CONFIG_PPC64 */ 48 - int probe_user_read_inst(struct ppc_inst *inst, 49 - struct ppc_inst __user *nip) 50 - { 51 - unsigned int val; 52 - int err; 53 - 54 - err = copy_from_user_nofault(&val, nip, sizeof(val)); 55 - if (!err) 56 - *inst = ppc_inst(val); 57 - 58 - return err; 59 - } 60 - 61 - int probe_kernel_read_inst(struct ppc_inst *inst, 62 - struct ppc_inst *src) 63 - { 64 - unsigned int val; 65 - int err; 66 - 67 - err = copy_from_kernel_nofault(&val, src, sizeof(val)); 68 - if (!err) 69 - *inst = ppc_inst(val); 70 - 71 - return err; 72 - } 73 - #endif /* CONFIG_PPC64 */

-13

arch/powerpc/lib/sstep.c

··· 1401 1401 break; 1402 1402 } 1403 1403 1404 - /* Following cases refer to regs->gpr[], so we need all regs */ 1405 - if (!FULL_REGS(regs)) 1406 - return -1; 1407 - 1408 1404 rd = (word >> 21) & 0x1f; 1409 1405 ra = (word >> 16) & 0x1f; 1410 1406 rb = (word >> 11) & 0x1f; ··· 3082 3086 */ 3083 3087 static nokprobe_inline int handle_stack_update(unsigned long ea, struct pt_regs *regs) 3084 3088 { 3085 - #ifdef CONFIG_PPC32 3086 - /* 3087 - * Check if we will touch kernel stack overflow 3088 - */ 3089 - if (ea - STACK_INT_FRAME_SIZE <= current->thread.ksp_limit) { 3090 - printk(KERN_CRIT "Can't kprobe this since kernel stack would overflow.\n"); 3091 - return -EINVAL; 3092 - } 3093 - #endif /* CONFIG_PPC32 */ 3094 3089 /* 3095 3090 * Check if we already set since that means we'll 3096 3091 * lose the previous value.

+2 -2

arch/powerpc/math-emu/math.c

··· 225 225 int 226 226 do_mathemu(struct pt_regs *regs) 227 227 { 228 - void *op0 = 0, *op1 = 0, *op2 = 0, *op3 = 0; 228 + void *op0 = NULL, *op1 = NULL, *op2 = NULL, *op3 = NULL; 229 229 unsigned long pc = regs->nip; 230 230 signed short sdisp; 231 231 u32 insn = 0; ··· 234 234 int type = 0; 235 235 int eflag, trap; 236 236 237 - if (get_user(insn, (u32 *)pc)) 237 + if (get_user(insn, (u32 __user *)pc)) 238 238 return -EFAULT; 239 239 240 240 switch (insn >> 26) {

+2 -1

arch/powerpc/mm/Makefile

··· 8 8 obj-y := fault.o mem.o pgtable.o mmap.o maccess.o \ 9 9 init_$(BITS).o pgtable_$(BITS).o \ 10 10 pgtable-frag.o ioremap.o ioremap_$(BITS).o \ 11 - init-common.o mmu_context.o drmem.o 11 + init-common.o mmu_context.o drmem.o \ 12 + cacheflush.o 12 13 obj-$(CONFIG_PPC_MMU_NOHASH) += nohash/ 13 14 obj-$(CONFIG_PPC_BOOK3S_32) += book3s32/ 14 15 obj-$(CONFIG_PPC_BOOK3S_64) += book3s64/

+1

arch/powerpc/mm/book3s32/Makefile

··· 9 9 obj-y += mmu.o mmu_context.o 10 10 obj-$(CONFIG_PPC_BOOK3S_603) += nohash_low.o 11 11 obj-$(CONFIG_PPC_BOOK3S_604) += hash_low.o tlb.o 12 + obj-$(CONFIG_PPC_KUEP) += kuep.o

-14

arch/powerpc/mm/book3s32/hash_low.S

··· 140 140 bne- .Lretry /* retry if someone got there first */ 141 141 142 142 mfsrin r3,r4 /* get segment reg for segment */ 143 - #ifndef CONFIG_VMAP_STACK 144 - mfctr r0 145 - stw r0,_CTR(r11) 146 - #endif 147 143 bl create_hpte /* add the hash table entry */ 148 144 149 145 #ifdef CONFIG_SMP ··· 148 152 li r0,0 149 153 stw r0, (mmu_hash_lock - PAGE_OFFSET)@l(r8) 150 154 #endif 151 - 152 - #ifdef CONFIG_VMAP_STACK 153 155 b fast_hash_page_return 154 - #else 155 - /* Return from the exception */ 156 - lwz r5,_CTR(r11) 157 - mtctr r5 158 - lwz r0,GPR0(r11) 159 - lwz r8,GPR8(r11) 160 - b fast_exception_return 161 - #endif 162 156 163 157 #ifdef CONFIG_SMP 164 158 .Lhash_page_out:

+40

arch/powerpc/mm/book3s32/kuep.c

··· 1 + // SPDX-License-Identifier: GPL-2.0-or-later 2 + 3 + #include <asm/kup.h> 4 + #include <asm/reg.h> 5 + #include <asm/task_size_32.h> 6 + #include <asm/mmu.h> 7 + 8 + #define KUEP_UPDATE_TWO_USER_SEGMENTS(n) do { \ 9 + if (TASK_SIZE > ((n) << 28)) \ 10 + mtsr(val1, (n) << 28); \ 11 + if (TASK_SIZE > (((n) + 1) << 28)) \ 12 + mtsr(val2, ((n) + 1) << 28); \ 13 + val1 = (val1 + 0x222) & 0xf0ffffff; \ 14 + val2 = (val2 + 0x222) & 0xf0ffffff; \ 15 + } while (0) 16 + 17 + static __always_inline void kuep_update(u32 val) 18 + { 19 + int val1 = val; 20 + int val2 = (val + 0x111) & 0xf0ffffff; 21 + 22 + KUEP_UPDATE_TWO_USER_SEGMENTS(0); 23 + KUEP_UPDATE_TWO_USER_SEGMENTS(2); 24 + KUEP_UPDATE_TWO_USER_SEGMENTS(4); 25 + KUEP_UPDATE_TWO_USER_SEGMENTS(6); 26 + KUEP_UPDATE_TWO_USER_SEGMENTS(8); 27 + KUEP_UPDATE_TWO_USER_SEGMENTS(10); 28 + KUEP_UPDATE_TWO_USER_SEGMENTS(12); 29 + KUEP_UPDATE_TWO_USER_SEGMENTS(14); 30 + } 31 + 32 + void kuep_lock(void) 33 + { 34 + kuep_update(mfsr(0) | SR_NX); 35 + } 36 + 37 + void kuep_unlock(void) 38 + { 39 + kuep_update(mfsr(0) & ~SR_NX); 40 + }

+1 -8

arch/powerpc/mm/book3s32/mmu.c

··· 162 162 unsigned long border = (unsigned long)__init_begin - PAGE_OFFSET; 163 163 164 164 165 - if (debug_pagealloc_enabled() || __map_without_bats) { 165 + if (debug_pagealloc_enabled_or_kfence() || __map_without_bats) { 166 166 pr_debug_once("Read-Write memory mapped without BATs\n"); 167 167 if (base >= border) 168 168 return base; ··· 184 184 { 185 185 if (!IS_ENABLED(CONFIG_MODULES)) 186 186 return false; 187 - #ifdef MODULES_VADDR 188 187 if (addr < ALIGN_DOWN(MODULES_VADDR, SZ_256M)) 189 188 return false; 190 189 if (addr > ALIGN(MODULES_END, SZ_256M) - 1) 191 190 return false; 192 - #else 193 - if (addr < ALIGN_DOWN(VMALLOC_START, SZ_256M)) 194 - return false; 195 - if (addr > ALIGN(VMALLOC_END, SZ_256M) - 1) 196 - return false; 197 - #endif 198 191 return true; 199 192 } 200 193

+121 -9

arch/powerpc/mm/book3s64/hash_pgtable.c

··· 8 8 #include <linux/sched.h> 9 9 #include <linux/mm_types.h> 10 10 #include <linux/mm.h> 11 + #include <linux/stop_machine.h> 11 12 12 13 #include <asm/sections.h> 13 14 #include <asm/mmu.h> ··· 401 400 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 402 401 403 402 #ifdef CONFIG_STRICT_KERNEL_RWX 403 + 404 + struct change_memory_parms { 405 + unsigned long start, end, newpp; 406 + unsigned int step, nr_cpus, master_cpu; 407 + atomic_t cpu_counter; 408 + }; 409 + 410 + // We'd rather this was on the stack but it has to be in the RMO 411 + static struct change_memory_parms chmem_parms; 412 + 413 + // And therefore we need a lock to protect it from concurrent use 414 + static DEFINE_MUTEX(chmem_lock); 415 + 416 + static void change_memory_range(unsigned long start, unsigned long end, 417 + unsigned int step, unsigned long newpp) 418 + { 419 + unsigned long idx; 420 + 421 + pr_debug("Changing page protection on range 0x%lx-0x%lx, to 0x%lx, step 0x%x\n", 422 + start, end, newpp, step); 423 + 424 + for (idx = start; idx < end; idx += step) 425 + /* Not sure if we can do much with the return value */ 426 + mmu_hash_ops.hpte_updateboltedpp(newpp, idx, mmu_linear_psize, 427 + mmu_kernel_ssize); 428 + } 429 + 430 + static int notrace chmem_secondary_loop(struct change_memory_parms *parms) 431 + { 432 + unsigned long msr, tmp, flags; 433 + int *p; 434 + 435 + p = &parms->cpu_counter.counter; 436 + 437 + local_irq_save(flags); 438 + hard_irq_disable(); 439 + 440 + asm volatile ( 441 + // Switch to real mode and leave interrupts off 442 + "mfmsr %[msr] ;" 443 + "li %[tmp], %[MSR_IR_DR] ;" 444 + "andc %[tmp], %[msr], %[tmp] ;" 445 + "mtmsrd %[tmp] ;" 446 + 447 + // Tell the master we are in real mode 448 + "1: " 449 + "lwarx %[tmp], 0, %[p] ;" 450 + "addic %[tmp], %[tmp], -1 ;" 451 + "stwcx. %[tmp], 0, %[p] ;" 452 + "bne- 1b ;" 453 + 454 + // Spin until the counter goes to zero 455 + "2: ;" 456 + "lwz %[tmp], 0(%[p]) ;" 457 + "cmpwi %[tmp], 0 ;" 458 + "bne- 2b ;" 459 + 460 + // Switch back to virtual mode 461 + "mtmsrd %[msr] ;" 462 + 463 + : // outputs 464 + [msr] "=&r" (msr), [tmp] "=&b" (tmp), "+m" (*p) 465 + : // inputs 466 + [p] "b" (p), [MSR_IR_DR] "i" (MSR_IR | MSR_DR) 467 + : // clobbers 468 + "cc", "xer" 469 + ); 470 + 471 + local_irq_restore(flags); 472 + 473 + return 0; 474 + } 475 + 476 + static int change_memory_range_fn(void *data) 477 + { 478 + struct change_memory_parms *parms = data; 479 + 480 + if (parms->master_cpu != smp_processor_id()) 481 + return chmem_secondary_loop(parms); 482 + 483 + // Wait for all but one CPU (this one) to call-in 484 + while (atomic_read(&parms->cpu_counter) > 1) 485 + barrier(); 486 + 487 + change_memory_range(parms->start, parms->end, parms->step, parms->newpp); 488 + 489 + mb(); 490 + 491 + // Signal the other CPUs that we're done 492 + atomic_dec(&parms->cpu_counter); 493 + 494 + return 0; 495 + } 496 + 404 497 static bool hash__change_memory_range(unsigned long start, unsigned long end, 405 498 unsigned long newpp) 406 499 { 407 - unsigned long idx; 408 500 unsigned int step, shift; 409 501 410 502 shift = mmu_psize_defs[mmu_linear_psize].shift; ··· 509 415 if (start >= end) 510 416 return false; 511 417 512 - pr_debug("Changing page protection on range 0x%lx-0x%lx, to 0x%lx, step 0x%x\n", 513 - start, end, newpp, step); 418 + if (firmware_has_feature(FW_FEATURE_LPAR)) { 419 + mutex_lock(&chmem_lock); 514 420 515 - for (idx = start; idx < end; idx += step) 516 - /* Not sure if we can do much with the return value */ 517 - mmu_hash_ops.hpte_updateboltedpp(newpp, idx, mmu_linear_psize, 518 - mmu_kernel_ssize); 421 + chmem_parms.start = start; 422 + chmem_parms.end = end; 423 + chmem_parms.step = step; 424 + chmem_parms.newpp = newpp; 425 + chmem_parms.master_cpu = smp_processor_id(); 426 + 427 + cpus_read_lock(); 428 + 429 + atomic_set(&chmem_parms.cpu_counter, num_online_cpus()); 430 + 431 + // Ensure state is consistent before we call the other CPUs 432 + mb(); 433 + 434 + stop_machine_cpuslocked(change_memory_range_fn, &chmem_parms, 435 + cpu_online_mask); 436 + 437 + cpus_read_unlock(); 438 + mutex_unlock(&chmem_lock); 439 + } else 440 + change_memory_range(start, end, step, newpp); 519 441 520 442 return true; 521 443 } 522 444 523 445 void hash__mark_rodata_ro(void) 524 446 { 525 - unsigned long start, end; 447 + unsigned long start, end, pp; 526 448 527 449 start = (unsigned long)_stext; 528 450 end = (unsigned long)__init_begin; 529 451 530 - WARN_ON(!hash__change_memory_range(start, end, PP_RXXX)); 452 + pp = htab_convert_pte_flags(pgprot_val(PAGE_KERNEL_ROX), HPTE_USE_KERNEL_KEY); 453 + 454 + WARN_ON(!hash__change_memory_range(start, end, pp)); 531 455 } 532 456 533 457 void hash__mark_initmem_nx(void)

+22 -13

arch/powerpc/mm/book3s64/hash_utils.c

··· 338 338 int htab_remove_mapping(unsigned long vstart, unsigned long vend, 339 339 int psize, int ssize) 340 340 { 341 - unsigned long vaddr; 341 + unsigned long vaddr, time_limit; 342 342 unsigned int step, shift; 343 343 int rc; 344 344 int ret = 0; ··· 351 351 352 352 /* Unmap the full range specificied */ 353 353 vaddr = ALIGN_DOWN(vstart, step); 354 + time_limit = jiffies + HZ; 355 + 354 356 for (;vaddr < vend; vaddr += step) { 355 357 rc = mmu_hash_ops.hpte_removebolted(vaddr, psize, ssize); 358 + 359 + /* 360 + * For large number of mappings introduce a cond_resched() 361 + * to prevent softlockup warnings. 362 + */ 363 + if (time_after(jiffies, time_limit)) { 364 + cond_resched(); 365 + time_limit = jiffies + HZ; 366 + } 356 367 if (rc == -ENOENT) { 357 368 ret = -ENOENT; 358 369 continue; ··· 1156 1145 1157 1146 /* page is dirty */ 1158 1147 if (!test_bit(PG_dcache_clean, &page->flags) && !PageReserved(page)) { 1159 - if (trap == 0x400) { 1148 + if (trap == INTERRUPT_INST_STORAGE) { 1160 1149 flush_dcache_icache_page(page); 1161 1150 set_bit(PG_dcache_clean, &page->flags); 1162 1151 } else ··· 1556 1545 if (user_mode(regs) || (region_id == USER_REGION_ID)) 1557 1546 access &= ~_PAGE_PRIVILEGED; 1558 1547 1559 - if (regs->trap == 0x400) 1548 + if (TRAP(regs) == INTERRUPT_INST_STORAGE) 1560 1549 access |= _PAGE_EXEC; 1561 1550 1562 - err = hash_page_mm(mm, ea, access, regs->trap, flags); 1551 + err = hash_page_mm(mm, ea, access, TRAP(regs), flags); 1563 1552 if (unlikely(err < 0)) { 1564 1553 // failed to instert a hash PTE due to an hypervisor error 1565 1554 if (user_mode(regs)) { ··· 1583 1572 DEFINE_INTERRUPT_HANDLER_RAW(do_hash_fault) 1584 1573 { 1585 1574 unsigned long dsisr = regs->dsisr; 1586 - long err; 1587 1575 1588 - if (unlikely(dsisr & (DSISR_BAD_FAULT_64S | DSISR_KEYFAULT))) 1589 - goto page_fault; 1576 + if (unlikely(dsisr & (DSISR_BAD_FAULT_64S | DSISR_KEYFAULT))) { 1577 + hash__do_page_fault(regs); 1578 + return 0; 1579 + } 1590 1580 1591 1581 /* 1592 1582 * If we are in an "NMI" (e.g., an interrupt when soft-disabled), then ··· 1607 1595 return 0; 1608 1596 } 1609 1597 1610 - err = __do_hash_fault(regs); 1611 - if (err) { 1612 - page_fault: 1613 - err = hash__do_page_fault(regs); 1614 - } 1598 + if (__do_hash_fault(regs)) 1599 + hash__do_page_fault(regs); 1615 1600 1616 - return err; 1601 + return 0; 1617 1602 } 1618 1603 1619 1604 #ifdef CONFIG_PPC_MM_SLICES

+1 -1

arch/powerpc/mm/book3s64/mmu_context.c

··· 119 119 /* This is fork. Copy hash_context details from current->mm */ 120 120 memcpy(mm->context.hash_context, current->mm->context.hash_context, sizeof(struct hash_mm_context)); 121 121 #ifdef CONFIG_PPC_SUBPAGE_PROT 122 - /* inherit subpage prot detalis if we have one. */ 122 + /* inherit subpage prot details if we have one. */ 123 123 if (current->mm->context.hash_context->spt) { 124 124 mm->context.hash_context->spt = kmalloc(sizeof(struct subpage_prot_table), 125 125 GFP_KERNEL);

+5 -15

arch/powerpc/mm/book3s64/pkeys.c

··· 301 301 } 302 302 #endif 303 303 304 - static inline void update_current_thread_amr(u64 value) 305 - { 306 - current->thread.regs->amr = value; 307 - } 308 - 309 - static inline void update_current_thread_iamr(u64 value) 310 - { 311 - if (!likely(pkey_execute_disable_supported)) 312 - return; 313 - 314 - current->thread.regs->iamr = value; 315 - } 316 - 317 304 #ifdef CONFIG_PPC_MEM_KEYS 318 305 void pkey_mm_init(struct mm_struct *mm) 319 306 { ··· 315 328 u64 new_amr_bits = (((u64)init_bits & 0x3UL) << pkeyshift(pkey)); 316 329 u64 old_amr = current_thread_amr() & ~((u64)(0x3ul) << pkeyshift(pkey)); 317 330 318 - update_current_thread_amr(old_amr | new_amr_bits); 331 + current->thread.regs->amr = old_amr | new_amr_bits; 319 332 } 320 333 321 334 static inline void init_iamr(int pkey, u8 init_bits) ··· 323 336 u64 new_iamr_bits = (((u64)init_bits & 0x1UL) << pkeyshift(pkey)); 324 337 u64 old_iamr = current_thread_iamr() & ~((u64)(0x1ul) << pkeyshift(pkey)); 325 338 326 - update_current_thread_iamr(old_iamr | new_iamr_bits); 339 + if (!likely(pkey_execute_disable_supported)) 340 + return; 341 + 342 + current->thread.regs->iamr = old_iamr | new_iamr_bits; 327 343 } 328 344 329 345 /*

+5 -5

arch/powerpc/mm/book3s64/radix_pgtable.c

··· 108 108 109 109 set_the_pte: 110 110 set_pte_at(&init_mm, ea, ptep, pfn_pte(pfn, flags)); 111 - smp_wmb(); 111 + asm volatile("ptesync": : :"memory"); 112 112 return 0; 113 113 } 114 114 ··· 168 168 169 169 set_the_pte: 170 170 set_pte_at(&init_mm, ea, ptep, pfn_pte(pfn, flags)); 171 - smp_wmb(); 171 + asm volatile("ptesync": : :"memory"); 172 172 return 0; 173 173 } 174 174 ··· 180 180 } 181 181 182 182 #ifdef CONFIG_STRICT_KERNEL_RWX 183 - void radix__change_memory_range(unsigned long start, unsigned long end, 184 - unsigned long clear) 183 + static void radix__change_memory_range(unsigned long start, unsigned long end, 184 + unsigned long clear) 185 185 { 186 186 unsigned long idx; 187 187 pgd_t *pgdp; ··· 1058 1058 * Book3S does not require a TLB flush when relaxing access 1059 1059 * restrictions when the address space is not attached to a 1060 1060 * NMMU, because the core MMU will reload the pte after taking 1061 - * an access fault, which is defined by the architectue. 1061 + * an access fault, which is defined by the architecture. 1062 1062 */ 1063 1063 } 1064 1064 /* See ptesync comment in radix__set_pte_at */

+234

arch/powerpc/mm/cacheflush.c

··· 1 + // SPDX-License-Identifier: GPL-2.0-or-later 2 + 3 + #include <linux/highmem.h> 4 + #include <linux/kprobes.h> 5 + 6 + /** 7 + * flush_coherent_icache() - if a CPU has a coherent icache, flush it 8 + * Return true if the cache was flushed, false otherwise 9 + */ 10 + static inline bool flush_coherent_icache(void) 11 + { 12 + /* 13 + * For a snooping icache, we still need a dummy icbi to purge all the 14 + * prefetched instructions from the ifetch buffers. We also need a sync 15 + * before the icbi to order the the actual stores to memory that might 16 + * have modified instructions with the icbi. 17 + */ 18 + if (cpu_has_feature(CPU_FTR_COHERENT_ICACHE)) { 19 + mb(); /* sync */ 20 + icbi((void *)PAGE_OFFSET); 21 + mb(); /* sync */ 22 + isync(); 23 + return true; 24 + } 25 + 26 + return false; 27 + } 28 + 29 + /** 30 + * invalidate_icache_range() - Flush the icache by issuing icbi across an address range 31 + * @start: the start address 32 + * @stop: the stop address (exclusive) 33 + */ 34 + static void invalidate_icache_range(unsigned long start, unsigned long stop) 35 + { 36 + unsigned long shift = l1_icache_shift(); 37 + unsigned long bytes = l1_icache_bytes(); 38 + char *addr = (char *)(start & ~(bytes - 1)); 39 + unsigned long size = stop - (unsigned long)addr + (bytes - 1); 40 + unsigned long i; 41 + 42 + for (i = 0; i < size >> shift; i++, addr += bytes) 43 + icbi(addr); 44 + 45 + mb(); /* sync */ 46 + isync(); 47 + } 48 + 49 + /** 50 + * flush_icache_range: Write any modified data cache blocks out to memory 51 + * and invalidate the corresponding blocks in the instruction cache 52 + * 53 + * Generic code will call this after writing memory, before executing from it. 54 + * 55 + * @start: the start address 56 + * @stop: the stop address (exclusive) 57 + */ 58 + void flush_icache_range(unsigned long start, unsigned long stop) 59 + { 60 + if (flush_coherent_icache()) 61 + return; 62 + 63 + clean_dcache_range(start, stop); 64 + 65 + if (IS_ENABLED(CONFIG_44x)) { 66 + /* 67 + * Flash invalidate on 44x because we are passed kmapped 68 + * addresses and this doesn't work for userspace pages due to 69 + * the virtually tagged icache. 70 + */ 71 + iccci((void *)start); 72 + mb(); /* sync */ 73 + isync(); 74 + } else 75 + invalidate_icache_range(start, stop); 76 + } 77 + EXPORT_SYMBOL(flush_icache_range); 78 + 79 + #ifdef CONFIG_HIGHMEM 80 + /** 81 + * flush_dcache_icache_phys() - Flush a page by it's physical address 82 + * @physaddr: the physical address of the page 83 + */ 84 + static void flush_dcache_icache_phys(unsigned long physaddr) 85 + { 86 + unsigned long bytes = l1_dcache_bytes(); 87 + unsigned long nb = PAGE_SIZE / bytes; 88 + unsigned long addr = physaddr & PAGE_MASK; 89 + unsigned long msr, msr0; 90 + unsigned long loop1 = addr, loop2 = addr; 91 + 92 + msr0 = mfmsr(); 93 + msr = msr0 & ~MSR_DR; 94 + /* 95 + * This must remain as ASM to prevent potential memory accesses 96 + * while the data MMU is disabled 97 + */ 98 + asm volatile( 99 + " mtctr %2;\n" 100 + " mtmsr %3;\n" 101 + " isync;\n" 102 + "0: dcbst 0, %0;\n" 103 + " addi %0, %0, %4;\n" 104 + " bdnz 0b;\n" 105 + " sync;\n" 106 + " mtctr %2;\n" 107 + "1: icbi 0, %1;\n" 108 + " addi %1, %1, %4;\n" 109 + " bdnz 1b;\n" 110 + " sync;\n" 111 + " mtmsr %5;\n" 112 + " isync;\n" 113 + : "+&r" (loop1), "+&r" (loop2) 114 + : "r" (nb), "r" (msr), "i" (bytes), "r" (msr0) 115 + : "ctr", "memory"); 116 + } 117 + NOKPROBE_SYMBOL(flush_dcache_icache_phys) 118 + #else 119 + static void flush_dcache_icache_phys(unsigned long physaddr) 120 + { 121 + } 122 + #endif 123 + 124 + /** 125 + * __flush_dcache_icache(): Flush a particular page from the data cache to RAM. 126 + * Note: this is necessary because the instruction cache does *not* 127 + * snoop from the data cache. 128 + * 129 + * @p: the address of the page to flush 130 + */ 131 + static void __flush_dcache_icache(void *p) 132 + { 133 + unsigned long addr = (unsigned long)p & PAGE_MASK; 134 + 135 + clean_dcache_range(addr, addr + PAGE_SIZE); 136 + 137 + /* 138 + * We don't flush the icache on 44x. Those have a virtual icache and we 139 + * don't have access to the virtual address here (it's not the page 140 + * vaddr but where it's mapped in user space). The flushing of the 141 + * icache on these is handled elsewhere, when a change in the address 142 + * space occurs, before returning to user space. 143 + */ 144 + 145 + if (mmu_has_feature(MMU_FTR_TYPE_44x)) 146 + return; 147 + 148 + invalidate_icache_range(addr, addr + PAGE_SIZE); 149 + } 150 + 151 + static void flush_dcache_icache_hugepage(struct page *page) 152 + { 153 + int i; 154 + int nr = compound_nr(page); 155 + 156 + if (!PageHighMem(page)) { 157 + for (i = 0; i < nr; i++) 158 + __flush_dcache_icache(lowmem_page_address(page + i)); 159 + } else { 160 + for (i = 0; i < nr; i++) { 161 + void *start = kmap_local_page(page + i); 162 + 163 + __flush_dcache_icache(start); 164 + kunmap_local(start); 165 + } 166 + } 167 + } 168 + 169 + void flush_dcache_icache_page(struct page *page) 170 + { 171 + if (flush_coherent_icache()) 172 + return; 173 + 174 + if (PageCompound(page)) 175 + return flush_dcache_icache_hugepage(page); 176 + 177 + if (!PageHighMem(page)) { 178 + __flush_dcache_icache(lowmem_page_address(page)); 179 + } else if (IS_ENABLED(CONFIG_BOOKE) || sizeof(phys_addr_t) > sizeof(void *)) { 180 + void *start = kmap_local_page(page); 181 + 182 + __flush_dcache_icache(start); 183 + kunmap_local(start); 184 + } else { 185 + flush_dcache_icache_phys(page_to_phys(page)); 186 + } 187 + } 188 + EXPORT_SYMBOL(flush_dcache_icache_page); 189 + 190 + void clear_user_page(void *page, unsigned long vaddr, struct page *pg) 191 + { 192 + clear_page(page); 193 + 194 + /* 195 + * We shouldn't have to do this, but some versions of glibc 196 + * require it (ld.so assumes zero filled pages are icache clean) 197 + * - Anton 198 + */ 199 + flush_dcache_page(pg); 200 + } 201 + EXPORT_SYMBOL(clear_user_page); 202 + 203 + void copy_user_page(void *vto, void *vfrom, unsigned long vaddr, 204 + struct page *pg) 205 + { 206 + copy_page(vto, vfrom); 207 + 208 + /* 209 + * We should be able to use the following optimisation, however 210 + * there are two problems. 211 + * Firstly a bug in some versions of binutils meant PLT sections 212 + * were not marked executable. 213 + * Secondly the first word in the GOT section is blrl, used 214 + * to establish the GOT address. Until recently the GOT was 215 + * not marked executable. 216 + * - Anton 217 + */ 218 + #if 0 219 + if (!vma->vm_file && ((vma->vm_flags & VM_EXEC) == 0)) 220 + return; 221 + #endif 222 + 223 + flush_dcache_page(pg); 224 + } 225 + 226 + void flush_icache_user_page(struct vm_area_struct *vma, struct page *page, 227 + unsigned long addr, int len) 228 + { 229 + void *maddr; 230 + 231 + maddr = kmap_local_page(page) + (addr & ~PAGE_MASK); 232 + flush_icache_range((unsigned long)maddr, (unsigned long)maddr + len); 233 + kunmap_local(maddr); 234 + }

+23 -37

arch/powerpc/mm/fault.c

··· 32 32 #include <linux/context_tracking.h> 33 33 #include <linux/hugetlb.h> 34 34 #include <linux/uaccess.h> 35 + #include <linux/kfence.h> 36 + #include <linux/pkeys.h> 35 37 36 38 #include <asm/firmware.h> 37 39 #include <asm/interrupt.h> ··· 89 87 return __bad_area(regs, address, SEGV_MAPERR); 90 88 } 91 89 92 - #ifdef CONFIG_PPC_MEM_KEYS 93 90 static noinline int bad_access_pkey(struct pt_regs *regs, unsigned long address, 94 91 struct vm_area_struct *vma) 95 92 { ··· 128 127 129 128 return 0; 130 129 } 131 - #endif 132 130 133 131 static noinline int bad_access(struct pt_regs *regs, unsigned long address) 134 132 { ··· 197 197 static bool bad_kernel_fault(struct pt_regs *regs, unsigned long error_code, 198 198 unsigned long address, bool is_write) 199 199 { 200 - int is_exec = TRAP(regs) == 0x400; 200 + int is_exec = TRAP(regs) == INTERRUPT_INST_STORAGE; 201 201 202 202 /* NX faults set DSISR_PROTFAULT on the 8xx, DSISR_NOEXEC_OR_G on others */ 203 203 if (is_exec && (error_code & (DSISR_NOEXEC_OR_G | DSISR_KEYFAULT | ··· 234 234 return false; 235 235 } 236 236 237 - #ifdef CONFIG_PPC_MEM_KEYS 238 237 static bool access_pkey_error(bool is_write, bool is_exec, bool is_pkey, 239 238 struct vm_area_struct *vma) 240 239 { ··· 247 248 248 249 return false; 249 250 } 250 - #endif 251 251 252 252 static bool access_error(bool is_write, bool is_exec, struct vm_area_struct *vma) 253 253 { ··· 391 393 struct vm_area_struct * vma; 392 394 struct mm_struct *mm = current->mm; 393 395 unsigned int flags = FAULT_FLAG_DEFAULT; 394 - int is_exec = TRAP(regs) == 0x400; 396 + int is_exec = TRAP(regs) == INTERRUPT_INST_STORAGE; 395 397 int is_user = user_mode(regs); 396 398 int is_write = page_fault_is_write(error_code); 397 399 vm_fault_t fault, major = 0; ··· 416 418 * take a page fault to a kernel address or a page fault to a user 417 419 * address outside of dedicated places 418 420 */ 419 - if (unlikely(!is_user && bad_kernel_fault(regs, error_code, address, is_write))) 421 + if (unlikely(!is_user && bad_kernel_fault(regs, error_code, address, is_write))) { 422 + if (kfence_handle_page_fault(address, is_write, regs)) 423 + return 0; 424 + 420 425 return SIGSEGV; 426 + } 421 427 422 428 /* 423 429 * If we're in an interrupt, have no user context or are running ··· 494 492 return bad_area(regs, address); 495 493 } 496 494 497 - #ifdef CONFIG_PPC_MEM_KEYS 498 495 if (unlikely(access_pkey_error(is_write, is_exec, 499 496 (error_code & DSISR_KEYFAULT), vma))) 500 497 return bad_access_pkey(regs, address, vma); 501 - #endif /* CONFIG_PPC_MEM_KEYS */ 502 498 503 499 if (unlikely(access_error(is_write, is_exec, vma))) 504 500 return bad_access(regs, address); ··· 539 539 } 540 540 NOKPROBE_SYMBOL(___do_page_fault); 541 541 542 - static long __do_page_fault(struct pt_regs *regs) 542 + static __always_inline void __do_page_fault(struct pt_regs *regs) 543 543 { 544 - const struct exception_table_entry *entry; 545 544 long err; 546 545 547 546 err = ___do_page_fault(regs, regs->dar, regs->dsisr); 548 - if (likely(!err)) 549 - return err; 550 - 551 - entry = search_exception_tables(regs->nip); 552 - if (likely(entry)) { 553 - instruction_pointer_set(regs, extable_fixup(entry)); 554 - return 0; 555 - } else if (IS_ENABLED(CONFIG_PPC_BOOK3S_64)) { 556 - __bad_page_fault(regs, err); 557 - return 0; 558 - } else { 559 - /* 32 and 64e handle the bad page fault in asm */ 560 - return err; 561 - } 547 + if (unlikely(err)) 548 + bad_page_fault(regs, err); 562 549 } 563 - NOKPROBE_SYMBOL(__do_page_fault); 564 550 565 - DEFINE_INTERRUPT_HANDLER_RET(do_page_fault) 551 + DEFINE_INTERRUPT_HANDLER(do_page_fault) 566 552 { 567 - return __do_page_fault(regs); 553 + __do_page_fault(regs); 568 554 } 569 555 570 556 #ifdef CONFIG_PPC_BOOK3S_64 571 557 /* Same as do_page_fault but interrupt entry has already run in do_hash_fault */ 572 - long hash__do_page_fault(struct pt_regs *regs) 558 + void hash__do_page_fault(struct pt_regs *regs) 573 559 { 574 - return __do_page_fault(regs); 560 + __do_page_fault(regs); 575 561 } 576 562 NOKPROBE_SYMBOL(hash__do_page_fault); 577 563 #endif ··· 567 581 * It is called from the DSI and ISI handlers in head.S and from some 568 582 * of the procedures in traps.c. 569 583 */ 570 - void __bad_page_fault(struct pt_regs *regs, int sig) 584 + static void __bad_page_fault(struct pt_regs *regs, int sig) 571 585 { 572 586 int is_write = page_fault_is_write(regs->dsisr); 573 587 574 588 /* kernel has accessed a bad area */ 575 589 576 590 switch (TRAP(regs)) { 577 - case 0x300: 578 - case 0x380: 579 - case 0xe00: 591 + case INTERRUPT_DATA_STORAGE: 592 + case INTERRUPT_DATA_SEGMENT: 593 + case INTERRUPT_H_DATA_STORAGE: 580 594 pr_alert("BUG: %s on %s at 0x%08lx\n", 581 595 regs->dar < PAGE_SIZE ? "Kernel NULL pointer dereference" : 582 596 "Unable to handle kernel data access", 583 597 is_write ? "write" : "read", regs->dar); 584 598 break; 585 - case 0x400: 586 - case 0x480: 599 + case INTERRUPT_INST_STORAGE: 600 + case INTERRUPT_INST_SEGMENT: 587 601 pr_alert("BUG: Unable to handle kernel instruction fetch%s", 588 602 regs->nip < PAGE_SIZE ? " (NULL pointer?)\n" : "\n"); 589 603 break; 590 - case 0x600: 604 + case INTERRUPT_ALIGNMENT: 591 605 pr_alert("BUG: Unable to handle kernel unaligned access at 0x%08lx\n", 592 606 regs->dar); 593 607 break;

+3

arch/powerpc/mm/init_32.c

··· 97 97 if (IS_ENABLED(CONFIG_PPC_8xx)) 98 98 return; 99 99 100 + if (IS_ENABLED(CONFIG_KFENCE)) 101 + __map_without_ltlbs = 1; 102 + 100 103 if (debug_pagealloc_enabled()) 101 104 __map_without_ltlbs = 1; 102 105

+21

arch/powerpc/mm/maccess.c

··· 3 3 #include <linux/uaccess.h> 4 4 #include <linux/kernel.h> 5 5 6 + #include <asm/disassemble.h> 7 + #include <asm/inst.h> 8 + #include <asm/ppc-opcode.h> 9 + 6 10 bool copy_from_kernel_nofault_allowed(const void *unsafe_src, size_t size) 7 11 { 8 12 return is_kernel_addr((unsigned long)unsafe_src); 13 + } 14 + 15 + int copy_inst_from_kernel_nofault(struct ppc_inst *inst, struct ppc_inst *src) 16 + { 17 + unsigned int val, suffix; 18 + int err; 19 + 20 + err = copy_from_kernel_nofault(&val, src, sizeof(val)); 21 + if (err) 22 + return err; 23 + if (IS_ENABLED(CONFIG_PPC64) && get_op(val) == OP_PREFIX) { 24 + err = copy_from_kernel_nofault(&suffix, (void *)src + 4, 4); 25 + *inst = ppc_inst_prefix(val, suffix); 26 + } else { 27 + *inst = ppc_inst(val); 28 + } 29 + return err; 9 30 }

+1 -282

arch/powerpc/mm/mem.c

··· 12 12 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds 13 13 */ 14 14 15 - #include <linux/export.h> 16 - #include <linux/sched.h> 17 - #include <linux/kernel.h> 18 - #include <linux/errno.h> 19 - #include <linux/string.h> 20 - #include <linux/gfp.h> 21 - #include <linux/types.h> 22 - #include <linux/mm.h> 23 - #include <linux/stddef.h> 24 - #include <linux/init.h> 25 15 #include <linux/memblock.h> 26 16 #include <linux/highmem.h> 27 - #include <linux/initrd.h> 28 - #include <linux/pagemap.h> 29 17 #include <linux/suspend.h> 30 - #include <linux/hugetlb.h> 31 - #include <linux/slab.h> 32 - #include <linux/vmalloc.h> 33 - #include <linux/memremap.h> 34 18 #include <linux/dma-direct.h> 35 - #include <linux/kprobes.h> 36 19 37 - #include <asm/prom.h> 38 - #include <asm/io.h> 39 - #include <asm/mmu_context.h> 40 - #include <asm/mmu.h> 41 - #include <asm/smp.h> 42 20 #include <asm/machdep.h> 43 - #include <asm/btext.h> 44 - #include <asm/tlb.h> 45 - #include <asm/sections.h> 46 - #include <asm/sparsemem.h> 47 - #include <asm/vdso.h> 48 - #include <asm/fixmap.h> 49 - #include <asm/swiotlb.h> 50 21 #include <asm/rtas.h> 51 22 #include <asm/kasan.h> 52 23 #include <asm/svm.h> 53 - #include <asm/mmzone.h> 54 24 55 25 #include <mm/mmu_decl.h> 56 26 57 - static DEFINE_MUTEX(linear_mapping_mutex); 58 27 unsigned long long memory_limit; 59 28 bool init_mem_is_free; 60 29 ··· 41 72 EXPORT_SYMBOL(phys_mem_access_prot); 42 73 43 74 #ifdef CONFIG_MEMORY_HOTPLUG 75 + static DEFINE_MUTEX(linear_mapping_mutex); 44 76 45 77 #ifdef CONFIG_NUMA 46 78 int memory_add_physaddr_to_nid(u64 start) ··· 308 338 mark_initmem_nx(); 309 339 init_mem_is_free = true; 310 340 free_initmem_default(POISON_FREE_INITMEM); 311 - } 312 - 313 - /** 314 - * flush_coherent_icache() - if a CPU has a coherent icache, flush it 315 - * @addr: The base address to use (can be any valid address, the whole cache will be flushed) 316 - * Return true if the cache was flushed, false otherwise 317 - */ 318 - static inline bool flush_coherent_icache(unsigned long addr) 319 - { 320 - /* 321 - * For a snooping icache, we still need a dummy icbi to purge all the 322 - * prefetched instructions from the ifetch buffers. We also need a sync 323 - * before the icbi to order the the actual stores to memory that might 324 - * have modified instructions with the icbi. 325 - */ 326 - if (cpu_has_feature(CPU_FTR_COHERENT_ICACHE)) { 327 - mb(); /* sync */ 328 - allow_read_from_user((const void __user *)addr, L1_CACHE_BYTES); 329 - icbi((void *)addr); 330 - prevent_read_from_user((const void __user *)addr, L1_CACHE_BYTES); 331 - mb(); /* sync */ 332 - isync(); 333 - return true; 334 - } 335 - 336 - return false; 337 - } 338 - 339 - /** 340 - * invalidate_icache_range() - Flush the icache by issuing icbi across an address range 341 - * @start: the start address 342 - * @stop: the stop address (exclusive) 343 - */ 344 - static void invalidate_icache_range(unsigned long start, unsigned long stop) 345 - { 346 - unsigned long shift = l1_icache_shift(); 347 - unsigned long bytes = l1_icache_bytes(); 348 - char *addr = (char *)(start & ~(bytes - 1)); 349 - unsigned long size = stop - (unsigned long)addr + (bytes - 1); 350 - unsigned long i; 351 - 352 - for (i = 0; i < size >> shift; i++, addr += bytes) 353 - icbi(addr); 354 - 355 - mb(); /* sync */ 356 - isync(); 357 - } 358 - 359 - /** 360 - * flush_icache_range: Write any modified data cache blocks out to memory 361 - * and invalidate the corresponding blocks in the instruction cache 362 - * 363 - * Generic code will call this after writing memory, before executing from it. 364 - * 365 - * @start: the start address 366 - * @stop: the stop address (exclusive) 367 - */ 368 - void flush_icache_range(unsigned long start, unsigned long stop) 369 - { 370 - if (flush_coherent_icache(start)) 371 - return; 372 - 373 - clean_dcache_range(start, stop); 374 - 375 - if (IS_ENABLED(CONFIG_44x)) { 376 - /* 377 - * Flash invalidate on 44x because we are passed kmapped 378 - * addresses and this doesn't work for userspace pages due to 379 - * the virtually tagged icache. 380 - */ 381 - iccci((void *)start); 382 - mb(); /* sync */ 383 - isync(); 384 - } else 385 - invalidate_icache_range(start, stop); 386 - } 387 - EXPORT_SYMBOL(flush_icache_range); 388 - 389 - #if !defined(CONFIG_PPC_8xx) && !defined(CONFIG_PPC64) 390 - /** 391 - * flush_dcache_icache_phys() - Flush a page by it's physical address 392 - * @physaddr: the physical address of the page 393 - */ 394 - static void flush_dcache_icache_phys(unsigned long physaddr) 395 - { 396 - unsigned long bytes = l1_dcache_bytes(); 397 - unsigned long nb = PAGE_SIZE / bytes; 398 - unsigned long addr = physaddr & PAGE_MASK; 399 - unsigned long msr, msr0; 400 - unsigned long loop1 = addr, loop2 = addr; 401 - 402 - msr0 = mfmsr(); 403 - msr = msr0 & ~MSR_DR; 404 - /* 405 - * This must remain as ASM to prevent potential memory accesses 406 - * while the data MMU is disabled 407 - */ 408 - asm volatile( 409 - " mtctr %2;\n" 410 - " mtmsr %3;\n" 411 - " isync;\n" 412 - "0: dcbst 0, %0;\n" 413 - " addi %0, %0, %4;\n" 414 - " bdnz 0b;\n" 415 - " sync;\n" 416 - " mtctr %2;\n" 417 - "1: icbi 0, %1;\n" 418 - " addi %1, %1, %4;\n" 419 - " bdnz 1b;\n" 420 - " sync;\n" 421 - " mtmsr %5;\n" 422 - " isync;\n" 423 - : "+&r" (loop1), "+&r" (loop2) 424 - : "r" (nb), "r" (msr), "i" (bytes), "r" (msr0) 425 - : "ctr", "memory"); 426 - } 427 - NOKPROBE_SYMBOL(flush_dcache_icache_phys) 428 - #endif // !defined(CONFIG_PPC_8xx) && !defined(CONFIG_PPC64) 429 - 430 - /* 431 - * This is called when a page has been modified by the kernel. 432 - * It just marks the page as not i-cache clean. We do the i-cache 433 - * flush later when the page is given to a user process, if necessary. 434 - */ 435 - void flush_dcache_page(struct page *page) 436 - { 437 - if (cpu_has_feature(CPU_FTR_COHERENT_ICACHE)) 438 - return; 439 - /* avoid an atomic op if possible */ 440 - if (test_bit(PG_dcache_clean, &page->flags)) 441 - clear_bit(PG_dcache_clean, &page->flags); 442 - } 443 - EXPORT_SYMBOL(flush_dcache_page); 444 - 445 - static void flush_dcache_icache_hugepage(struct page *page) 446 - { 447 - int i; 448 - void *start; 449 - 450 - BUG_ON(!PageCompound(page)); 451 - 452 - for (i = 0; i < compound_nr(page); i++) { 453 - if (!PageHighMem(page)) { 454 - __flush_dcache_icache(page_address(page+i)); 455 - } else { 456 - start = kmap_atomic(page+i); 457 - __flush_dcache_icache(start); 458 - kunmap_atomic(start); 459 - } 460 - } 461 - } 462 - 463 - void flush_dcache_icache_page(struct page *page) 464 - { 465 - 466 - if (PageCompound(page)) 467 - return flush_dcache_icache_hugepage(page); 468 - 469 - #if defined(CONFIG_PPC_8xx) || defined(CONFIG_PPC64) 470 - /* On 8xx there is no need to kmap since highmem is not supported */ 471 - __flush_dcache_icache(page_address(page)); 472 - #else 473 - if (IS_ENABLED(CONFIG_BOOKE) || sizeof(phys_addr_t) > sizeof(void *)) { 474 - void *start = kmap_atomic(page); 475 - __flush_dcache_icache(start); 476 - kunmap_atomic(start); 477 - } else { 478 - unsigned long addr = page_to_pfn(page) << PAGE_SHIFT; 479 - 480 - if (flush_coherent_icache(addr)) 481 - return; 482 - flush_dcache_icache_phys(addr); 483 - } 484 - #endif 485 - } 486 - EXPORT_SYMBOL(flush_dcache_icache_page); 487 - 488 - /** 489 - * __flush_dcache_icache(): Flush a particular page from the data cache to RAM. 490 - * Note: this is necessary because the instruction cache does *not* 491 - * snoop from the data cache. 492 - * 493 - * @page: the address of the page to flush 494 - */ 495 - void __flush_dcache_icache(void *p) 496 - { 497 - unsigned long addr = (unsigned long)p; 498 - 499 - if (flush_coherent_icache(addr)) 500 - return; 501 - 502 - clean_dcache_range(addr, addr + PAGE_SIZE); 503 - 504 - /* 505 - * We don't flush the icache on 44x. Those have a virtual icache and we 506 - * don't have access to the virtual address here (it's not the page 507 - * vaddr but where it's mapped in user space). The flushing of the 508 - * icache on these is handled elsewhere, when a change in the address 509 - * space occurs, before returning to user space. 510 - */ 511 - 512 - if (mmu_has_feature(MMU_FTR_TYPE_44x)) 513 - return; 514 - 515 - invalidate_icache_range(addr, addr + PAGE_SIZE); 516 - } 517 - 518 - void clear_user_page(void *page, unsigned long vaddr, struct page *pg) 519 - { 520 - clear_page(page); 521 - 522 - /* 523 - * We shouldn't have to do this, but some versions of glibc 524 - * require it (ld.so assumes zero filled pages are icache clean) 525 - * - Anton 526 - */ 527 - flush_dcache_page(pg); 528 - } 529 - EXPORT_SYMBOL(clear_user_page); 530 - 531 - void copy_user_page(void *vto, void *vfrom, unsigned long vaddr, 532 - struct page *pg) 533 - { 534 - copy_page(vto, vfrom); 535 - 536 - /* 537 - * We should be able to use the following optimisation, however 538 - * there are two problems. 539 - * Firstly a bug in some versions of binutils meant PLT sections 540 - * were not marked executable. 541 - * Secondly the first word in the GOT section is blrl, used 542 - * to establish the GOT address. Until recently the GOT was 543 - * not marked executable. 544 - * - Anton 545 - */ 546 - #if 0 547 - if (!vma->vm_file && ((vma->vm_flags & VM_EXEC) == 0)) 548 - return; 549 - #endif 550 - 551 - flush_dcache_page(pg); 552 - } 553 - 554 - void flush_icache_user_page(struct vm_area_struct *vma, struct page *page, 555 - unsigned long addr, int len) 556 - { 557 - unsigned long maddr; 558 - 559 - maddr = (unsigned long) kmap(page) + (addr & ~PAGE_MASK); 560 - flush_icache_range(maddr, maddr + len); 561 - kunmap(page); 562 341 } 563 342 564 343 /*

+13 -11

arch/powerpc/mm/mmu_context.c

··· 43 43 44 44 /* 45 45 * This full barrier orders the store to the cpumask above vs 46 - * a subsequent operation which allows this CPU to begin loading 47 - * translations for next. 46 + * a subsequent load which allows this CPU/MMU to begin loading 47 + * translations for 'next' from page table PTEs into the TLB. 48 48 * 49 - * When using the radix MMU that operation is the load of the 49 + * When using the radix MMU, that operation is the load of the 50 50 * MMU context id, which is then moved to SPRN_PID. 51 51 * 52 52 * For the hash MMU it is either the first load from slb_cache 53 - * in switch_slb(), and/or the store of paca->mm_ctx_id in 54 - * copy_mm_to_paca(). 53 + * in switch_slb() to preload the SLBs, or the load of 54 + * get_user_context which loads the context for the VSID hash 55 + * to insert a new SLB, in the SLB fault handler. 55 56 * 56 57 * On the other side, the barrier is in mm/tlb-radix.c for 57 - * radix which orders earlier stores to clear the PTEs vs 58 - * the load of mm_cpumask. And pte_xchg which does the same 59 - * thing for hash. 58 + * radix which orders earlier stores to clear the PTEs before 59 + * the load of mm_cpumask to check which CPU TLBs should be 60 + * flushed. For hash, pte_xchg to clear the PTE includes the 61 + * barrier. 60 62 * 61 - * This full barrier is needed by membarrier when switching 62 - * between processes after store to rq->curr, before user-space 63 - * memory accesses. 63 + * This full barrier is also needed by membarrier when 64 + * switching between processes after store to rq->curr, before 65 + * user-space memory accesses. 64 66 */ 65 67 smp_mb(); 66 68

+5

arch/powerpc/mm/mmu_decl.h

··· 185 185 #else 186 186 static inline void ptdump_check_wx(void) { } 187 187 #endif 188 + 189 + static inline bool debug_pagealloc_enabled_or_kfence(void) 190 + { 191 + return IS_ENABLED(CONFIG_KFENCE) || debug_pagealloc_enabled(); 192 + }

+2 -2

arch/powerpc/mm/nohash/8xx.c

··· 149 149 { 150 150 unsigned long etext8 = ALIGN(__pa(_etext), SZ_8M); 151 151 unsigned long sinittext = __pa(_sinittext); 152 - bool strict_boundary = strict_kernel_rwx_enabled() || debug_pagealloc_enabled(); 152 + bool strict_boundary = strict_kernel_rwx_enabled() || debug_pagealloc_enabled_or_kfence(); 153 153 unsigned long boundary = strict_boundary ? sinittext : etext8; 154 154 unsigned long einittext8 = ALIGN(__pa(_einittext), SZ_8M); 155 155 ··· 161 161 return 0; 162 162 163 163 mmu_mapin_ram_chunk(0, boundary, PAGE_KERNEL_TEXT, true); 164 - if (debug_pagealloc_enabled()) { 164 + if (debug_pagealloc_enabled_or_kfence()) { 165 165 top = boundary; 166 166 } else { 167 167 mmu_mapin_ram_chunk(boundary, einittext8, PAGE_KERNEL_TEXT, true);

+1 -5

arch/powerpc/net/Makefile

··· 2 2 # 3 3 # Arch-specific network modules 4 4 # 5 - ifdef CONFIG_PPC64 6 - obj-$(CONFIG_BPF_JIT) += bpf_jit_comp64.o 7 - else 8 - obj-$(CONFIG_BPF_JIT) += bpf_jit_asm.o bpf_jit_comp.o 9 - endif 5 + obj-$(CONFIG_BPF_JIT) += bpf_jit_comp.o bpf_jit_comp$(BITS).o

+64

arch/powerpc/net/bpf_jit.h

··· 26 26 /* Long jump; (unconditional 'branch') */ 27 27 #define PPC_JMP(dest) EMIT(PPC_INST_BRANCH | \ 28 28 (((dest) - (ctx->idx * 4)) & 0x03fffffc)) 29 + /* blr; (unconditional 'branch' with link) to absolute address */ 30 + #define PPC_BL_ABS(dest) EMIT(PPC_INST_BL | \ 31 + (((dest) - (unsigned long)(image + ctx->idx)) & 0x03fffffc)) 29 32 /* "cond" here covers BO:BI fields. */ 30 33 #define PPC_BCC_SHORT(cond, dest) EMIT(PPC_INST_BRANCH_COND | \ 31 34 (((cond) & 0x3ff) << 16) | \ ··· 44 41 if (IMM_L(i)) \ 45 42 EMIT(PPC_RAW_ORI(d, d, IMM_L(i))); \ 46 43 } } while(0) 44 + 45 + #ifdef CONFIG_PPC32 46 + #define PPC_EX32(r, i) EMIT(PPC_RAW_LI((r), (i) < 0 ? -1 : 0)) 47 + #endif 47 48 48 49 #define PPC_LI64(d, i) do { \ 49 50 if ((long)(i) >= -2147483648 && \ ··· 114 107 #define COND_NE (CR0_EQ | COND_CMP_FALSE) 115 108 #define COND_LT (CR0_LT | COND_CMP_TRUE) 116 109 #define COND_LE (CR0_GT | COND_CMP_FALSE) 110 + 111 + #define SEEN_FUNC 0x20000000 /* might call external helpers */ 112 + #define SEEN_STACK 0x40000000 /* uses BPF stack */ 113 + #define SEEN_TAILCALL 0x80000000 /* uses tail calls */ 114 + 115 + #define SEEN_VREG_MASK 0x1ff80000 /* Volatile registers r3-r12 */ 116 + #define SEEN_NVREG_MASK 0x0003ffff /* Non volatile registers r14-r31 */ 117 + 118 + #ifdef CONFIG_PPC64 119 + extern const int b2p[MAX_BPF_JIT_REG + 2]; 120 + #else 121 + extern const int b2p[MAX_BPF_JIT_REG + 1]; 122 + #endif 123 + 124 + struct codegen_context { 125 + /* 126 + * This is used to track register usage as well 127 + * as calls to external helpers. 128 + * - register usage is tracked with corresponding 129 + * bits (r3-r31) 130 + * - rest of the bits can be used to track other 131 + * things -- for now, we use bits 0 to 2 132 + * encoded in SEEN_* macros above 133 + */ 134 + unsigned int seen; 135 + unsigned int idx; 136 + unsigned int stack_size; 137 + int b2p[ARRAY_SIZE(b2p)]; 138 + }; 139 + 140 + static inline void bpf_flush_icache(void *start, void *end) 141 + { 142 + smp_wmb(); /* smp write barrier */ 143 + flush_icache_range((unsigned long)start, (unsigned long)end); 144 + } 145 + 146 + static inline bool bpf_is_seen_register(struct codegen_context *ctx, int i) 147 + { 148 + return ctx->seen & (1 << (31 - i)); 149 + } 150 + 151 + static inline void bpf_set_seen_register(struct codegen_context *ctx, int i) 152 + { 153 + ctx->seen |= 1 << (31 - i); 154 + } 155 + 156 + static inline void bpf_clear_seen_register(struct codegen_context *ctx, int i) 157 + { 158 + ctx->seen &= ~(1 << (31 - i)); 159 + } 160 + 161 + void bpf_jit_emit_func_call_rel(u32 *image, struct codegen_context *ctx, u64 func); 162 + int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, struct codegen_context *ctx, 163 + u32 *addrs, bool extra_pass); 164 + void bpf_jit_build_prologue(u32 *image, struct codegen_context *ctx); 165 + void bpf_jit_build_epilogue(u32 *image, struct codegen_context *ctx); 166 + void bpf_jit_realloc_regs(struct codegen_context *ctx); 117 167 118 168 #endif 119 169

-139

arch/powerpc/net/bpf_jit32.h

··· 1 - /* SPDX-License-Identifier: GPL-2.0-only */ 2 - /* 3 - * bpf_jit32.h: BPF JIT compiler for PPC 4 - * 5 - * Copyright 2011 Matt Evans <matt@ozlabs.org>, IBM Corporation 6 - * 7 - * Split from bpf_jit.h 8 - */ 9 - #ifndef _BPF_JIT32_H 10 - #define _BPF_JIT32_H 11 - 12 - #include <asm/asm-compat.h> 13 - #include "bpf_jit.h" 14 - 15 - #ifdef CONFIG_PPC64 16 - #define BPF_PPC_STACK_R3_OFF 48 17 - #define BPF_PPC_STACK_LOCALS 32 18 - #define BPF_PPC_STACK_BASIC (48+64) 19 - #define BPF_PPC_STACK_SAVE (18*8) 20 - #define BPF_PPC_STACKFRAME (BPF_PPC_STACK_BASIC+BPF_PPC_STACK_LOCALS+ \ 21 - BPF_PPC_STACK_SAVE) 22 - #define BPF_PPC_SLOWPATH_FRAME (48+64) 23 - #else 24 - #define BPF_PPC_STACK_R3_OFF 24 25 - #define BPF_PPC_STACK_LOCALS 16 26 - #define BPF_PPC_STACK_BASIC (24+32) 27 - #define BPF_PPC_STACK_SAVE (18*4) 28 - #define BPF_PPC_STACKFRAME (BPF_PPC_STACK_BASIC+BPF_PPC_STACK_LOCALS+ \ 29 - BPF_PPC_STACK_SAVE) 30 - #define BPF_PPC_SLOWPATH_FRAME (24+32) 31 - #endif 32 - 33 - #define REG_SZ (BITS_PER_LONG/8) 34 - 35 - /* 36 - * Generated code register usage: 37 - * 38 - * As normal PPC C ABI (e.g. r1=sp, r2=TOC), with: 39 - * 40 - * skb r3 (Entry parameter) 41 - * A register r4 42 - * X register r5 43 - * addr param r6 44 - * r7-r10 scratch 45 - * skb->data r14 46 - * skb headlen r15 (skb->len - skb->data_len) 47 - * m[0] r16 48 - * m[...] ... 49 - * m[15] r31 50 - */ 51 - #define r_skb 3 52 - #define r_ret 3 53 - #define r_A 4 54 - #define r_X 5 55 - #define r_addr 6 56 - #define r_scratch1 7 57 - #define r_scratch2 8 58 - #define r_D 14 59 - #define r_HL 15 60 - #define r_M 16 61 - 62 - #ifndef __ASSEMBLY__ 63 - 64 - /* 65 - * Assembly helpers from arch/powerpc/net/bpf_jit.S: 66 - */ 67 - #define DECLARE_LOAD_FUNC(func) \ 68 - extern u8 func[], func##_negative_offset[], func##_positive_offset[] 69 - 70 - DECLARE_LOAD_FUNC(sk_load_word); 71 - DECLARE_LOAD_FUNC(sk_load_half); 72 - DECLARE_LOAD_FUNC(sk_load_byte); 73 - DECLARE_LOAD_FUNC(sk_load_byte_msh); 74 - 75 - #define PPC_LBZ_OFFS(r, base, i) do { if ((i) < 32768) EMIT(PPC_RAW_LBZ(r, base, i)); \ 76 - else { EMIT(PPC_RAW_ADDIS(r, base, IMM_HA(i))); \ 77 - EMIT(PPC_RAW_LBZ(r, r, IMM_L(i))); } } while(0) 78 - 79 - #define PPC_LD_OFFS(r, base, i) do { if ((i) < 32768) EMIT(PPC_RAW_LD(r, base, i)); \ 80 - else { EMIT(PPC_RAW_ADDIS(r, base, IMM_HA(i))); \ 81 - EMIT(PPC_RAW_LD(r, r, IMM_L(i))); } } while(0) 82 - 83 - #define PPC_LWZ_OFFS(r, base, i) do { if ((i) < 32768) EMIT(PPC_RAW_LWZ(r, base, i)); \ 84 - else { EMIT(PPC_RAW_ADDIS(r, base, IMM_HA(i))); \ 85 - EMIT(PPC_RAW_LWZ(r, r, IMM_L(i))); } } while(0) 86 - 87 - #define PPC_LHZ_OFFS(r, base, i) do { if ((i) < 32768) EMIT(PPC_RAW_LHZ(r, base, i)); \ 88 - else { EMIT(PPC_RAW_ADDIS(r, base, IMM_HA(i))); \ 89 - EMIT(PPC_RAW_LHZ(r, r, IMM_L(i))); } } while(0) 90 - 91 - #ifdef CONFIG_PPC64 92 - #define PPC_LL_OFFS(r, base, i) do { PPC_LD_OFFS(r, base, i); } while(0) 93 - #else 94 - #define PPC_LL_OFFS(r, base, i) do { PPC_LWZ_OFFS(r, base, i); } while(0) 95 - #endif 96 - 97 - #ifdef CONFIG_SMP 98 - #ifdef CONFIG_PPC64 99 - #define PPC_BPF_LOAD_CPU(r) \ 100 - do { BUILD_BUG_ON(sizeof_field(struct paca_struct, paca_index) != 2); \ 101 - PPC_LHZ_OFFS(r, 13, offsetof(struct paca_struct, paca_index)); \ 102 - } while (0) 103 - #else 104 - #define PPC_BPF_LOAD_CPU(r) \ 105 - do { BUILD_BUG_ON(sizeof_field(struct task_struct, cpu) != 4); \ 106 - PPC_LHZ_OFFS(r, 2, offsetof(struct task_struct, cpu)); \ 107 - } while(0) 108 - #endif 109 - #else 110 - #define PPC_BPF_LOAD_CPU(r) do { EMIT(PPC_RAW_LI(r, 0)); } while(0) 111 - #endif 112 - 113 - #define PPC_LHBRX_OFFS(r, base, i) \ 114 - do { PPC_LI32(r, i); EMIT(PPC_RAW_LHBRX(r, r, base)); } while(0) 115 - #ifdef __LITTLE_ENDIAN__ 116 - #define PPC_NTOHS_OFFS(r, base, i) PPC_LHBRX_OFFS(r, base, i) 117 - #else 118 - #define PPC_NTOHS_OFFS(r, base, i) PPC_LHZ_OFFS(r, base, i) 119 - #endif 120 - 121 - #define PPC_BPF_LL(r, base, i) do { EMIT(PPC_RAW_LWZ(r, base, i)); } while(0) 122 - #define PPC_BPF_STL(r, base, i) do { EMIT(PPC_RAW_STW(r, base, i)); } while(0) 123 - #define PPC_BPF_STLU(r, base, i) do { EMIT(PPC_RAW_STWU(r, base, i)); } while(0) 124 - 125 - #define SEEN_DATAREF 0x10000 /* might call external helpers */ 126 - #define SEEN_XREG 0x20000 /* X reg is used */ 127 - #define SEEN_MEM 0x40000 /* SEEN_MEM+(1<<n) = use mem[n] for temporary 128 - * storage */ 129 - #define SEEN_MEM_MSK 0x0ffff 130 - 131 - struct codegen_context { 132 - unsigned int seen; 133 - unsigned int idx; 134 - int pc_ret0; /* bpf index of first RET #0 instruction (if any) */ 135 - }; 136 - 137 - #endif 138 - 139 - #endif

+1 -20

arch/powerpc/net/bpf_jit64.h

··· 39 39 #define TMP_REG_2 (MAX_BPF_JIT_REG + 1) 40 40 41 41 /* BPF to ppc register mappings */ 42 - static const int b2p[] = { 42 + const int b2p[MAX_BPF_JIT_REG + 2] = { 43 43 /* function return value */ 44 44 [BPF_REG_0] = 8, 45 45 /* function arguments */ ··· 85 85 EMIT(PPC_RAW_STD(r, base, i)); \ 86 86 } while(0) 87 87 #define PPC_BPF_STLU(r, base, i) do { EMIT(PPC_RAW_STDU(r, base, i)); } while(0) 88 - 89 - #define SEEN_FUNC 0x1000 /* might call external helpers */ 90 - #define SEEN_STACK 0x2000 /* uses BPF stack */ 91 - #define SEEN_TAILCALL 0x4000 /* uses tail calls */ 92 - 93 - struct codegen_context { 94 - /* 95 - * This is used to track register usage as well 96 - * as calls to external helpers. 97 - * - register usage is tracked with corresponding 98 - * bits (r3-r10 and r27-r31) 99 - * - rest of the bits can be used to track other 100 - * things -- for now, we use bits 16 to 23 101 - * encoded in SEEN_* macros above 102 - */ 103 - unsigned int seen; 104 - unsigned int idx; 105 - unsigned int stack_size; 106 - }; 107 88 108 89 #endif /* !__ASSEMBLY__ */ 109 90

-226

arch/powerpc/net/bpf_jit_asm.S

··· 1 - /* SPDX-License-Identifier: GPL-2.0-only */ 2 - /* bpf_jit.S: Packet/header access helper functions 3 - * for PPC64 BPF compiler. 4 - * 5 - * Copyright 2011 Matt Evans <matt@ozlabs.org>, IBM Corporation 6 - */ 7 - 8 - #include <asm/ppc_asm.h> 9 - #include <asm/asm-compat.h> 10 - #include "bpf_jit32.h" 11 - 12 - /* 13 - * All of these routines are called directly from generated code, 14 - * whose register usage is: 15 - * 16 - * r3 skb 17 - * r4,r5 A,X 18 - * r6 *** address parameter to helper *** 19 - * r7-r10 scratch 20 - * r14 skb->data 21 - * r15 skb headlen 22 - * r16-31 M[] 23 - */ 24 - 25 - /* 26 - * To consider: These helpers are so small it could be better to just 27 - * generate them inline. Inline code can do the simple headlen check 28 - * then branch directly to slow_path_XXX if required. (In fact, could 29 - * load a spare GPR with the address of slow_path_generic and pass size 30 - * as an argument, making the call site a mtlr, li and bllr.) 31 - */ 32 - .globl sk_load_word 33 - sk_load_word: 34 - PPC_LCMPI r_addr, 0 35 - blt bpf_slow_path_word_neg 36 - .globl sk_load_word_positive_offset 37 - sk_load_word_positive_offset: 38 - /* Are we accessing past headlen? */ 39 - subi r_scratch1, r_HL, 4 40 - PPC_LCMP r_scratch1, r_addr 41 - blt bpf_slow_path_word 42 - /* Nope, just hitting the header. cr0 here is eq or gt! */ 43 - #ifdef __LITTLE_ENDIAN__ 44 - lwbrx r_A, r_D, r_addr 45 - #else 46 - lwzx r_A, r_D, r_addr 47 - #endif 48 - blr /* Return success, cr0 != LT */ 49 - 50 - .globl sk_load_half 51 - sk_load_half: 52 - PPC_LCMPI r_addr, 0 53 - blt bpf_slow_path_half_neg 54 - .globl sk_load_half_positive_offset 55 - sk_load_half_positive_offset: 56 - subi r_scratch1, r_HL, 2 57 - PPC_LCMP r_scratch1, r_addr 58 - blt bpf_slow_path_half 59 - #ifdef __LITTLE_ENDIAN__ 60 - lhbrx r_A, r_D, r_addr 61 - #else 62 - lhzx r_A, r_D, r_addr 63 - #endif 64 - blr 65 - 66 - .globl sk_load_byte 67 - sk_load_byte: 68 - PPC_LCMPI r_addr, 0 69 - blt bpf_slow_path_byte_neg 70 - .globl sk_load_byte_positive_offset 71 - sk_load_byte_positive_offset: 72 - PPC_LCMP r_HL, r_addr 73 - ble bpf_slow_path_byte 74 - lbzx r_A, r_D, r_addr 75 - blr 76 - 77 - /* 78 - * BPF_LDX | BPF_B | BPF_MSH: ldxb 4*([offset]&0xf) 79 - * r_addr is the offset value 80 - */ 81 - .globl sk_load_byte_msh 82 - sk_load_byte_msh: 83 - PPC_LCMPI r_addr, 0 84 - blt bpf_slow_path_byte_msh_neg 85 - .globl sk_load_byte_msh_positive_offset 86 - sk_load_byte_msh_positive_offset: 87 - PPC_LCMP r_HL, r_addr 88 - ble bpf_slow_path_byte_msh 89 - lbzx r_X, r_D, r_addr 90 - rlwinm r_X, r_X, 2, 32-4-2, 31-2 91 - blr 92 - 93 - /* Call out to skb_copy_bits: 94 - * We'll need to back up our volatile regs first; we have 95 - * local variable space at r1+(BPF_PPC_STACK_BASIC). 96 - * Allocate a new stack frame here to remain ABI-compliant in 97 - * stashing LR. 98 - */ 99 - #define bpf_slow_path_common(SIZE) \ 100 - mflr r0; \ 101 - PPC_STL r0, PPC_LR_STKOFF(r1); \ 102 - /* R3 goes in parameter space of caller's frame */ \ 103 - PPC_STL r_skb, (BPF_PPC_STACKFRAME+BPF_PPC_STACK_R3_OFF)(r1); \ 104 - PPC_STL r_A, (BPF_PPC_STACK_BASIC+(0*REG_SZ))(r1); \ 105 - PPC_STL r_X, (BPF_PPC_STACK_BASIC+(1*REG_SZ))(r1); \ 106 - addi r5, r1, BPF_PPC_STACK_BASIC+(2*REG_SZ); \ 107 - PPC_STLU r1, -BPF_PPC_SLOWPATH_FRAME(r1); \ 108 - /* R3 = r_skb, as passed */ \ 109 - mr r4, r_addr; \ 110 - li r6, SIZE; \ 111 - bl skb_copy_bits; \ 112 - nop; \ 113 - /* R3 = 0 on success */ \ 114 - addi r1, r1, BPF_PPC_SLOWPATH_FRAME; \ 115 - PPC_LL r0, PPC_LR_STKOFF(r1); \ 116 - PPC_LL r_A, (BPF_PPC_STACK_BASIC+(0*REG_SZ))(r1); \ 117 - PPC_LL r_X, (BPF_PPC_STACK_BASIC+(1*REG_SZ))(r1); \ 118 - mtlr r0; \ 119 - PPC_LCMPI r3, 0; \ 120 - blt bpf_error; /* cr0 = LT */ \ 121 - PPC_LL r_skb, (BPF_PPC_STACKFRAME+BPF_PPC_STACK_R3_OFF)(r1); \ 122 - /* Great success! */ 123 - 124 - bpf_slow_path_word: 125 - bpf_slow_path_common(4) 126 - /* Data value is on stack, and cr0 != LT */ 127 - lwz r_A, BPF_PPC_STACK_BASIC+(2*REG_SZ)(r1) 128 - blr 129 - 130 - bpf_slow_path_half: 131 - bpf_slow_path_common(2) 132 - lhz r_A, BPF_PPC_STACK_BASIC+(2*8)(r1) 133 - blr 134 - 135 - bpf_slow_path_byte: 136 - bpf_slow_path_common(1) 137 - lbz r_A, BPF_PPC_STACK_BASIC+(2*8)(r1) 138 - blr 139 - 140 - bpf_slow_path_byte_msh: 141 - bpf_slow_path_common(1) 142 - lbz r_X, BPF_PPC_STACK_BASIC+(2*8)(r1) 143 - rlwinm r_X, r_X, 2, 32-4-2, 31-2 144 - blr 145 - 146 - /* Call out to bpf_internal_load_pointer_neg_helper: 147 - * We'll need to back up our volatile regs first; we have 148 - * local variable space at r1+(BPF_PPC_STACK_BASIC). 149 - * Allocate a new stack frame here to remain ABI-compliant in 150 - * stashing LR. 151 - */ 152 - #define sk_negative_common(SIZE) \ 153 - mflr r0; \ 154 - PPC_STL r0, PPC_LR_STKOFF(r1); \ 155 - /* R3 goes in parameter space of caller's frame */ \ 156 - PPC_STL r_skb, (BPF_PPC_STACKFRAME+BPF_PPC_STACK_R3_OFF)(r1); \ 157 - PPC_STL r_A, (BPF_PPC_STACK_BASIC+(0*REG_SZ))(r1); \ 158 - PPC_STL r_X, (BPF_PPC_STACK_BASIC+(1*REG_SZ))(r1); \ 159 - PPC_STLU r1, -BPF_PPC_SLOWPATH_FRAME(r1); \ 160 - /* R3 = r_skb, as passed */ \ 161 - mr r4, r_addr; \ 162 - li r5, SIZE; \ 163 - bl bpf_internal_load_pointer_neg_helper; \ 164 - nop; \ 165 - /* R3 != 0 on success */ \ 166 - addi r1, r1, BPF_PPC_SLOWPATH_FRAME; \ 167 - PPC_LL r0, PPC_LR_STKOFF(r1); \ 168 - PPC_LL r_A, (BPF_PPC_STACK_BASIC+(0*REG_SZ))(r1); \ 169 - PPC_LL r_X, (BPF_PPC_STACK_BASIC+(1*REG_SZ))(r1); \ 170 - mtlr r0; \ 171 - PPC_LCMPLI r3, 0; \ 172 - beq bpf_error_slow; /* cr0 = EQ */ \ 173 - mr r_addr, r3; \ 174 - PPC_LL r_skb, (BPF_PPC_STACKFRAME+BPF_PPC_STACK_R3_OFF)(r1); \ 175 - /* Great success! */ 176 - 177 - bpf_slow_path_word_neg: 178 - lis r_scratch1,-32 /* SKF_LL_OFF */ 179 - PPC_LCMP r_addr, r_scratch1 /* addr < SKF_* */ 180 - blt bpf_error /* cr0 = LT */ 181 - .globl sk_load_word_negative_offset 182 - sk_load_word_negative_offset: 183 - sk_negative_common(4) 184 - lwz r_A, 0(r_addr) 185 - blr 186 - 187 - bpf_slow_path_half_neg: 188 - lis r_scratch1,-32 /* SKF_LL_OFF */ 189 - PPC_LCMP r_addr, r_scratch1 /* addr < SKF_* */ 190 - blt bpf_error /* cr0 = LT */ 191 - .globl sk_load_half_negative_offset 192 - sk_load_half_negative_offset: 193 - sk_negative_common(2) 194 - lhz r_A, 0(r_addr) 195 - blr 196 - 197 - bpf_slow_path_byte_neg: 198 - lis r_scratch1,-32 /* SKF_LL_OFF */ 199 - PPC_LCMP r_addr, r_scratch1 /* addr < SKF_* */ 200 - blt bpf_error /* cr0 = LT */ 201 - .globl sk_load_byte_negative_offset 202 - sk_load_byte_negative_offset: 203 - sk_negative_common(1) 204 - lbz r_A, 0(r_addr) 205 - blr 206 - 207 - bpf_slow_path_byte_msh_neg: 208 - lis r_scratch1,-32 /* SKF_LL_OFF */ 209 - PPC_LCMP r_addr, r_scratch1 /* addr < SKF_* */ 210 - blt bpf_error /* cr0 = LT */ 211 - .globl sk_load_byte_msh_negative_offset 212 - sk_load_byte_msh_negative_offset: 213 - sk_negative_common(1) 214 - lbz r_X, 0(r_addr) 215 - rlwinm r_X, r_X, 2, 32-4-2, 31-2 216 - blr 217 - 218 - bpf_error_slow: 219 - /* fabricate a cr0 = lt */ 220 - li r_scratch1, -1 221 - PPC_LCMPI r_scratch1, 0 222 - bpf_error: 223 - /* Entered with cr0 = lt */ 224 - li r3, 0 225 - /* Generated code will 'blt epilogue', returning 0. */ 226 - blr

+189 -601

arch/powerpc/net/bpf_jit_comp.c

··· 1 1 // SPDX-License-Identifier: GPL-2.0-only 2 - /* bpf_jit_comp.c: BPF JIT compiler 2 + /* 3 + * eBPF JIT compiler 3 4 * 4 - * Copyright 2011 Matt Evans <matt@ozlabs.org>, IBM Corporation 5 + * Copyright 2016 Naveen N. Rao <naveen.n.rao@linux.vnet.ibm.com> 6 + * IBM Corporation 5 7 * 6 - * Based on the x86 BPF compiler, by Eric Dumazet (eric.dumazet@gmail.com) 7 - * Ported to ppc32 by Denis Kirjanov <kda@linux-powerpc.org> 8 + * Based on the powerpc classic BPF JIT compiler by Matt Evans 8 9 */ 9 10 #include <linux/moduleloader.h> 10 11 #include <asm/cacheflush.h> ··· 13 12 #include <linux/netdevice.h> 14 13 #include <linux/filter.h> 15 14 #include <linux/if_vlan.h> 15 + #include <asm/kprobes.h> 16 + #include <linux/bpf.h> 16 17 17 - #include "bpf_jit32.h" 18 + #include "bpf_jit.h" 18 19 19 - static inline void bpf_flush_icache(void *start, void *end) 20 + static void bpf_jit_fill_ill_insns(void *area, unsigned int size) 20 21 { 21 - smp_wmb(); 22 - flush_icache_range((unsigned long)start, (unsigned long)end); 22 + memset32(area, BREAKPOINT_INSTRUCTION, size / 4); 23 23 } 24 24 25 - static void bpf_jit_build_prologue(struct bpf_prog *fp, u32 *image, 26 - struct codegen_context *ctx) 25 + /* Fix the branch target addresses for subprog calls */ 26 + static int bpf_jit_fixup_subprog_calls(struct bpf_prog *fp, u32 *image, 27 + struct codegen_context *ctx, u32 *addrs) 27 28 { 28 - int i; 29 - const struct sock_filter *filter = fp->insns; 29 + const struct bpf_insn *insn = fp->insnsi; 30 + bool func_addr_fixed; 31 + u64 func_addr; 32 + u32 tmp_idx; 33 + int i, ret; 30 34 31 - if (ctx->seen & (SEEN_MEM | SEEN_DATAREF)) { 32 - /* Make stackframe */ 33 - if (ctx->seen & SEEN_DATAREF) { 34 - /* If we call any helpers (for loads), save LR */ 35 - EMIT(PPC_INST_MFLR | __PPC_RT(R0)); 36 - PPC_BPF_STL(0, 1, PPC_LR_STKOFF); 37 - 38 - /* Back up non-volatile regs. */ 39 - PPC_BPF_STL(r_D, 1, -(REG_SZ*(32-r_D))); 40 - PPC_BPF_STL(r_HL, 1, -(REG_SZ*(32-r_HL))); 41 - } 42 - if (ctx->seen & SEEN_MEM) { 43 - /* 44 - * Conditionally save regs r15-r31 as some will be used 45 - * for M[] data. 46 - */ 47 - for (i = r_M; i < (r_M+16); i++) { 48 - if (ctx->seen & (1 << (i-r_M))) 49 - PPC_BPF_STL(i, 1, -(REG_SZ*(32-i))); 50 - } 51 - } 52 - PPC_BPF_STLU(1, 1, -BPF_PPC_STACKFRAME); 53 - } 54 - 55 - if (ctx->seen & SEEN_DATAREF) { 35 + for (i = 0; i < fp->len; i++) { 56 36 /* 57 - * If this filter needs to access skb data, 58 - * prepare r_D and r_HL: 59 - * r_HL = skb->len - skb->data_len 60 - * r_D = skb->data 37 + * During the extra pass, only the branch target addresses for 38 + * the subprog calls need to be fixed. All other instructions 39 + * can left untouched. 40 + * 41 + * The JITed image length does not change because we already 42 + * ensure that the JITed instruction sequence for these calls 43 + * are of fixed length by padding them with NOPs. 61 44 */ 62 - PPC_LWZ_OFFS(r_scratch1, r_skb, offsetof(struct sk_buff, 63 - data_len)); 64 - PPC_LWZ_OFFS(r_HL, r_skb, offsetof(struct sk_buff, len)); 65 - EMIT(PPC_RAW_SUB(r_HL, r_HL, r_scratch1)); 66 - PPC_LL_OFFS(r_D, r_skb, offsetof(struct sk_buff, data)); 67 - } 45 + if (insn[i].code == (BPF_JMP | BPF_CALL) && 46 + insn[i].src_reg == BPF_PSEUDO_CALL) { 47 + ret = bpf_jit_get_func_addr(fp, &insn[i], true, 48 + &func_addr, 49 + &func_addr_fixed); 50 + if (ret < 0) 51 + return ret; 68 52 69 - if (ctx->seen & SEEN_XREG) { 70 - /* 71 - * TODO: Could also detect whether first instr. sets X and 72 - * avoid this (as below, with A). 73 - */ 74 - EMIT(PPC_RAW_LI(r_X, 0)); 75 - } 53 + /* 54 + * Save ctx->idx as this would currently point to the 55 + * end of the JITed image and set it to the offset of 56 + * the instruction sequence corresponding to the 57 + * subprog call temporarily. 58 + */ 59 + tmp_idx = ctx->idx; 60 + ctx->idx = addrs[i] / 4; 61 + bpf_jit_emit_func_call_rel(image, ctx, func_addr); 76 62 77 - /* make sure we dont leak kernel information to user */ 78 - if (bpf_needs_clear_a(&filter[0])) 79 - EMIT(PPC_RAW_LI(r_A, 0)); 80 - } 81 - 82 - static void bpf_jit_build_epilogue(u32 *image, struct codegen_context *ctx) 83 - { 84 - int i; 85 - 86 - if (ctx->seen & (SEEN_MEM | SEEN_DATAREF)) { 87 - EMIT(PPC_RAW_ADDI(1, 1, BPF_PPC_STACKFRAME)); 88 - if (ctx->seen & SEEN_DATAREF) { 89 - PPC_BPF_LL(0, 1, PPC_LR_STKOFF); 90 - EMIT(PPC_RAW_MTLR(0)); 91 - PPC_BPF_LL(r_D, 1, -(REG_SZ*(32-r_D))); 92 - PPC_BPF_LL(r_HL, 1, -(REG_SZ*(32-r_HL))); 93 - } 94 - if (ctx->seen & SEEN_MEM) { 95 - /* Restore any saved non-vol registers */ 96 - for (i = r_M; i < (r_M+16); i++) { 97 - if (ctx->seen & (1 << (i-r_M))) 98 - PPC_BPF_LL(i, 1, -(REG_SZ*(32-i))); 99 - } 63 + /* 64 + * Restore ctx->idx here. This is safe as the length 65 + * of the JITed sequence remains unchanged. 66 + */ 67 + ctx->idx = tmp_idx; 100 68 } 101 69 } 102 - /* The RETs have left a return value in R3. */ 103 - 104 - EMIT(PPC_RAW_BLR()); 105 - } 106 - 107 - #define CHOOSE_LOAD_FUNC(K, func) \ 108 - ((int)K < 0 ? ((int)K >= SKF_LL_OFF ? func##_negative_offset : func) : func##_positive_offset) 109 - 110 - /* Assemble the body code between the prologue & epilogue. */ 111 - static int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, 112 - struct codegen_context *ctx, 113 - unsigned int *addrs) 114 - { 115 - const struct sock_filter *filter = fp->insns; 116 - int flen = fp->len; 117 - u8 *func; 118 - unsigned int true_cond; 119 - int i; 120 - 121 - /* Start of epilogue code */ 122 - unsigned int exit_addr = addrs[flen]; 123 - 124 - for (i = 0; i < flen; i++) { 125 - unsigned int K = filter[i].k; 126 - u16 code = bpf_anc_helper(&filter[i]); 127 - 128 - /* 129 - * addrs[] maps a BPF bytecode address into a real offset from 130 - * the start of the body code. 131 - */ 132 - addrs[i] = ctx->idx * 4; 133 - 134 - switch (code) { 135 - /*** ALU ops ***/ 136 - case BPF_ALU | BPF_ADD | BPF_X: /* A += X; */ 137 - ctx->seen |= SEEN_XREG; 138 - EMIT(PPC_RAW_ADD(r_A, r_A, r_X)); 139 - break; 140 - case BPF_ALU | BPF_ADD | BPF_K: /* A += K; */ 141 - if (!K) 142 - break; 143 - EMIT(PPC_RAW_ADDI(r_A, r_A, IMM_L(K))); 144 - if (K >= 32768) 145 - EMIT(PPC_RAW_ADDIS(r_A, r_A, IMM_HA(K))); 146 - break; 147 - case BPF_ALU | BPF_SUB | BPF_X: /* A -= X; */ 148 - ctx->seen |= SEEN_XREG; 149 - EMIT(PPC_RAW_SUB(r_A, r_A, r_X)); 150 - break; 151 - case BPF_ALU | BPF_SUB | BPF_K: /* A -= K */ 152 - if (!K) 153 - break; 154 - EMIT(PPC_RAW_ADDI(r_A, r_A, IMM_L(-K))); 155 - if (K >= 32768) 156 - EMIT(PPC_RAW_ADDIS(r_A, r_A, IMM_HA(-K))); 157 - break; 158 - case BPF_ALU | BPF_MUL | BPF_X: /* A *= X; */ 159 - ctx->seen |= SEEN_XREG; 160 - EMIT(PPC_RAW_MULW(r_A, r_A, r_X)); 161 - break; 162 - case BPF_ALU | BPF_MUL | BPF_K: /* A *= K */ 163 - if (K < 32768) 164 - EMIT(PPC_RAW_MULI(r_A, r_A, K)); 165 - else { 166 - PPC_LI32(r_scratch1, K); 167 - EMIT(PPC_RAW_MULW(r_A, r_A, r_scratch1)); 168 - } 169 - break; 170 - case BPF_ALU | BPF_MOD | BPF_X: /* A %= X; */ 171 - case BPF_ALU | BPF_DIV | BPF_X: /* A /= X; */ 172 - ctx->seen |= SEEN_XREG; 173 - EMIT(PPC_RAW_CMPWI(r_X, 0)); 174 - if (ctx->pc_ret0 != -1) { 175 - PPC_BCC(COND_EQ, addrs[ctx->pc_ret0]); 176 - } else { 177 - PPC_BCC_SHORT(COND_NE, (ctx->idx*4)+12); 178 - EMIT(PPC_RAW_LI(r_ret, 0)); 179 - PPC_JMP(exit_addr); 180 - } 181 - if (code == (BPF_ALU | BPF_MOD | BPF_X)) { 182 - EMIT(PPC_RAW_DIVWU(r_scratch1, r_A, r_X)); 183 - EMIT(PPC_RAW_MULW(r_scratch1, r_X, r_scratch1)); 184 - EMIT(PPC_RAW_SUB(r_A, r_A, r_scratch1)); 185 - } else { 186 - EMIT(PPC_RAW_DIVWU(r_A, r_A, r_X)); 187 - } 188 - break; 189 - case BPF_ALU | BPF_MOD | BPF_K: /* A %= K; */ 190 - PPC_LI32(r_scratch2, K); 191 - EMIT(PPC_RAW_DIVWU(r_scratch1, r_A, r_scratch2)); 192 - EMIT(PPC_RAW_MULW(r_scratch1, r_scratch2, r_scratch1)); 193 - EMIT(PPC_RAW_SUB(r_A, r_A, r_scratch1)); 194 - break; 195 - case BPF_ALU | BPF_DIV | BPF_K: /* A /= K */ 196 - if (K == 1) 197 - break; 198 - PPC_LI32(r_scratch1, K); 199 - EMIT(PPC_RAW_DIVWU(r_A, r_A, r_scratch1)); 200 - break; 201 - case BPF_ALU | BPF_AND | BPF_X: 202 - ctx->seen |= SEEN_XREG; 203 - EMIT(PPC_RAW_AND(r_A, r_A, r_X)); 204 - break; 205 - case BPF_ALU | BPF_AND | BPF_K: 206 - if (!IMM_H(K)) 207 - EMIT(PPC_RAW_ANDI(r_A, r_A, K)); 208 - else { 209 - PPC_LI32(r_scratch1, K); 210 - EMIT(PPC_RAW_AND(r_A, r_A, r_scratch1)); 211 - } 212 - break; 213 - case BPF_ALU | BPF_OR | BPF_X: 214 - ctx->seen |= SEEN_XREG; 215 - EMIT(PPC_RAW_OR(r_A, r_A, r_X)); 216 - break; 217 - case BPF_ALU | BPF_OR | BPF_K: 218 - if (IMM_L(K)) 219 - EMIT(PPC_RAW_ORI(r_A, r_A, IMM_L(K))); 220 - if (K >= 65536) 221 - EMIT(PPC_RAW_ORIS(r_A, r_A, IMM_H(K))); 222 - break; 223 - case BPF_ANC | SKF_AD_ALU_XOR_X: 224 - case BPF_ALU | BPF_XOR | BPF_X: /* A ^= X */ 225 - ctx->seen |= SEEN_XREG; 226 - EMIT(PPC_RAW_XOR(r_A, r_A, r_X)); 227 - break; 228 - case BPF_ALU | BPF_XOR | BPF_K: /* A ^= K */ 229 - if (IMM_L(K)) 230 - EMIT(PPC_RAW_XORI(r_A, r_A, IMM_L(K))); 231 - if (K >= 65536) 232 - EMIT(PPC_RAW_XORIS(r_A, r_A, IMM_H(K))); 233 - break; 234 - case BPF_ALU | BPF_LSH | BPF_X: /* A <<= X; */ 235 - ctx->seen |= SEEN_XREG; 236 - EMIT(PPC_RAW_SLW(r_A, r_A, r_X)); 237 - break; 238 - case BPF_ALU | BPF_LSH | BPF_K: 239 - if (K == 0) 240 - break; 241 - else 242 - EMIT(PPC_RAW_SLWI(r_A, r_A, K)); 243 - break; 244 - case BPF_ALU | BPF_RSH | BPF_X: /* A >>= X; */ 245 - ctx->seen |= SEEN_XREG; 246 - EMIT(PPC_RAW_SRW(r_A, r_A, r_X)); 247 - break; 248 - case BPF_ALU | BPF_RSH | BPF_K: /* A >>= K; */ 249 - if (K == 0) 250 - break; 251 - else 252 - EMIT(PPC_RAW_SRWI(r_A, r_A, K)); 253 - break; 254 - case BPF_ALU | BPF_NEG: 255 - EMIT(PPC_RAW_NEG(r_A, r_A)); 256 - break; 257 - case BPF_RET | BPF_K: 258 - PPC_LI32(r_ret, K); 259 - if (!K) { 260 - if (ctx->pc_ret0 == -1) 261 - ctx->pc_ret0 = i; 262 - } 263 - /* 264 - * If this isn't the very last instruction, branch to 265 - * the epilogue if we've stuff to clean up. Otherwise, 266 - * if there's nothing to tidy, just return. If we /are/ 267 - * the last instruction, we're about to fall through to 268 - * the epilogue to return. 269 - */ 270 - if (i != flen - 1) { 271 - /* 272 - * Note: 'seen' is properly valid only on pass 273 - * #2. Both parts of this conditional are the 274 - * same instruction size though, meaning the 275 - * first pass will still correctly determine the 276 - * code size/addresses. 277 - */ 278 - if (ctx->seen) 279 - PPC_JMP(exit_addr); 280 - else 281 - EMIT(PPC_RAW_BLR()); 282 - } 283 - break; 284 - case BPF_RET | BPF_A: 285 - EMIT(PPC_RAW_MR(r_ret, r_A)); 286 - if (i != flen - 1) { 287 - if (ctx->seen) 288 - PPC_JMP(exit_addr); 289 - else 290 - EMIT(PPC_RAW_BLR()); 291 - } 292 - break; 293 - case BPF_MISC | BPF_TAX: /* X = A */ 294 - EMIT(PPC_RAW_MR(r_X, r_A)); 295 - break; 296 - case BPF_MISC | BPF_TXA: /* A = X */ 297 - ctx->seen |= SEEN_XREG; 298 - EMIT(PPC_RAW_MR(r_A, r_X)); 299 - break; 300 - 301 - /*** Constant loads/M[] access ***/ 302 - case BPF_LD | BPF_IMM: /* A = K */ 303 - PPC_LI32(r_A, K); 304 - break; 305 - case BPF_LDX | BPF_IMM: /* X = K */ 306 - PPC_LI32(r_X, K); 307 - break; 308 - case BPF_LD | BPF_MEM: /* A = mem[K] */ 309 - EMIT(PPC_RAW_MR(r_A, r_M + (K & 0xf))); 310 - ctx->seen |= SEEN_MEM | (1<<(K & 0xf)); 311 - break; 312 - case BPF_LDX | BPF_MEM: /* X = mem[K] */ 313 - EMIT(PPC_RAW_MR(r_X, r_M + (K & 0xf))); 314 - ctx->seen |= SEEN_MEM | (1<<(K & 0xf)); 315 - break; 316 - case BPF_ST: /* mem[K] = A */ 317 - EMIT(PPC_RAW_MR(r_M + (K & 0xf), r_A)); 318 - ctx->seen |= SEEN_MEM | (1<<(K & 0xf)); 319 - break; 320 - case BPF_STX: /* mem[K] = X */ 321 - EMIT(PPC_RAW_MR(r_M + (K & 0xf), r_X)); 322 - ctx->seen |= SEEN_XREG | SEEN_MEM | (1<<(K & 0xf)); 323 - break; 324 - case BPF_LD | BPF_W | BPF_LEN: /* A = skb->len; */ 325 - BUILD_BUG_ON(sizeof_field(struct sk_buff, len) != 4); 326 - PPC_LWZ_OFFS(r_A, r_skb, offsetof(struct sk_buff, len)); 327 - break; 328 - case BPF_LDX | BPF_W | BPF_ABS: /* A = *((u32 *)(seccomp_data + K)); */ 329 - PPC_LWZ_OFFS(r_A, r_skb, K); 330 - break; 331 - case BPF_LDX | BPF_W | BPF_LEN: /* X = skb->len; */ 332 - PPC_LWZ_OFFS(r_X, r_skb, offsetof(struct sk_buff, len)); 333 - break; 334 - 335 - /*** Ancillary info loads ***/ 336 - case BPF_ANC | SKF_AD_PROTOCOL: /* A = ntohs(skb->protocol); */ 337 - BUILD_BUG_ON(sizeof_field(struct sk_buff, 338 - protocol) != 2); 339 - PPC_NTOHS_OFFS(r_A, r_skb, offsetof(struct sk_buff, 340 - protocol)); 341 - break; 342 - case BPF_ANC | SKF_AD_IFINDEX: 343 - case BPF_ANC | SKF_AD_HATYPE: 344 - BUILD_BUG_ON(sizeof_field(struct net_device, 345 - ifindex) != 4); 346 - BUILD_BUG_ON(sizeof_field(struct net_device, 347 - type) != 2); 348 - PPC_LL_OFFS(r_scratch1, r_skb, offsetof(struct sk_buff, 349 - dev)); 350 - EMIT(PPC_RAW_CMPDI(r_scratch1, 0)); 351 - if (ctx->pc_ret0 != -1) { 352 - PPC_BCC(COND_EQ, addrs[ctx->pc_ret0]); 353 - } else { 354 - /* Exit, returning 0; first pass hits here. */ 355 - PPC_BCC_SHORT(COND_NE, ctx->idx * 4 + 12); 356 - EMIT(PPC_RAW_LI(r_ret, 0)); 357 - PPC_JMP(exit_addr); 358 - } 359 - if (code == (BPF_ANC | SKF_AD_IFINDEX)) { 360 - PPC_LWZ_OFFS(r_A, r_scratch1, 361 - offsetof(struct net_device, ifindex)); 362 - } else { 363 - PPC_LHZ_OFFS(r_A, r_scratch1, 364 - offsetof(struct net_device, type)); 365 - } 366 - 367 - break; 368 - case BPF_ANC | SKF_AD_MARK: 369 - BUILD_BUG_ON(sizeof_field(struct sk_buff, mark) != 4); 370 - PPC_LWZ_OFFS(r_A, r_skb, offsetof(struct sk_buff, 371 - mark)); 372 - break; 373 - case BPF_ANC | SKF_AD_RXHASH: 374 - BUILD_BUG_ON(sizeof_field(struct sk_buff, hash) != 4); 375 - PPC_LWZ_OFFS(r_A, r_skb, offsetof(struct sk_buff, 376 - hash)); 377 - break; 378 - case BPF_ANC | SKF_AD_VLAN_TAG: 379 - BUILD_BUG_ON(sizeof_field(struct sk_buff, vlan_tci) != 2); 380 - 381 - PPC_LHZ_OFFS(r_A, r_skb, offsetof(struct sk_buff, 382 - vlan_tci)); 383 - break; 384 - case BPF_ANC | SKF_AD_VLAN_TAG_PRESENT: 385 - PPC_LBZ_OFFS(r_A, r_skb, PKT_VLAN_PRESENT_OFFSET()); 386 - if (PKT_VLAN_PRESENT_BIT) 387 - EMIT(PPC_RAW_SRWI(r_A, r_A, PKT_VLAN_PRESENT_BIT)); 388 - if (PKT_VLAN_PRESENT_BIT < 7) 389 - EMIT(PPC_RAW_ANDI(r_A, r_A, 1)); 390 - break; 391 - case BPF_ANC | SKF_AD_QUEUE: 392 - BUILD_BUG_ON(sizeof_field(struct sk_buff, 393 - queue_mapping) != 2); 394 - PPC_LHZ_OFFS(r_A, r_skb, offsetof(struct sk_buff, 395 - queue_mapping)); 396 - break; 397 - case BPF_ANC | SKF_AD_PKTTYPE: 398 - PPC_LBZ_OFFS(r_A, r_skb, PKT_TYPE_OFFSET()); 399 - EMIT(PPC_RAW_ANDI(r_A, r_A, PKT_TYPE_MAX)); 400 - EMIT(PPC_RAW_SRWI(r_A, r_A, 5)); 401 - break; 402 - case BPF_ANC | SKF_AD_CPU: 403 - PPC_BPF_LOAD_CPU(r_A); 404 - break; 405 - /*** Absolute loads from packet header/data ***/ 406 - case BPF_LD | BPF_W | BPF_ABS: 407 - func = CHOOSE_LOAD_FUNC(K, sk_load_word); 408 - goto common_load; 409 - case BPF_LD | BPF_H | BPF_ABS: 410 - func = CHOOSE_LOAD_FUNC(K, sk_load_half); 411 - goto common_load; 412 - case BPF_LD | BPF_B | BPF_ABS: 413 - func = CHOOSE_LOAD_FUNC(K, sk_load_byte); 414 - common_load: 415 - /* Load from [K]. */ 416 - ctx->seen |= SEEN_DATAREF; 417 - PPC_FUNC_ADDR(r_scratch1, func); 418 - EMIT(PPC_RAW_MTLR(r_scratch1)); 419 - PPC_LI32(r_addr, K); 420 - EMIT(PPC_RAW_BLRL()); 421 - /* 422 - * Helper returns 'lt' condition on error, and an 423 - * appropriate return value in r3 424 - */ 425 - PPC_BCC(COND_LT, exit_addr); 426 - break; 427 - 428 - /*** Indirect loads from packet header/data ***/ 429 - case BPF_LD | BPF_W | BPF_IND: 430 - func = sk_load_word; 431 - goto common_load_ind; 432 - case BPF_LD | BPF_H | BPF_IND: 433 - func = sk_load_half; 434 - goto common_load_ind; 435 - case BPF_LD | BPF_B | BPF_IND: 436 - func = sk_load_byte; 437 - common_load_ind: 438 - /* 439 - * Load from [X + K]. Negative offsets are tested for 440 - * in the helper functions. 441 - */ 442 - ctx->seen |= SEEN_DATAREF | SEEN_XREG; 443 - PPC_FUNC_ADDR(r_scratch1, func); 444 - EMIT(PPC_RAW_MTLR(r_scratch1)); 445 - EMIT(PPC_RAW_ADDI(r_addr, r_X, IMM_L(K))); 446 - if (K >= 32768) 447 - EMIT(PPC_RAW_ADDIS(r_addr, r_addr, IMM_HA(K))); 448 - EMIT(PPC_RAW_BLRL()); 449 - /* If error, cr0.LT set */ 450 - PPC_BCC(COND_LT, exit_addr); 451 - break; 452 - 453 - case BPF_LDX | BPF_B | BPF_MSH: 454 - func = CHOOSE_LOAD_FUNC(K, sk_load_byte_msh); 455 - goto common_load; 456 - break; 457 - 458 - /*** Jump and branches ***/ 459 - case BPF_JMP | BPF_JA: 460 - if (K != 0) 461 - PPC_JMP(addrs[i + 1 + K]); 462 - break; 463 - 464 - case BPF_JMP | BPF_JGT | BPF_K: 465 - case BPF_JMP | BPF_JGT | BPF_X: 466 - true_cond = COND_GT; 467 - goto cond_branch; 468 - case BPF_JMP | BPF_JGE | BPF_K: 469 - case BPF_JMP | BPF_JGE | BPF_X: 470 - true_cond = COND_GE; 471 - goto cond_branch; 472 - case BPF_JMP | BPF_JEQ | BPF_K: 473 - case BPF_JMP | BPF_JEQ | BPF_X: 474 - true_cond = COND_EQ; 475 - goto cond_branch; 476 - case BPF_JMP | BPF_JSET | BPF_K: 477 - case BPF_JMP | BPF_JSET | BPF_X: 478 - true_cond = COND_NE; 479 - cond_branch: 480 - /* same targets, can avoid doing the test :) */ 481 - if (filter[i].jt == filter[i].jf) { 482 - if (filter[i].jt > 0) 483 - PPC_JMP(addrs[i + 1 + filter[i].jt]); 484 - break; 485 - } 486 - 487 - switch (code) { 488 - case BPF_JMP | BPF_JGT | BPF_X: 489 - case BPF_JMP | BPF_JGE | BPF_X: 490 - case BPF_JMP | BPF_JEQ | BPF_X: 491 - ctx->seen |= SEEN_XREG; 492 - EMIT(PPC_RAW_CMPLW(r_A, r_X)); 493 - break; 494 - case BPF_JMP | BPF_JSET | BPF_X: 495 - ctx->seen |= SEEN_XREG; 496 - EMIT(PPC_RAW_AND_DOT(r_scratch1, r_A, r_X)); 497 - break; 498 - case BPF_JMP | BPF_JEQ | BPF_K: 499 - case BPF_JMP | BPF_JGT | BPF_K: 500 - case BPF_JMP | BPF_JGE | BPF_K: 501 - if (K < 32768) 502 - EMIT(PPC_RAW_CMPLWI(r_A, K)); 503 - else { 504 - PPC_LI32(r_scratch1, K); 505 - EMIT(PPC_RAW_CMPLW(r_A, r_scratch1)); 506 - } 507 - break; 508 - case BPF_JMP | BPF_JSET | BPF_K: 509 - if (K < 32768) 510 - /* PPC_ANDI is /only/ dot-form */ 511 - EMIT(PPC_RAW_ANDI(r_scratch1, r_A, K)); 512 - else { 513 - PPC_LI32(r_scratch1, K); 514 - EMIT(PPC_RAW_AND_DOT(r_scratch1, r_A, 515 - r_scratch1)); 516 - } 517 - break; 518 - } 519 - /* Sometimes branches are constructed "backward", with 520 - * the false path being the branch and true path being 521 - * a fallthrough to the next instruction. 522 - */ 523 - if (filter[i].jt == 0) 524 - /* Swap the sense of the branch */ 525 - PPC_BCC(true_cond ^ COND_CMP_TRUE, 526 - addrs[i + 1 + filter[i].jf]); 527 - else { 528 - PPC_BCC(true_cond, addrs[i + 1 + filter[i].jt]); 529 - if (filter[i].jf != 0) 530 - PPC_JMP(addrs[i + 1 + filter[i].jf]); 531 - } 532 - break; 533 - default: 534 - /* The filter contains something cruel & unusual. 535 - * We don't handle it, but also there shouldn't be 536 - * anything missing from our list. 537 - */ 538 - if (printk_ratelimit()) 539 - pr_err("BPF filter opcode %04x (@%d) unsupported\n", 540 - filter[i].code, i); 541 - return -ENOTSUPP; 542 - } 543 - 544 - } 545 - /* Set end-of-body-code address for exit. */ 546 - addrs[i] = ctx->idx * 4; 547 70 548 71 return 0; 549 72 } 550 73 551 - void bpf_jit_compile(struct bpf_prog *fp) 74 + struct powerpc64_jit_data { 75 + struct bpf_binary_header *header; 76 + u32 *addrs; 77 + u8 *image; 78 + u32 proglen; 79 + struct codegen_context ctx; 80 + }; 81 + 82 + bool bpf_jit_needs_zext(void) 552 83 { 553 - unsigned int proglen; 554 - unsigned int alloclen; 555 - u32 *image = NULL; 84 + return true; 85 + } 86 + 87 + struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp) 88 + { 89 + u32 proglen; 90 + u32 alloclen; 91 + u8 *image = NULL; 556 92 u32 *code_base; 557 - unsigned int *addrs; 93 + u32 *addrs; 94 + struct powerpc64_jit_data *jit_data; 558 95 struct codegen_context cgctx; 559 96 int pass; 560 - int flen = fp->len; 97 + int flen; 98 + struct bpf_binary_header *bpf_hdr; 99 + struct bpf_prog *org_fp = fp; 100 + struct bpf_prog *tmp_fp; 101 + bool bpf_blinded = false; 102 + bool extra_pass = false; 561 103 562 - if (!bpf_jit_enable) 563 - return; 104 + if (!fp->jit_requested) 105 + return org_fp; 106 + 107 + tmp_fp = bpf_jit_blind_constants(org_fp); 108 + if (IS_ERR(tmp_fp)) 109 + return org_fp; 110 + 111 + if (tmp_fp != org_fp) { 112 + bpf_blinded = true; 113 + fp = tmp_fp; 114 + } 115 + 116 + jit_data = fp->aux->jit_data; 117 + if (!jit_data) { 118 + jit_data = kzalloc(sizeof(*jit_data), GFP_KERNEL); 119 + if (!jit_data) { 120 + fp = org_fp; 121 + goto out; 122 + } 123 + fp->aux->jit_data = jit_data; 124 + } 125 + 126 + flen = fp->len; 127 + addrs = jit_data->addrs; 128 + if (addrs) { 129 + cgctx = jit_data->ctx; 130 + image = jit_data->image; 131 + bpf_hdr = jit_data->header; 132 + proglen = jit_data->proglen; 133 + alloclen = proglen + FUNCTION_DESCR_SIZE; 134 + extra_pass = true; 135 + goto skip_init_ctx; 136 + } 564 137 565 138 addrs = kcalloc(flen + 1, sizeof(*addrs), GFP_KERNEL); 566 - if (addrs == NULL) 567 - return; 139 + if (addrs == NULL) { 140 + fp = org_fp; 141 + goto out_addrs; 142 + } 143 + 144 + memset(&cgctx, 0, sizeof(struct codegen_context)); 145 + memcpy(cgctx.b2p, b2p, sizeof(cgctx.b2p)); 146 + 147 + /* Make sure that the stack is quadword aligned. */ 148 + cgctx.stack_size = round_up(fp->aux->stack_depth, 16); 149 + 150 + /* Scouting faux-generate pass 0 */ 151 + if (bpf_jit_build_body(fp, 0, &cgctx, addrs, false)) { 152 + /* We hit something illegal or unsupported. */ 153 + fp = org_fp; 154 + goto out_addrs; 155 + } 568 156 569 157 /* 570 - * There are multiple assembly passes as the generated code will change 571 - * size as it settles down, figuring out the max branch offsets/exit 572 - * paths required. 573 - * 574 - * The range of standard conditional branches is +/- 32Kbytes. Since 575 - * BPF_MAXINSNS = 4096, we can only jump from (worst case) start to 576 - * finish with 8 bytes/instruction. Not feasible, so long jumps are 577 - * used, distinct from short branches. 578 - * 579 - * Current: 580 - * 581 - * For now, both branch types assemble to 2 words (short branches padded 582 - * with a NOP); this is less efficient, but assembly will always complete 583 - * after exactly 3 passes: 584 - * 585 - * First pass: No code buffer; Program is "faux-generated" -- no code 586 - * emitted but maximum size of output determined (and addrs[] filled 587 - * in). Also, we note whether we use M[], whether we use skb data, etc. 588 - * All generation choices assumed to be 'worst-case', e.g. branches all 589 - * far (2 instructions), return path code reduction not available, etc. 590 - * 591 - * Second pass: Code buffer allocated with size determined previously. 592 - * Prologue generated to support features we have seen used. Exit paths 593 - * determined and addrs[] is filled in again, as code may be slightly 594 - * smaller as a result. 595 - * 596 - * Third pass: Code generated 'for real', and branch destinations 597 - * determined from now-accurate addrs[] map. 598 - * 599 - * Ideal: 600 - * 601 - * If we optimise this, near branches will be shorter. On the 602 - * first assembly pass, we should err on the side of caution and 603 - * generate the biggest code. On subsequent passes, branches will be 604 - * generated short or long and code size will reduce. With smaller 605 - * code, more branches may fall into the short category, and code will 606 - * reduce more. 607 - * 608 - * Finally, if we see one pass generate code the same size as the 609 - * previous pass we have converged and should now generate code for 610 - * real. Allocating at the end will also save the memory that would 611 - * otherwise be wasted by the (small) current code shrinkage. 612 - * Preferably, we should do a small number of passes (e.g. 5) and if we 613 - * haven't converged by then, get impatient and force code to generate 614 - * as-is, even if the odd branch would be left long. The chances of a 615 - * long jump are tiny with all but the most enormous of BPF filter 616 - * inputs, so we should usually converge on the third pass. 158 + * If we have seen a tail call, we need a second pass. 159 + * This is because bpf_jit_emit_common_epilogue() is called 160 + * from bpf_jit_emit_tail_call() with a not yet stable ctx->seen. 617 161 */ 162 + if (cgctx.seen & SEEN_TAILCALL) { 163 + cgctx.idx = 0; 164 + if (bpf_jit_build_body(fp, 0, &cgctx, addrs, false)) { 165 + fp = org_fp; 166 + goto out_addrs; 167 + } 168 + } 618 169 619 - cgctx.idx = 0; 620 - cgctx.seen = 0; 621 - cgctx.pc_ret0 = -1; 622 - /* Scouting faux-generate pass 0 */ 623 - if (bpf_jit_build_body(fp, 0, &cgctx, addrs)) 624 - /* We hit something illegal or unsupported. */ 625 - goto out; 626 - 170 + bpf_jit_realloc_regs(&cgctx); 627 171 /* 628 172 * Pretend to build prologue, given the features we've seen. This will 629 173 * update ctgtx.idx as it pretends to output instructions, then we can 630 174 * calculate total size from idx. 631 175 */ 632 - bpf_jit_build_prologue(fp, 0, &cgctx); 176 + bpf_jit_build_prologue(0, &cgctx); 633 177 bpf_jit_build_epilogue(0, &cgctx); 634 178 635 179 proglen = cgctx.idx * 4; 636 180 alloclen = proglen + FUNCTION_DESCR_SIZE; 637 - image = module_alloc(alloclen); 638 - if (!image) 639 - goto out; 640 181 641 - code_base = image + (FUNCTION_DESCR_SIZE/4); 182 + bpf_hdr = bpf_jit_binary_alloc(alloclen, &image, 4, bpf_jit_fill_ill_insns); 183 + if (!bpf_hdr) { 184 + fp = org_fp; 185 + goto out_addrs; 186 + } 187 + 188 + skip_init_ctx: 189 + code_base = (u32 *)(image + FUNCTION_DESCR_SIZE); 190 + 191 + if (extra_pass) { 192 + /* 193 + * Do not touch the prologue and epilogue as they will remain 194 + * unchanged. Only fix the branch target address for subprog 195 + * calls in the body. 196 + * 197 + * This does not change the offsets and lengths of the subprog 198 + * call instruction sequences and hence, the size of the JITed 199 + * image as well. 200 + */ 201 + bpf_jit_fixup_subprog_calls(fp, code_base, &cgctx, addrs); 202 + 203 + /* There is no need to perform the usual passes. */ 204 + goto skip_codegen_passes; 205 + } 642 206 643 207 /* Code generation passes 1-2 */ 644 208 for (pass = 1; pass < 3; pass++) { 645 209 /* Now build the prologue, body code & epilogue for real. */ 646 210 cgctx.idx = 0; 647 - bpf_jit_build_prologue(fp, code_base, &cgctx); 648 - bpf_jit_build_body(fp, code_base, &cgctx, addrs); 211 + bpf_jit_build_prologue(code_base, &cgctx); 212 + bpf_jit_build_body(fp, code_base, &cgctx, addrs, extra_pass); 649 213 bpf_jit_build_epilogue(code_base, &cgctx); 650 214 651 215 if (bpf_jit_enable > 1) ··· 218 652 proglen - (cgctx.idx * 4), cgctx.seen); 219 653 } 220 654 655 + skip_codegen_passes: 221 656 if (bpf_jit_enable > 1) 222 - /* Note that we output the base address of the code_base 657 + /* 658 + * Note that we output the base address of the code_base 223 659 * rather than image, since opcodes are in code_base. 224 660 */ 225 661 bpf_jit_dump(flen, proglen, pass, code_base); 226 662 227 - bpf_flush_icache(code_base, code_base + (proglen/4)); 228 - 229 - #ifdef CONFIG_PPC64 663 + #ifdef PPC64_ELF_ABI_v1 230 664 /* Function descriptor nastiness: Address + TOC */ 231 665 ((u64 *)image)[0] = (u64)code_base; 232 666 ((u64 *)image)[1] = local_paca->kernel_toc; ··· 234 668 235 669 fp->bpf_func = (void *)image; 236 670 fp->jited = 1; 671 + fp->jited_len = alloclen; 672 + 673 + bpf_flush_icache(bpf_hdr, (u8 *)bpf_hdr + (bpf_hdr->pages * PAGE_SIZE)); 674 + if (!fp->is_func || extra_pass) { 675 + bpf_prog_fill_jited_linfo(fp, addrs); 676 + out_addrs: 677 + kfree(addrs); 678 + kfree(jit_data); 679 + fp->aux->jit_data = NULL; 680 + } else { 681 + jit_data->addrs = addrs; 682 + jit_data->ctx = cgctx; 683 + jit_data->proglen = proglen; 684 + jit_data->image = image; 685 + jit_data->header = bpf_hdr; 686 + } 237 687 238 688 out: 239 - kfree(addrs); 240 - return; 689 + if (bpf_blinded) 690 + bpf_jit_prog_release_other(fp, fp == org_fp ? tmp_fp : org_fp); 691 + 692 + return fp; 241 693 } 242 694 695 + /* Overriding bpf_jit_free() as we don't set images read-only. */ 243 696 void bpf_jit_free(struct bpf_prog *fp) 244 697 { 698 + unsigned long addr = (unsigned long)fp->bpf_func & PAGE_MASK; 699 + struct bpf_binary_header *bpf_hdr = (void *)addr; 700 + 245 701 if (fp->jited) 246 - module_memfree(fp->bpf_func); 702 + bpf_jit_binary_free(bpf_hdr); 247 703 248 704 bpf_prog_unlock_free(fp); 249 705 }

+1100

arch/powerpc/net/bpf_jit_comp32.c

··· 1 + // SPDX-License-Identifier: GPL-2.0-only 2 + /* 3 + * eBPF JIT compiler for PPC32 4 + * 5 + * Copyright 2020 Christophe Leroy <christophe.leroy@csgroup.eu> 6 + * CS GROUP France 7 + * 8 + * Based on PPC64 eBPF JIT compiler by Naveen N. Rao 9 + */ 10 + #include <linux/moduleloader.h> 11 + #include <asm/cacheflush.h> 12 + #include <asm/asm-compat.h> 13 + #include <linux/netdevice.h> 14 + #include <linux/filter.h> 15 + #include <linux/if_vlan.h> 16 + #include <asm/kprobes.h> 17 + #include <linux/bpf.h> 18 + 19 + #include "bpf_jit.h" 20 + 21 + /* 22 + * Stack layout: 23 + * 24 + * [ prev sp ] <------------- 25 + * [ nv gpr save area ] 16 * 4 | 26 + * fp (r31) --> [ ebpf stack space ] upto 512 | 27 + * [ frame header ] 16 | 28 + * sp (r1) ---> [ stack pointer ] -------------- 29 + */ 30 + 31 + /* for gpr non volatile registers r17 to r31 (14) + tail call */ 32 + #define BPF_PPC_STACK_SAVE (15 * 4 + 4) 33 + /* stack frame, ensure this is quadword aligned */ 34 + #define BPF_PPC_STACKFRAME(ctx) (STACK_FRAME_MIN_SIZE + BPF_PPC_STACK_SAVE + (ctx)->stack_size) 35 + 36 + /* BPF register usage */ 37 + #define TMP_REG (MAX_BPF_JIT_REG + 0) 38 + 39 + /* BPF to ppc register mappings */ 40 + const int b2p[MAX_BPF_JIT_REG + 1] = { 41 + /* function return value */ 42 + [BPF_REG_0] = 12, 43 + /* function arguments */ 44 + [BPF_REG_1] = 4, 45 + [BPF_REG_2] = 6, 46 + [BPF_REG_3] = 8, 47 + [BPF_REG_4] = 10, 48 + [BPF_REG_5] = 22, 49 + /* non volatile registers */ 50 + [BPF_REG_6] = 24, 51 + [BPF_REG_7] = 26, 52 + [BPF_REG_8] = 28, 53 + [BPF_REG_9] = 30, 54 + /* frame pointer aka BPF_REG_10 */ 55 + [BPF_REG_FP] = 18, 56 + /* eBPF jit internal registers */ 57 + [BPF_REG_AX] = 20, 58 + [TMP_REG] = 31, /* 32 bits */ 59 + }; 60 + 61 + static int bpf_to_ppc(struct codegen_context *ctx, int reg) 62 + { 63 + return ctx->b2p[reg]; 64 + } 65 + 66 + /* PPC NVR range -- update this if we ever use NVRs below r17 */ 67 + #define BPF_PPC_NVR_MIN 17 68 + #define BPF_PPC_TC 16 69 + 70 + static int bpf_jit_stack_offsetof(struct codegen_context *ctx, int reg) 71 + { 72 + if ((reg >= BPF_PPC_NVR_MIN && reg < 32) || reg == BPF_PPC_TC) 73 + return BPF_PPC_STACKFRAME(ctx) - 4 * (32 - reg); 74 + 75 + WARN(true, "BPF JIT is asking about unknown registers, will crash the stack"); 76 + /* Use the hole we have left for alignment */ 77 + return BPF_PPC_STACKFRAME(ctx) - 4; 78 + } 79 + 80 + void bpf_jit_realloc_regs(struct codegen_context *ctx) 81 + { 82 + if (ctx->seen & SEEN_FUNC) 83 + return; 84 + 85 + while (ctx->seen & SEEN_NVREG_MASK && 86 + (ctx->seen & SEEN_VREG_MASK) != SEEN_VREG_MASK) { 87 + int old = 32 - fls(ctx->seen & (SEEN_NVREG_MASK & 0xaaaaaaab)); 88 + int new = 32 - fls(~ctx->seen & (SEEN_VREG_MASK & 0xaaaaaaaa)); 89 + int i; 90 + 91 + for (i = BPF_REG_0; i <= TMP_REG; i++) { 92 + if (ctx->b2p[i] != old) 93 + continue; 94 + ctx->b2p[i] = new; 95 + bpf_set_seen_register(ctx, new); 96 + bpf_clear_seen_register(ctx, old); 97 + if (i != TMP_REG) { 98 + bpf_set_seen_register(ctx, new - 1); 99 + bpf_clear_seen_register(ctx, old - 1); 100 + } 101 + break; 102 + } 103 + } 104 + } 105 + 106 + void bpf_jit_build_prologue(u32 *image, struct codegen_context *ctx) 107 + { 108 + int i; 109 + 110 + /* First arg comes in as a 32 bits pointer. */ 111 + EMIT(PPC_RAW_MR(bpf_to_ppc(ctx, BPF_REG_1), __REG_R3)); 112 + EMIT(PPC_RAW_LI(bpf_to_ppc(ctx, BPF_REG_1) - 1, 0)); 113 + EMIT(PPC_RAW_STWU(__REG_R1, __REG_R1, -BPF_PPC_STACKFRAME(ctx))); 114 + 115 + /* 116 + * Initialize tail_call_cnt in stack frame if we do tail calls. 117 + * Otherwise, put in NOPs so that it can be skipped when we are 118 + * invoked through a tail call. 119 + */ 120 + if (ctx->seen & SEEN_TAILCALL) { 121 + EMIT(PPC_RAW_STW(bpf_to_ppc(ctx, BPF_REG_1) - 1, __REG_R1, bpf_jit_stack_offsetof(ctx, BPF_PPC_TC))); 122 + } else { 123 + EMIT(PPC_RAW_NOP()); 124 + } 125 + 126 + #define BPF_TAILCALL_PROLOGUE_SIZE 16 127 + 128 + /* 129 + * We need a stack frame, but we don't necessarily need to 130 + * save/restore LR unless we call other functions 131 + */ 132 + if (ctx->seen & SEEN_FUNC) 133 + EMIT(PPC_RAW_MFLR(__REG_R0)); 134 + 135 + /* 136 + * Back up non-volatile regs -- registers r18-r31 137 + */ 138 + for (i = BPF_PPC_NVR_MIN; i <= 31; i++) 139 + if (bpf_is_seen_register(ctx, i)) 140 + EMIT(PPC_RAW_STW(i, __REG_R1, bpf_jit_stack_offsetof(ctx, i))); 141 + 142 + /* If needed retrieve arguments 9 and 10, ie 5th 64 bits arg.*/ 143 + if (bpf_is_seen_register(ctx, bpf_to_ppc(ctx, BPF_REG_5))) { 144 + EMIT(PPC_RAW_LWZ(bpf_to_ppc(ctx, BPF_REG_5) - 1, __REG_R1, BPF_PPC_STACKFRAME(ctx)) + 8); 145 + EMIT(PPC_RAW_LWZ(bpf_to_ppc(ctx, BPF_REG_5), __REG_R1, BPF_PPC_STACKFRAME(ctx)) + 12); 146 + } 147 + 148 + /* Setup frame pointer to point to the bpf stack area */ 149 + if (bpf_is_seen_register(ctx, bpf_to_ppc(ctx, BPF_REG_FP))) { 150 + EMIT(PPC_RAW_LI(bpf_to_ppc(ctx, BPF_REG_FP) - 1, 0)); 151 + EMIT(PPC_RAW_ADDI(bpf_to_ppc(ctx, BPF_REG_FP), __REG_R1, 152 + STACK_FRAME_MIN_SIZE + ctx->stack_size)); 153 + } 154 + 155 + if (ctx->seen & SEEN_FUNC) 156 + EMIT(PPC_RAW_STW(__REG_R0, __REG_R1, BPF_PPC_STACKFRAME(ctx) + PPC_LR_STKOFF)); 157 + } 158 + 159 + static void bpf_jit_emit_common_epilogue(u32 *image, struct codegen_context *ctx) 160 + { 161 + int i; 162 + 163 + /* Restore NVRs */ 164 + for (i = BPF_PPC_NVR_MIN; i <= 31; i++) 165 + if (bpf_is_seen_register(ctx, i)) 166 + EMIT(PPC_RAW_LWZ(i, __REG_R1, bpf_jit_stack_offsetof(ctx, i))); 167 + } 168 + 169 + void bpf_jit_build_epilogue(u32 *image, struct codegen_context *ctx) 170 + { 171 + EMIT(PPC_RAW_MR(__REG_R3, bpf_to_ppc(ctx, BPF_REG_0))); 172 + 173 + bpf_jit_emit_common_epilogue(image, ctx); 174 + 175 + /* Tear down our stack frame */ 176 + 177 + if (ctx->seen & SEEN_FUNC) 178 + EMIT(PPC_RAW_LWZ(__REG_R0, __REG_R1, BPF_PPC_STACKFRAME(ctx) + PPC_LR_STKOFF)); 179 + 180 + EMIT(PPC_RAW_ADDI(__REG_R1, __REG_R1, BPF_PPC_STACKFRAME(ctx))); 181 + 182 + if (ctx->seen & SEEN_FUNC) 183 + EMIT(PPC_RAW_MTLR(__REG_R0)); 184 + 185 + EMIT(PPC_RAW_BLR()); 186 + } 187 + 188 + void bpf_jit_emit_func_call_rel(u32 *image, struct codegen_context *ctx, u64 func) 189 + { 190 + s32 rel = (s32)func - (s32)(image + ctx->idx); 191 + 192 + if (image && rel < 0x2000000 && rel >= -0x2000000) { 193 + PPC_BL_ABS(func); 194 + } else { 195 + /* Load function address into r0 */ 196 + EMIT(PPC_RAW_LIS(__REG_R0, IMM_H(func))); 197 + EMIT(PPC_RAW_ORI(__REG_R0, __REG_R0, IMM_L(func))); 198 + EMIT(PPC_RAW_MTLR(__REG_R0)); 199 + EMIT(PPC_RAW_BLRL()); 200 + } 201 + } 202 + 203 + static void bpf_jit_emit_tail_call(u32 *image, struct codegen_context *ctx, u32 out) 204 + { 205 + /* 206 + * By now, the eBPF program has already setup parameters in r3-r6 207 + * r3-r4/BPF_REG_1 - pointer to ctx -- passed as is to the next bpf program 208 + * r5-r6/BPF_REG_2 - pointer to bpf_array 209 + * r7-r8/BPF_REG_3 - index in bpf_array 210 + */ 211 + int b2p_bpf_array = bpf_to_ppc(ctx, BPF_REG_2); 212 + int b2p_index = bpf_to_ppc(ctx, BPF_REG_3); 213 + 214 + /* 215 + * if (index >= array->map.max_entries) 216 + * goto out; 217 + */ 218 + EMIT(PPC_RAW_LWZ(__REG_R0, b2p_bpf_array, offsetof(struct bpf_array, map.max_entries))); 219 + EMIT(PPC_RAW_CMPLW(b2p_index, __REG_R0)); 220 + EMIT(PPC_RAW_LWZ(__REG_R0, __REG_R1, bpf_jit_stack_offsetof(ctx, BPF_PPC_TC))); 221 + PPC_BCC(COND_GE, out); 222 + 223 + /* 224 + * if (tail_call_cnt > MAX_TAIL_CALL_CNT) 225 + * goto out; 226 + */ 227 + EMIT(PPC_RAW_CMPLWI(__REG_R0, MAX_TAIL_CALL_CNT)); 228 + /* tail_call_cnt++; */ 229 + EMIT(PPC_RAW_ADDIC(__REG_R0, __REG_R0, 1)); 230 + PPC_BCC(COND_GT, out); 231 + 232 + /* prog = array->ptrs[index]; */ 233 + EMIT(PPC_RAW_RLWINM(__REG_R3, b2p_index, 2, 0, 29)); 234 + EMIT(PPC_RAW_ADD(__REG_R3, __REG_R3, b2p_bpf_array)); 235 + EMIT(PPC_RAW_LWZ(__REG_R3, __REG_R3, offsetof(struct bpf_array, ptrs))); 236 + EMIT(PPC_RAW_STW(__REG_R0, __REG_R1, bpf_jit_stack_offsetof(ctx, BPF_PPC_TC))); 237 + 238 + /* 239 + * if (prog == NULL) 240 + * goto out; 241 + */ 242 + EMIT(PPC_RAW_CMPLWI(__REG_R3, 0)); 243 + PPC_BCC(COND_EQ, out); 244 + 245 + /* goto *(prog->bpf_func + prologue_size); */ 246 + EMIT(PPC_RAW_LWZ(__REG_R3, __REG_R3, offsetof(struct bpf_prog, bpf_func))); 247 + 248 + if (ctx->seen & SEEN_FUNC) 249 + EMIT(PPC_RAW_LWZ(__REG_R0, __REG_R1, BPF_PPC_STACKFRAME(ctx) + PPC_LR_STKOFF)); 250 + 251 + EMIT(PPC_RAW_ADDIC(__REG_R3, __REG_R3, BPF_TAILCALL_PROLOGUE_SIZE)); 252 + 253 + if (ctx->seen & SEEN_FUNC) 254 + EMIT(PPC_RAW_MTLR(__REG_R0)); 255 + 256 + EMIT(PPC_RAW_MTCTR(__REG_R3)); 257 + 258 + EMIT(PPC_RAW_MR(__REG_R3, bpf_to_ppc(ctx, BPF_REG_1))); 259 + 260 + /* tear restore NVRs, ... */ 261 + bpf_jit_emit_common_epilogue(image, ctx); 262 + 263 + EMIT(PPC_RAW_BCTR()); 264 + /* out: */ 265 + } 266 + 267 + /* Assemble the body code between the prologue & epilogue */ 268 + int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, struct codegen_context *ctx, 269 + u32 *addrs, bool extra_pass) 270 + { 271 + const struct bpf_insn *insn = fp->insnsi; 272 + int flen = fp->len; 273 + int i, ret; 274 + 275 + /* Start of epilogue code - will only be valid 2nd pass onwards */ 276 + u32 exit_addr = addrs[flen]; 277 + 278 + for (i = 0; i < flen; i++) { 279 + u32 code = insn[i].code; 280 + u32 dst_reg = bpf_to_ppc(ctx, insn[i].dst_reg); 281 + u32 dst_reg_h = dst_reg - 1; 282 + u32 src_reg = bpf_to_ppc(ctx, insn[i].src_reg); 283 + u32 src_reg_h = src_reg - 1; 284 + u32 tmp_reg = bpf_to_ppc(ctx, TMP_REG); 285 + s16 off = insn[i].off; 286 + s32 imm = insn[i].imm; 287 + bool func_addr_fixed; 288 + u64 func_addr; 289 + u32 true_cond; 290 + 291 + /* 292 + * addrs[] maps a BPF bytecode address into a real offset from 293 + * the start of the body code. 294 + */ 295 + addrs[i] = ctx->idx * 4; 296 + 297 + /* 298 + * As an optimization, we note down which registers 299 + * are used so that we can only save/restore those in our 300 + * prologue and epilogue. We do this here regardless of whether 301 + * the actual BPF instruction uses src/dst registers or not 302 + * (for instance, BPF_CALL does not use them). The expectation 303 + * is that those instructions will have src_reg/dst_reg set to 304 + * 0. Even otherwise, we just lose some prologue/epilogue 305 + * optimization but everything else should work without 306 + * any issues. 307 + */ 308 + if (dst_reg >= 3 && dst_reg < 32) { 309 + bpf_set_seen_register(ctx, dst_reg); 310 + bpf_set_seen_register(ctx, dst_reg_h); 311 + } 312 + 313 + if (src_reg >= 3 && src_reg < 32) { 314 + bpf_set_seen_register(ctx, src_reg); 315 + bpf_set_seen_register(ctx, src_reg_h); 316 + } 317 + 318 + switch (code) { 319 + /* 320 + * Arithmetic operations: ADD/SUB/MUL/DIV/MOD/NEG 321 + */ 322 + case BPF_ALU | BPF_ADD | BPF_X: /* (u32) dst += (u32) src */ 323 + EMIT(PPC_RAW_ADD(dst_reg, dst_reg, src_reg)); 324 + break; 325 + case BPF_ALU64 | BPF_ADD | BPF_X: /* dst += src */ 326 + EMIT(PPC_RAW_ADDC(dst_reg, dst_reg, src_reg)); 327 + EMIT(PPC_RAW_ADDE(dst_reg_h, dst_reg_h, src_reg_h)); 328 + break; 329 + case BPF_ALU | BPF_SUB | BPF_X: /* (u32) dst -= (u32) src */ 330 + EMIT(PPC_RAW_SUB(dst_reg, dst_reg, src_reg)); 331 + break; 332 + case BPF_ALU64 | BPF_SUB | BPF_X: /* dst -= src */ 333 + EMIT(PPC_RAW_SUBFC(dst_reg, src_reg, dst_reg)); 334 + EMIT(PPC_RAW_SUBFE(dst_reg_h, src_reg_h, dst_reg_h)); 335 + break; 336 + case BPF_ALU | BPF_SUB | BPF_K: /* (u32) dst -= (u32) imm */ 337 + imm = -imm; 338 + fallthrough; 339 + case BPF_ALU | BPF_ADD | BPF_K: /* (u32) dst += (u32) imm */ 340 + if (IMM_HA(imm) & 0xffff) 341 + EMIT(PPC_RAW_ADDIS(dst_reg, dst_reg, IMM_HA(imm))); 342 + if (IMM_L(imm)) 343 + EMIT(PPC_RAW_ADDI(dst_reg, dst_reg, IMM_L(imm))); 344 + break; 345 + case BPF_ALU64 | BPF_SUB | BPF_K: /* dst -= imm */ 346 + imm = -imm; 347 + fallthrough; 348 + case BPF_ALU64 | BPF_ADD | BPF_K: /* dst += imm */ 349 + if (!imm) 350 + break; 351 + 352 + if (imm >= -32768 && imm < 32768) { 353 + EMIT(PPC_RAW_ADDIC(dst_reg, dst_reg, imm)); 354 + } else { 355 + PPC_LI32(__REG_R0, imm); 356 + EMIT(PPC_RAW_ADDC(dst_reg, dst_reg, __REG_R0)); 357 + } 358 + if (imm >= 0) 359 + EMIT(PPC_RAW_ADDZE(dst_reg_h, dst_reg_h)); 360 + else 361 + EMIT(PPC_RAW_ADDME(dst_reg_h, dst_reg_h)); 362 + break; 363 + case BPF_ALU64 | BPF_MUL | BPF_X: /* dst *= src */ 364 + bpf_set_seen_register(ctx, tmp_reg); 365 + EMIT(PPC_RAW_MULW(__REG_R0, dst_reg, src_reg_h)); 366 + EMIT(PPC_RAW_MULW(dst_reg_h, dst_reg_h, src_reg)); 367 + EMIT(PPC_RAW_MULHWU(tmp_reg, dst_reg, src_reg)); 368 + EMIT(PPC_RAW_MULW(dst_reg, dst_reg, src_reg)); 369 + EMIT(PPC_RAW_ADD(dst_reg_h, dst_reg_h, __REG_R0)); 370 + EMIT(PPC_RAW_ADD(dst_reg_h, dst_reg_h, tmp_reg)); 371 + break; 372 + case BPF_ALU | BPF_MUL | BPF_X: /* (u32) dst *= (u32) src */ 373 + EMIT(PPC_RAW_MULW(dst_reg, dst_reg, src_reg)); 374 + break; 375 + case BPF_ALU | BPF_MUL | BPF_K: /* (u32) dst *= (u32) imm */ 376 + if (imm >= -32768 && imm < 32768) { 377 + EMIT(PPC_RAW_MULI(dst_reg, dst_reg, imm)); 378 + } else { 379 + PPC_LI32(__REG_R0, imm); 380 + EMIT(PPC_RAW_MULW(dst_reg, dst_reg, __REG_R0)); 381 + } 382 + break; 383 + case BPF_ALU64 | BPF_MUL | BPF_K: /* dst *= imm */ 384 + if (!imm) { 385 + PPC_LI32(dst_reg, 0); 386 + PPC_LI32(dst_reg_h, 0); 387 + break; 388 + } 389 + if (imm == 1) 390 + break; 391 + if (imm == -1) { 392 + EMIT(PPC_RAW_SUBFIC(dst_reg, dst_reg, 0)); 393 + EMIT(PPC_RAW_SUBFZE(dst_reg_h, dst_reg_h)); 394 + break; 395 + } 396 + bpf_set_seen_register(ctx, tmp_reg); 397 + PPC_LI32(tmp_reg, imm); 398 + EMIT(PPC_RAW_MULW(dst_reg_h, dst_reg_h, tmp_reg)); 399 + if (imm < 0) 400 + EMIT(PPC_RAW_SUB(dst_reg_h, dst_reg_h, dst_reg)); 401 + EMIT(PPC_RAW_MULHWU(__REG_R0, dst_reg, tmp_reg)); 402 + EMIT(PPC_RAW_MULW(dst_reg, dst_reg, tmp_reg)); 403 + EMIT(PPC_RAW_ADD(dst_reg_h, dst_reg_h, __REG_R0)); 404 + break; 405 + case BPF_ALU | BPF_DIV | BPF_X: /* (u32) dst /= (u32) src */ 406 + EMIT(PPC_RAW_DIVWU(dst_reg, dst_reg, src_reg)); 407 + break; 408 + case BPF_ALU | BPF_MOD | BPF_X: /* (u32) dst %= (u32) src */ 409 + EMIT(PPC_RAW_DIVWU(__REG_R0, dst_reg, src_reg)); 410 + EMIT(PPC_RAW_MULW(__REG_R0, src_reg, __REG_R0)); 411 + EMIT(PPC_RAW_SUB(dst_reg, dst_reg, __REG_R0)); 412 + break; 413 + case BPF_ALU64 | BPF_DIV | BPF_X: /* dst /= src */ 414 + return -EOPNOTSUPP; 415 + case BPF_ALU64 | BPF_MOD | BPF_X: /* dst %= src */ 416 + return -EOPNOTSUPP; 417 + case BPF_ALU | BPF_DIV | BPF_K: /* (u32) dst /= (u32) imm */ 418 + if (!imm) 419 + return -EINVAL; 420 + if (imm == 1) 421 + break; 422 + 423 + PPC_LI32(__REG_R0, imm); 424 + EMIT(PPC_RAW_DIVWU(dst_reg, dst_reg, __REG_R0)); 425 + break; 426 + case BPF_ALU | BPF_MOD | BPF_K: /* (u32) dst %= (u32) imm */ 427 + if (!imm) 428 + return -EINVAL; 429 + 430 + if (!is_power_of_2((u32)imm)) { 431 + bpf_set_seen_register(ctx, tmp_reg); 432 + PPC_LI32(tmp_reg, imm); 433 + EMIT(PPC_RAW_DIVWU(__REG_R0, dst_reg, tmp_reg)); 434 + EMIT(PPC_RAW_MULW(__REG_R0, tmp_reg, __REG_R0)); 435 + EMIT(PPC_RAW_SUB(dst_reg, dst_reg, __REG_R0)); 436 + break; 437 + } 438 + if (imm == 1) 439 + EMIT(PPC_RAW_LI(dst_reg, 0)); 440 + else 441 + EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, 0, 32 - ilog2((u32)imm), 31)); 442 + 443 + break; 444 + case BPF_ALU64 | BPF_MOD | BPF_K: /* dst %= imm */ 445 + if (!imm) 446 + return -EINVAL; 447 + if (imm < 0) 448 + imm = -imm; 449 + if (!is_power_of_2(imm)) 450 + return -EOPNOTSUPP; 451 + if (imm == 1) 452 + EMIT(PPC_RAW_LI(dst_reg, 0)); 453 + else 454 + EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, 0, 32 - ilog2(imm), 31)); 455 + EMIT(PPC_RAW_LI(dst_reg_h, 0)); 456 + break; 457 + case BPF_ALU64 | BPF_DIV | BPF_K: /* dst /= imm */ 458 + if (!imm) 459 + return -EINVAL; 460 + if (!is_power_of_2(abs(imm))) 461 + return -EOPNOTSUPP; 462 + 463 + if (imm < 0) { 464 + EMIT(PPC_RAW_SUBFIC(dst_reg, dst_reg, 0)); 465 + EMIT(PPC_RAW_SUBFZE(dst_reg_h, dst_reg_h)); 466 + imm = -imm; 467 + } 468 + if (imm == 1) 469 + break; 470 + imm = ilog2(imm); 471 + EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, 32 - imm, imm, 31)); 472 + EMIT(PPC_RAW_RLWIMI(dst_reg, dst_reg_h, 32 - imm, 0, imm - 1)); 473 + EMIT(PPC_RAW_SRAWI(dst_reg_h, dst_reg_h, imm)); 474 + break; 475 + case BPF_ALU | BPF_NEG: /* (u32) dst = -dst */ 476 + EMIT(PPC_RAW_NEG(dst_reg, dst_reg)); 477 + break; 478 + case BPF_ALU64 | BPF_NEG: /* dst = -dst */ 479 + EMIT(PPC_RAW_SUBFIC(dst_reg, dst_reg, 0)); 480 + EMIT(PPC_RAW_SUBFZE(dst_reg_h, dst_reg_h)); 481 + break; 482 + 483 + /* 484 + * Logical operations: AND/OR/XOR/[A]LSH/[A]RSH 485 + */ 486 + case BPF_ALU64 | BPF_AND | BPF_X: /* dst = dst & src */ 487 + EMIT(PPC_RAW_AND(dst_reg, dst_reg, src_reg)); 488 + EMIT(PPC_RAW_AND(dst_reg_h, dst_reg_h, src_reg_h)); 489 + break; 490 + case BPF_ALU | BPF_AND | BPF_X: /* (u32) dst = dst & src */ 491 + EMIT(PPC_RAW_AND(dst_reg, dst_reg, src_reg)); 492 + break; 493 + case BPF_ALU64 | BPF_AND | BPF_K: /* dst = dst & imm */ 494 + if (imm >= 0) 495 + EMIT(PPC_RAW_LI(dst_reg_h, 0)); 496 + fallthrough; 497 + case BPF_ALU | BPF_AND | BPF_K: /* (u32) dst = dst & imm */ 498 + if (!IMM_H(imm)) { 499 + EMIT(PPC_RAW_ANDI(dst_reg, dst_reg, IMM_L(imm))); 500 + } else if (!IMM_L(imm)) { 501 + EMIT(PPC_RAW_ANDIS(dst_reg, dst_reg, IMM_H(imm))); 502 + } else if (imm == (((1 << fls(imm)) - 1) ^ ((1 << (ffs(i) - 1)) - 1))) { 503 + EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, 0, 504 + 32 - fls(imm), 32 - ffs(imm))); 505 + } else { 506 + PPC_LI32(__REG_R0, imm); 507 + EMIT(PPC_RAW_AND(dst_reg, dst_reg, __REG_R0)); 508 + } 509 + break; 510 + case BPF_ALU64 | BPF_OR | BPF_X: /* dst = dst | src */ 511 + EMIT(PPC_RAW_OR(dst_reg, dst_reg, src_reg)); 512 + EMIT(PPC_RAW_OR(dst_reg_h, dst_reg_h, src_reg_h)); 513 + break; 514 + case BPF_ALU | BPF_OR | BPF_X: /* dst = (u32) dst | (u32) src */ 515 + EMIT(PPC_RAW_OR(dst_reg, dst_reg, src_reg)); 516 + break; 517 + case BPF_ALU64 | BPF_OR | BPF_K:/* dst = dst | imm */ 518 + /* Sign-extended */ 519 + if (imm < 0) 520 + EMIT(PPC_RAW_LI(dst_reg_h, -1)); 521 + fallthrough; 522 + case BPF_ALU | BPF_OR | BPF_K:/* dst = (u32) dst | (u32) imm */ 523 + if (IMM_L(imm)) 524 + EMIT(PPC_RAW_ORI(dst_reg, dst_reg, IMM_L(imm))); 525 + if (IMM_H(imm)) 526 + EMIT(PPC_RAW_ORIS(dst_reg, dst_reg, IMM_H(imm))); 527 + break; 528 + case BPF_ALU64 | BPF_XOR | BPF_X: /* dst ^= src */ 529 + if (dst_reg == src_reg) { 530 + EMIT(PPC_RAW_LI(dst_reg, 0)); 531 + EMIT(PPC_RAW_LI(dst_reg_h, 0)); 532 + } else { 533 + EMIT(PPC_RAW_XOR(dst_reg, dst_reg, src_reg)); 534 + EMIT(PPC_RAW_XOR(dst_reg_h, dst_reg_h, src_reg_h)); 535 + } 536 + break; 537 + case BPF_ALU | BPF_XOR | BPF_X: /* (u32) dst ^= src */ 538 + if (dst_reg == src_reg) 539 + EMIT(PPC_RAW_LI(dst_reg, 0)); 540 + else 541 + EMIT(PPC_RAW_XOR(dst_reg, dst_reg, src_reg)); 542 + break; 543 + case BPF_ALU64 | BPF_XOR | BPF_K: /* dst ^= imm */ 544 + if (imm < 0) 545 + EMIT(PPC_RAW_NOR(dst_reg_h, dst_reg_h, dst_reg_h)); 546 + fallthrough; 547 + case BPF_ALU | BPF_XOR | BPF_K: /* (u32) dst ^= (u32) imm */ 548 + if (IMM_L(imm)) 549 + EMIT(PPC_RAW_XORI(dst_reg, dst_reg, IMM_L(imm))); 550 + if (IMM_H(imm)) 551 + EMIT(PPC_RAW_XORIS(dst_reg, dst_reg, IMM_H(imm))); 552 + break; 553 + case BPF_ALU | BPF_LSH | BPF_X: /* (u32) dst <<= (u32) src */ 554 + EMIT(PPC_RAW_SLW(dst_reg, dst_reg, src_reg)); 555 + break; 556 + case BPF_ALU64 | BPF_LSH | BPF_X: /* dst <<= src; */ 557 + bpf_set_seen_register(ctx, tmp_reg); 558 + EMIT(PPC_RAW_SUBFIC(__REG_R0, src_reg, 32)); 559 + EMIT(PPC_RAW_SLW(dst_reg_h, dst_reg_h, src_reg)); 560 + EMIT(PPC_RAW_ADDI(tmp_reg, src_reg, 32)); 561 + EMIT(PPC_RAW_SRW(__REG_R0, dst_reg, __REG_R0)); 562 + EMIT(PPC_RAW_SLW(tmp_reg, dst_reg, tmp_reg)); 563 + EMIT(PPC_RAW_OR(dst_reg_h, dst_reg_h, __REG_R0)); 564 + EMIT(PPC_RAW_SLW(dst_reg, dst_reg, src_reg)); 565 + EMIT(PPC_RAW_OR(dst_reg_h, dst_reg_h, tmp_reg)); 566 + break; 567 + case BPF_ALU | BPF_LSH | BPF_K: /* (u32) dst <<= (u32) imm */ 568 + if (!imm) 569 + break; 570 + EMIT(PPC_RAW_SLWI(dst_reg, dst_reg, imm)); 571 + break; 572 + case BPF_ALU64 | BPF_LSH | BPF_K: /* dst <<= imm */ 573 + if (imm < 0) 574 + return -EINVAL; 575 + if (!imm) 576 + break; 577 + if (imm < 32) { 578 + EMIT(PPC_RAW_RLWINM(dst_reg_h, dst_reg_h, imm, 0, 31 - imm)); 579 + EMIT(PPC_RAW_RLWIMI(dst_reg_h, dst_reg, imm, 32 - imm, 31)); 580 + EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, imm, 0, 31 - imm)); 581 + break; 582 + } 583 + if (imm < 64) 584 + EMIT(PPC_RAW_RLWINM(dst_reg_h, dst_reg, imm, 0, 31 - imm)); 585 + else 586 + EMIT(PPC_RAW_LI(dst_reg_h, 0)); 587 + EMIT(PPC_RAW_LI(dst_reg, 0)); 588 + break; 589 + case BPF_ALU | BPF_RSH | BPF_X: /* (u32) dst >>= (u32) src */ 590 + EMIT(PPC_RAW_SRW(dst_reg, dst_reg, src_reg)); 591 + break; 592 + case BPF_ALU64 | BPF_RSH | BPF_X: /* dst >>= src */ 593 + bpf_set_seen_register(ctx, tmp_reg); 594 + EMIT(PPC_RAW_SUBFIC(__REG_R0, src_reg, 32)); 595 + EMIT(PPC_RAW_SRW(dst_reg, dst_reg, src_reg)); 596 + EMIT(PPC_RAW_ADDI(tmp_reg, src_reg, 32)); 597 + EMIT(PPC_RAW_SLW(__REG_R0, dst_reg_h, __REG_R0)); 598 + EMIT(PPC_RAW_SRW(tmp_reg, dst_reg_h, tmp_reg)); 599 + EMIT(PPC_RAW_OR(dst_reg, dst_reg, __REG_R0)); 600 + EMIT(PPC_RAW_SRW(dst_reg_h, dst_reg_h, src_reg)); 601 + EMIT(PPC_RAW_OR(dst_reg, dst_reg, tmp_reg)); 602 + break; 603 + case BPF_ALU | BPF_RSH | BPF_K: /* (u32) dst >>= (u32) imm */ 604 + if (!imm) 605 + break; 606 + EMIT(PPC_RAW_SRWI(dst_reg, dst_reg, imm)); 607 + break; 608 + case BPF_ALU64 | BPF_RSH | BPF_K: /* dst >>= imm */ 609 + if (imm < 0) 610 + return -EINVAL; 611 + if (!imm) 612 + break; 613 + if (imm < 32) { 614 + EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, 32 - imm, imm, 31)); 615 + EMIT(PPC_RAW_RLWIMI(dst_reg, dst_reg_h, 32 - imm, 0, imm - 1)); 616 + EMIT(PPC_RAW_RLWINM(dst_reg_h, dst_reg_h, 32 - imm, imm, 31)); 617 + break; 618 + } 619 + if (imm < 64) 620 + EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg_h, 64 - imm, imm - 32, 31)); 621 + else 622 + EMIT(PPC_RAW_LI(dst_reg, 0)); 623 + EMIT(PPC_RAW_LI(dst_reg_h, 0)); 624 + break; 625 + case BPF_ALU | BPF_ARSH | BPF_X: /* (s32) dst >>= src */ 626 + EMIT(PPC_RAW_SRAW(dst_reg_h, dst_reg, src_reg)); 627 + break; 628 + case BPF_ALU64 | BPF_ARSH | BPF_X: /* (s64) dst >>= src */ 629 + bpf_set_seen_register(ctx, tmp_reg); 630 + EMIT(PPC_RAW_SUBFIC(__REG_R0, src_reg, 32)); 631 + EMIT(PPC_RAW_SRW(dst_reg, dst_reg, src_reg)); 632 + EMIT(PPC_RAW_SLW(__REG_R0, dst_reg_h, __REG_R0)); 633 + EMIT(PPC_RAW_ADDI(tmp_reg, src_reg, 32)); 634 + EMIT(PPC_RAW_OR(dst_reg, dst_reg, __REG_R0)); 635 + EMIT(PPC_RAW_RLWINM(__REG_R0, tmp_reg, 0, 26, 26)); 636 + EMIT(PPC_RAW_SRAW(tmp_reg, dst_reg_h, tmp_reg)); 637 + EMIT(PPC_RAW_SRAW(dst_reg_h, dst_reg_h, src_reg)); 638 + EMIT(PPC_RAW_SLW(tmp_reg, tmp_reg, __REG_R0)); 639 + EMIT(PPC_RAW_OR(dst_reg, dst_reg, tmp_reg)); 640 + break; 641 + case BPF_ALU | BPF_ARSH | BPF_K: /* (s32) dst >>= imm */ 642 + if (!imm) 643 + break; 644 + EMIT(PPC_RAW_SRAWI(dst_reg, dst_reg, imm)); 645 + break; 646 + case BPF_ALU64 | BPF_ARSH | BPF_K: /* (s64) dst >>= imm */ 647 + if (imm < 0) 648 + return -EINVAL; 649 + if (!imm) 650 + break; 651 + if (imm < 32) { 652 + EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, 32 - imm, imm, 31)); 653 + EMIT(PPC_RAW_RLWIMI(dst_reg, dst_reg_h, 32 - imm, 0, imm - 1)); 654 + EMIT(PPC_RAW_SRAWI(dst_reg_h, dst_reg_h, imm)); 655 + break; 656 + } 657 + if (imm < 64) 658 + EMIT(PPC_RAW_SRAWI(dst_reg, dst_reg_h, imm - 32)); 659 + else 660 + EMIT(PPC_RAW_SRAWI(dst_reg, dst_reg_h, 31)); 661 + EMIT(PPC_RAW_SRAWI(dst_reg_h, dst_reg_h, 31)); 662 + break; 663 + 664 + /* 665 + * MOV 666 + */ 667 + case BPF_ALU64 | BPF_MOV | BPF_X: /* dst = src */ 668 + if (dst_reg == src_reg) 669 + break; 670 + EMIT(PPC_RAW_MR(dst_reg, src_reg)); 671 + EMIT(PPC_RAW_MR(dst_reg_h, src_reg_h)); 672 + break; 673 + case BPF_ALU | BPF_MOV | BPF_X: /* (u32) dst = src */ 674 + /* special mov32 for zext */ 675 + if (imm == 1) 676 + EMIT(PPC_RAW_LI(dst_reg_h, 0)); 677 + else if (dst_reg != src_reg) 678 + EMIT(PPC_RAW_MR(dst_reg, src_reg)); 679 + break; 680 + case BPF_ALU64 | BPF_MOV | BPF_K: /* dst = (s64) imm */ 681 + PPC_LI32(dst_reg, imm); 682 + PPC_EX32(dst_reg_h, imm); 683 + break; 684 + case BPF_ALU | BPF_MOV | BPF_K: /* (u32) dst = imm */ 685 + PPC_LI32(dst_reg, imm); 686 + break; 687 + 688 + /* 689 + * BPF_FROM_BE/LE 690 + */ 691 + case BPF_ALU | BPF_END | BPF_FROM_LE: 692 + switch (imm) { 693 + case 16: 694 + /* Copy 16 bits to upper part */ 695 + EMIT(PPC_RAW_RLWIMI(dst_reg, dst_reg, 16, 0, 15)); 696 + /* Rotate 8 bits right & mask */ 697 + EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, 24, 16, 31)); 698 + break; 699 + case 32: 700 + /* 701 + * Rotate word left by 8 bits: 702 + * 2 bytes are already in their final position 703 + * -- byte 2 and 4 (of bytes 1, 2, 3 and 4) 704 + */ 705 + EMIT(PPC_RAW_RLWINM(__REG_R0, dst_reg, 8, 0, 31)); 706 + /* Rotate 24 bits and insert byte 1 */ 707 + EMIT(PPC_RAW_RLWIMI(__REG_R0, dst_reg, 24, 0, 7)); 708 + /* Rotate 24 bits and insert byte 3 */ 709 + EMIT(PPC_RAW_RLWIMI(__REG_R0, dst_reg, 24, 16, 23)); 710 + EMIT(PPC_RAW_MR(dst_reg, __REG_R0)); 711 + break; 712 + case 64: 713 + bpf_set_seen_register(ctx, tmp_reg); 714 + EMIT(PPC_RAW_RLWINM(tmp_reg, dst_reg, 8, 0, 31)); 715 + EMIT(PPC_RAW_RLWINM(__REG_R0, dst_reg_h, 8, 0, 31)); 716 + /* Rotate 24 bits and insert byte 1 */ 717 + EMIT(PPC_RAW_RLWIMI(tmp_reg, dst_reg, 24, 0, 7)); 718 + EMIT(PPC_RAW_RLWIMI(__REG_R0, dst_reg_h, 24, 0, 7)); 719 + /* Rotate 24 bits and insert byte 3 */ 720 + EMIT(PPC_RAW_RLWIMI(tmp_reg, dst_reg, 24, 16, 23)); 721 + EMIT(PPC_RAW_RLWIMI(__REG_R0, dst_reg_h, 24, 16, 23)); 722 + EMIT(PPC_RAW_MR(dst_reg, __REG_R0)); 723 + EMIT(PPC_RAW_MR(dst_reg_h, tmp_reg)); 724 + break; 725 + } 726 + break; 727 + case BPF_ALU | BPF_END | BPF_FROM_BE: 728 + switch (imm) { 729 + case 16: 730 + /* zero-extend 16 bits into 32 bits */ 731 + EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, 0, 16, 31)); 732 + break; 733 + case 32: 734 + case 64: 735 + /* nop */ 736 + break; 737 + } 738 + break; 739 + 740 + /* 741 + * BPF_ST(X) 742 + */ 743 + case BPF_STX | BPF_MEM | BPF_B: /* *(u8 *)(dst + off) = src */ 744 + EMIT(PPC_RAW_STB(src_reg, dst_reg, off)); 745 + break; 746 + case BPF_ST | BPF_MEM | BPF_B: /* *(u8 *)(dst + off) = imm */ 747 + PPC_LI32(__REG_R0, imm); 748 + EMIT(PPC_RAW_STB(__REG_R0, dst_reg, off)); 749 + break; 750 + case BPF_STX | BPF_MEM | BPF_H: /* (u16 *)(dst + off) = src */ 751 + EMIT(PPC_RAW_STH(src_reg, dst_reg, off)); 752 + break; 753 + case BPF_ST | BPF_MEM | BPF_H: /* (u16 *)(dst + off) = imm */ 754 + PPC_LI32(__REG_R0, imm); 755 + EMIT(PPC_RAW_STH(__REG_R0, dst_reg, off)); 756 + break; 757 + case BPF_STX | BPF_MEM | BPF_W: /* *(u32 *)(dst + off) = src */ 758 + EMIT(PPC_RAW_STW(src_reg, dst_reg, off)); 759 + break; 760 + case BPF_ST | BPF_MEM | BPF_W: /* *(u32 *)(dst + off) = imm */ 761 + PPC_LI32(__REG_R0, imm); 762 + EMIT(PPC_RAW_STW(__REG_R0, dst_reg, off)); 763 + break; 764 + case BPF_STX | BPF_MEM | BPF_DW: /* (u64 *)(dst + off) = src */ 765 + EMIT(PPC_RAW_STW(src_reg_h, dst_reg, off)); 766 + EMIT(PPC_RAW_STW(src_reg, dst_reg, off + 4)); 767 + break; 768 + case BPF_ST | BPF_MEM | BPF_DW: /* *(u64 *)(dst + off) = imm */ 769 + PPC_LI32(__REG_R0, imm); 770 + EMIT(PPC_RAW_STW(__REG_R0, dst_reg, off + 4)); 771 + PPC_EX32(__REG_R0, imm); 772 + EMIT(PPC_RAW_STW(__REG_R0, dst_reg, off)); 773 + break; 774 + 775 + /* 776 + * BPF_STX XADD (atomic_add) 777 + */ 778 + case BPF_STX | BPF_XADD | BPF_W: /* *(u32 *)(dst + off) += src */ 779 + bpf_set_seen_register(ctx, tmp_reg); 780 + /* Get offset into TMP_REG */ 781 + EMIT(PPC_RAW_LI(tmp_reg, off)); 782 + /* load value from memory into r0 */ 783 + EMIT(PPC_RAW_LWARX(__REG_R0, tmp_reg, dst_reg, 0)); 784 + /* add value from src_reg into this */ 785 + EMIT(PPC_RAW_ADD(__REG_R0, __REG_R0, src_reg)); 786 + /* store result back */ 787 + EMIT(PPC_RAW_STWCX(__REG_R0, tmp_reg, dst_reg)); 788 + /* we're done if this succeeded */ 789 + PPC_BCC_SHORT(COND_NE, (ctx->idx - 3) * 4); 790 + break; 791 + 792 + case BPF_STX | BPF_XADD | BPF_DW: /* *(u64 *)(dst + off) += src */ 793 + return -EOPNOTSUPP; 794 + 795 + /* 796 + * BPF_LDX 797 + */ 798 + case BPF_LDX | BPF_MEM | BPF_B: /* dst = *(u8 *)(ul) (src + off) */ 799 + EMIT(PPC_RAW_LBZ(dst_reg, src_reg, off)); 800 + if (!fp->aux->verifier_zext) 801 + EMIT(PPC_RAW_LI(dst_reg_h, 0)); 802 + break; 803 + case BPF_LDX | BPF_MEM | BPF_H: /* dst = *(u16 *)(ul) (src + off) */ 804 + EMIT(PPC_RAW_LHZ(dst_reg, src_reg, off)); 805 + if (!fp->aux->verifier_zext) 806 + EMIT(PPC_RAW_LI(dst_reg_h, 0)); 807 + break; 808 + case BPF_LDX | BPF_MEM | BPF_W: /* dst = *(u32 *)(ul) (src + off) */ 809 + EMIT(PPC_RAW_LWZ(dst_reg, src_reg, off)); 810 + if (!fp->aux->verifier_zext) 811 + EMIT(PPC_RAW_LI(dst_reg_h, 0)); 812 + break; 813 + case BPF_LDX | BPF_MEM | BPF_DW: /* dst = *(u64 *)(ul) (src + off) */ 814 + EMIT(PPC_RAW_LWZ(dst_reg_h, src_reg, off)); 815 + EMIT(PPC_RAW_LWZ(dst_reg, src_reg, off + 4)); 816 + break; 817 + 818 + /* 819 + * Doubleword load 820 + * 16 byte instruction that uses two 'struct bpf_insn' 821 + */ 822 + case BPF_LD | BPF_IMM | BPF_DW: /* dst = (u64) imm */ 823 + PPC_LI32(dst_reg_h, (u32)insn[i + 1].imm); 824 + PPC_LI32(dst_reg, (u32)insn[i].imm); 825 + /* Adjust for two bpf instructions */ 826 + addrs[++i] = ctx->idx * 4; 827 + break; 828 + 829 + /* 830 + * Return/Exit 831 + */ 832 + case BPF_JMP | BPF_EXIT: 833 + /* 834 + * If this isn't the very last instruction, branch to 835 + * the epilogue. If we _are_ the last instruction, 836 + * we'll just fall through to the epilogue. 837 + */ 838 + if (i != flen - 1) 839 + PPC_JMP(exit_addr); 840 + /* else fall through to the epilogue */ 841 + break; 842 + 843 + /* 844 + * Call kernel helper or bpf function 845 + */ 846 + case BPF_JMP | BPF_CALL: 847 + ctx->seen |= SEEN_FUNC; 848 + 849 + ret = bpf_jit_get_func_addr(fp, &insn[i], extra_pass, 850 + &func_addr, &func_addr_fixed); 851 + if (ret < 0) 852 + return ret; 853 + 854 + if (bpf_is_seen_register(ctx, bpf_to_ppc(ctx, BPF_REG_5))) { 855 + EMIT(PPC_RAW_STW(bpf_to_ppc(ctx, BPF_REG_5) - 1, __REG_R1, 8)); 856 + EMIT(PPC_RAW_STW(bpf_to_ppc(ctx, BPF_REG_5), __REG_R1, 12)); 857 + } 858 + 859 + bpf_jit_emit_func_call_rel(image, ctx, func_addr); 860 + 861 + EMIT(PPC_RAW_MR(bpf_to_ppc(ctx, BPF_REG_0) - 1, __REG_R3)); 862 + EMIT(PPC_RAW_MR(bpf_to_ppc(ctx, BPF_REG_0), __REG_R4)); 863 + break; 864 + 865 + /* 866 + * Jumps and branches 867 + */ 868 + case BPF_JMP | BPF_JA: 869 + PPC_JMP(addrs[i + 1 + off]); 870 + break; 871 + 872 + case BPF_JMP | BPF_JGT | BPF_K: 873 + case BPF_JMP | BPF_JGT | BPF_X: 874 + case BPF_JMP | BPF_JSGT | BPF_K: 875 + case BPF_JMP | BPF_JSGT | BPF_X: 876 + case BPF_JMP32 | BPF_JGT | BPF_K: 877 + case BPF_JMP32 | BPF_JGT | BPF_X: 878 + case BPF_JMP32 | BPF_JSGT | BPF_K: 879 + case BPF_JMP32 | BPF_JSGT | BPF_X: 880 + true_cond = COND_GT; 881 + goto cond_branch; 882 + case BPF_JMP | BPF_JLT | BPF_K: 883 + case BPF_JMP | BPF_JLT | BPF_X: 884 + case BPF_JMP | BPF_JSLT | BPF_K: 885 + case BPF_JMP | BPF_JSLT | BPF_X: 886 + case BPF_JMP32 | BPF_JLT | BPF_K: 887 + case BPF_JMP32 | BPF_JLT | BPF_X: 888 + case BPF_JMP32 | BPF_JSLT | BPF_K: 889 + case BPF_JMP32 | BPF_JSLT | BPF_X: 890 + true_cond = COND_LT; 891 + goto cond_branch; 892 + case BPF_JMP | BPF_JGE | BPF_K: 893 + case BPF_JMP | BPF_JGE | BPF_X: 894 + case BPF_JMP | BPF_JSGE | BPF_K: 895 + case BPF_JMP | BPF_JSGE | BPF_X: 896 + case BPF_JMP32 | BPF_JGE | BPF_K: 897 + case BPF_JMP32 | BPF_JGE | BPF_X: 898 + case BPF_JMP32 | BPF_JSGE | BPF_K: 899 + case BPF_JMP32 | BPF_JSGE | BPF_X: 900 + true_cond = COND_GE; 901 + goto cond_branch; 902 + case BPF_JMP | BPF_JLE | BPF_K: 903 + case BPF_JMP | BPF_JLE | BPF_X: 904 + case BPF_JMP | BPF_JSLE | BPF_K: 905 + case BPF_JMP | BPF_JSLE | BPF_X: 906 + case BPF_JMP32 | BPF_JLE | BPF_K: 907 + case BPF_JMP32 | BPF_JLE | BPF_X: 908 + case BPF_JMP32 | BPF_JSLE | BPF_K: 909 + case BPF_JMP32 | BPF_JSLE | BPF_X: 910 + true_cond = COND_LE; 911 + goto cond_branch; 912 + case BPF_JMP | BPF_JEQ | BPF_K: 913 + case BPF_JMP | BPF_JEQ | BPF_X: 914 + case BPF_JMP32 | BPF_JEQ | BPF_K: 915 + case BPF_JMP32 | BPF_JEQ | BPF_X: 916 + true_cond = COND_EQ; 917 + goto cond_branch; 918 + case BPF_JMP | BPF_JNE | BPF_K: 919 + case BPF_JMP | BPF_JNE | BPF_X: 920 + case BPF_JMP32 | BPF_JNE | BPF_K: 921 + case BPF_JMP32 | BPF_JNE | BPF_X: 922 + true_cond = COND_NE; 923 + goto cond_branch; 924 + case BPF_JMP | BPF_JSET | BPF_K: 925 + case BPF_JMP | BPF_JSET | BPF_X: 926 + case BPF_JMP32 | BPF_JSET | BPF_K: 927 + case BPF_JMP32 | BPF_JSET | BPF_X: 928 + true_cond = COND_NE; 929 + /* fallthrough; */ 930 + 931 + cond_branch: 932 + switch (code) { 933 + case BPF_JMP | BPF_JGT | BPF_X: 934 + case BPF_JMP | BPF_JLT | BPF_X: 935 + case BPF_JMP | BPF_JGE | BPF_X: 936 + case BPF_JMP | BPF_JLE | BPF_X: 937 + case BPF_JMP | BPF_JEQ | BPF_X: 938 + case BPF_JMP | BPF_JNE | BPF_X: 939 + /* unsigned comparison */ 940 + EMIT(PPC_RAW_CMPLW(dst_reg_h, src_reg_h)); 941 + PPC_BCC_SHORT(COND_NE, (ctx->idx + 2) * 4); 942 + EMIT(PPC_RAW_CMPLW(dst_reg, src_reg)); 943 + break; 944 + case BPF_JMP32 | BPF_JGT | BPF_X: 945 + case BPF_JMP32 | BPF_JLT | BPF_X: 946 + case BPF_JMP32 | BPF_JGE | BPF_X: 947 + case BPF_JMP32 | BPF_JLE | BPF_X: 948 + case BPF_JMP32 | BPF_JEQ | BPF_X: 949 + case BPF_JMP32 | BPF_JNE | BPF_X: 950 + /* unsigned comparison */ 951 + EMIT(PPC_RAW_CMPLW(dst_reg, src_reg)); 952 + break; 953 + case BPF_JMP | BPF_JSGT | BPF_X: 954 + case BPF_JMP | BPF_JSLT | BPF_X: 955 + case BPF_JMP | BPF_JSGE | BPF_X: 956 + case BPF_JMP | BPF_JSLE | BPF_X: 957 + /* signed comparison */ 958 + EMIT(PPC_RAW_CMPW(dst_reg_h, src_reg_h)); 959 + PPC_BCC_SHORT(COND_NE, (ctx->idx + 2) * 4); 960 + EMIT(PPC_RAW_CMPLW(dst_reg, src_reg)); 961 + break; 962 + case BPF_JMP32 | BPF_JSGT | BPF_X: 963 + case BPF_JMP32 | BPF_JSLT | BPF_X: 964 + case BPF_JMP32 | BPF_JSGE | BPF_X: 965 + case BPF_JMP32 | BPF_JSLE | BPF_X: 966 + /* signed comparison */ 967 + EMIT(PPC_RAW_CMPW(dst_reg, src_reg)); 968 + break; 969 + case BPF_JMP | BPF_JSET | BPF_X: 970 + EMIT(PPC_RAW_AND_DOT(__REG_R0, dst_reg_h, src_reg_h)); 971 + PPC_BCC_SHORT(COND_NE, (ctx->idx + 2) * 4); 972 + EMIT(PPC_RAW_AND_DOT(__REG_R0, dst_reg, src_reg)); 973 + break; 974 + case BPF_JMP32 | BPF_JSET | BPF_X: { 975 + EMIT(PPC_RAW_AND_DOT(__REG_R0, dst_reg, src_reg)); 976 + break; 977 + case BPF_JMP | BPF_JNE | BPF_K: 978 + case BPF_JMP | BPF_JEQ | BPF_K: 979 + case BPF_JMP | BPF_JGT | BPF_K: 980 + case BPF_JMP | BPF_JLT | BPF_K: 981 + case BPF_JMP | BPF_JGE | BPF_K: 982 + case BPF_JMP | BPF_JLE | BPF_K: 983 + /* 984 + * Need sign-extended load, so only positive 985 + * values can be used as imm in cmplwi 986 + */ 987 + if (imm >= 0 && imm < 32768) { 988 + EMIT(PPC_RAW_CMPLWI(dst_reg_h, 0)); 989 + PPC_BCC_SHORT(COND_NE, (ctx->idx + 2) * 4); 990 + EMIT(PPC_RAW_CMPLWI(dst_reg, imm)); 991 + } else { 992 + /* sign-extending load ... but unsigned comparison */ 993 + PPC_EX32(__REG_R0, imm); 994 + EMIT(PPC_RAW_CMPLW(dst_reg_h, __REG_R0)); 995 + PPC_LI32(__REG_R0, imm); 996 + PPC_BCC_SHORT(COND_NE, (ctx->idx + 2) * 4); 997 + EMIT(PPC_RAW_CMPLW(dst_reg, __REG_R0)); 998 + } 999 + break; 1000 + case BPF_JMP32 | BPF_JNE | BPF_K: 1001 + case BPF_JMP32 | BPF_JEQ | BPF_K: 1002 + case BPF_JMP32 | BPF_JGT | BPF_K: 1003 + case BPF_JMP32 | BPF_JLT | BPF_K: 1004 + case BPF_JMP32 | BPF_JGE | BPF_K: 1005 + case BPF_JMP32 | BPF_JLE | BPF_K: 1006 + if (imm >= 0 && imm < 65536) { 1007 + EMIT(PPC_RAW_CMPLWI(dst_reg, imm)); 1008 + } else { 1009 + PPC_LI32(__REG_R0, imm); 1010 + EMIT(PPC_RAW_CMPLW(dst_reg, __REG_R0)); 1011 + } 1012 + break; 1013 + } 1014 + case BPF_JMP | BPF_JSGT | BPF_K: 1015 + case BPF_JMP | BPF_JSLT | BPF_K: 1016 + case BPF_JMP | BPF_JSGE | BPF_K: 1017 + case BPF_JMP | BPF_JSLE | BPF_K: 1018 + if (imm >= 0 && imm < 65536) { 1019 + EMIT(PPC_RAW_CMPWI(dst_reg_h, imm < 0 ? -1 : 0)); 1020 + PPC_BCC_SHORT(COND_NE, (ctx->idx + 2) * 4); 1021 + EMIT(PPC_RAW_CMPLWI(dst_reg, imm)); 1022 + } else { 1023 + /* sign-extending load */ 1024 + EMIT(PPC_RAW_CMPWI(dst_reg_h, imm < 0 ? -1 : 0)); 1025 + PPC_LI32(__REG_R0, imm); 1026 + PPC_BCC_SHORT(COND_NE, (ctx->idx + 2) * 4); 1027 + EMIT(PPC_RAW_CMPLW(dst_reg, __REG_R0)); 1028 + } 1029 + break; 1030 + case BPF_JMP32 | BPF_JSGT | BPF_K: 1031 + case BPF_JMP32 | BPF_JSLT | BPF_K: 1032 + case BPF_JMP32 | BPF_JSGE | BPF_K: 1033 + case BPF_JMP32 | BPF_JSLE | BPF_K: 1034 + /* 1035 + * signed comparison, so any 16-bit value 1036 + * can be used in cmpwi 1037 + */ 1038 + if (imm >= -32768 && imm < 32768) { 1039 + EMIT(PPC_RAW_CMPWI(dst_reg, imm)); 1040 + } else { 1041 + /* sign-extending load */ 1042 + PPC_LI32(__REG_R0, imm); 1043 + EMIT(PPC_RAW_CMPW(dst_reg, __REG_R0)); 1044 + } 1045 + break; 1046 + case BPF_JMP | BPF_JSET | BPF_K: 1047 + /* andi does not sign-extend the immediate */ 1048 + if (imm >= 0 && imm < 32768) { 1049 + /* PPC_ANDI is _only/always_ dot-form */ 1050 + EMIT(PPC_RAW_ANDI(__REG_R0, dst_reg, imm)); 1051 + } else { 1052 + PPC_LI32(__REG_R0, imm); 1053 + if (imm < 0) { 1054 + EMIT(PPC_RAW_CMPWI(dst_reg_h, 0)); 1055 + PPC_BCC_SHORT(COND_NE, (ctx->idx + 2) * 4); 1056 + } 1057 + EMIT(PPC_RAW_AND_DOT(__REG_R0, dst_reg, __REG_R0)); 1058 + } 1059 + break; 1060 + case BPF_JMP32 | BPF_JSET | BPF_K: 1061 + /* andi does not sign-extend the immediate */ 1062 + if (imm >= -32768 && imm < 32768) { 1063 + /* PPC_ANDI is _only/always_ dot-form */ 1064 + EMIT(PPC_RAW_ANDI(__REG_R0, dst_reg, imm)); 1065 + } else { 1066 + PPC_LI32(__REG_R0, imm); 1067 + EMIT(PPC_RAW_AND_DOT(__REG_R0, dst_reg, __REG_R0)); 1068 + } 1069 + break; 1070 + } 1071 + PPC_BCC(true_cond, addrs[i + 1 + off]); 1072 + break; 1073 + 1074 + /* 1075 + * Tail call 1076 + */ 1077 + case BPF_JMP | BPF_TAIL_CALL: 1078 + ctx->seen |= SEEN_TAILCALL; 1079 + bpf_jit_emit_tail_call(image, ctx, addrs[i + 1]); 1080 + break; 1081 + 1082 + default: 1083 + /* 1084 + * The filter contains something cruel & unusual. 1085 + * We don't handle it, but also there shouldn't be 1086 + * anything missing from our list. 1087 + */ 1088 + pr_err_ratelimited("eBPF filter opcode %04x (@%d) unsupported\n", code, i); 1089 + return -EOPNOTSUPP; 1090 + } 1091 + if (BPF_CLASS(code) == BPF_ALU && !fp->aux->verifier_zext && 1092 + !insn_is_zext(&insn[i + 1])) 1093 + EMIT(PPC_RAW_LI(dst_reg_h, 0)); 1094 + } 1095 + 1096 + /* Set end-of-body-code address for exit. */ 1097 + addrs[i] = ctx->idx * 4; 1098 + 1099 + return 0; 1100 + }

+15 -280

arch/powerpc/net/bpf_jit_comp64.c

··· 18 18 19 19 #include "bpf_jit64.h" 20 20 21 - static void bpf_jit_fill_ill_insns(void *area, unsigned int size) 22 - { 23 - memset32(area, BREAKPOINT_INSTRUCTION, size/4); 24 - } 25 - 26 - static inline void bpf_flush_icache(void *start, void *end) 27 - { 28 - smp_wmb(); 29 - flush_icache_range((unsigned long)start, (unsigned long)end); 30 - } 31 - 32 - static inline bool bpf_is_seen_register(struct codegen_context *ctx, int i) 33 - { 34 - return (ctx->seen & (1 << (31 - b2p[i]))); 35 - } 36 - 37 - static inline void bpf_set_seen_register(struct codegen_context *ctx, int i) 38 - { 39 - ctx->seen |= (1 << (31 - b2p[i])); 40 - } 41 - 42 21 static inline bool bpf_has_stack_frame(struct codegen_context *ctx) 43 22 { 44 23 /* ··· 26 47 * - the bpf program uses its stack area 27 48 * The latter condition is deduced from the usage of BPF_REG_FP 28 49 */ 29 - return ctx->seen & SEEN_FUNC || bpf_is_seen_register(ctx, BPF_REG_FP); 50 + return ctx->seen & SEEN_FUNC || bpf_is_seen_register(ctx, b2p[BPF_REG_FP]); 30 51 } 31 52 32 53 /* ··· 64 85 BUG(); 65 86 } 66 87 67 - static void bpf_jit_build_prologue(u32 *image, struct codegen_context *ctx) 88 + void bpf_jit_realloc_regs(struct codegen_context *ctx) 89 + { 90 + } 91 + 92 + void bpf_jit_build_prologue(u32 *image, struct codegen_context *ctx) 68 93 { 69 94 int i; 70 95 ··· 107 124 * in the protected zone below the previous stack frame 108 125 */ 109 126 for (i = BPF_REG_6; i <= BPF_REG_10; i++) 110 - if (bpf_is_seen_register(ctx, i)) 127 + if (bpf_is_seen_register(ctx, b2p[i])) 111 128 PPC_BPF_STL(b2p[i], 1, bpf_jit_stack_offsetof(ctx, b2p[i])); 112 129 113 130 /* Setup frame pointer to point to the bpf stack area */ 114 - if (bpf_is_seen_register(ctx, BPF_REG_FP)) 131 + if (bpf_is_seen_register(ctx, b2p[BPF_REG_FP])) 115 132 EMIT(PPC_RAW_ADDI(b2p[BPF_REG_FP], 1, 116 133 STACK_FRAME_MIN_SIZE + ctx->stack_size)); 117 134 } ··· 122 139 123 140 /* Restore NVRs */ 124 141 for (i = BPF_REG_6; i <= BPF_REG_10; i++) 125 - if (bpf_is_seen_register(ctx, i)) 142 + if (bpf_is_seen_register(ctx, b2p[i])) 126 143 PPC_BPF_LL(b2p[i], 1, bpf_jit_stack_offsetof(ctx, b2p[i])); 127 144 128 145 /* Tear down our stack frame */ ··· 135 152 } 136 153 } 137 154 138 - static void bpf_jit_build_epilogue(u32 *image, struct codegen_context *ctx) 155 + void bpf_jit_build_epilogue(u32 *image, struct codegen_context *ctx) 139 156 { 140 157 bpf_jit_emit_common_epilogue(image, ctx); 141 158 ··· 170 187 EMIT(PPC_RAW_BLRL()); 171 188 } 172 189 173 - static void bpf_jit_emit_func_call_rel(u32 *image, struct codegen_context *ctx, 174 - u64 func) 190 + void bpf_jit_emit_func_call_rel(u32 *image, struct codegen_context *ctx, u64 func) 175 191 { 176 192 unsigned int i, ctx_idx = ctx->idx; 177 193 ··· 271 289 } 272 290 273 291 /* Assemble the body code between the prologue & epilogue */ 274 - static int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, 275 - struct codegen_context *ctx, 276 - u32 *addrs, bool extra_pass) 292 + int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, struct codegen_context *ctx, 293 + u32 *addrs, bool extra_pass) 277 294 { 278 295 const struct bpf_insn *insn = fp->insnsi; 279 296 int flen = fp->len; ··· 311 330 * any issues. 312 331 */ 313 332 if (dst_reg >= BPF_PPC_NVR_MIN && dst_reg < 32) 314 - bpf_set_seen_register(ctx, insn[i].dst_reg); 333 + bpf_set_seen_register(ctx, dst_reg); 315 334 if (src_reg >= BPF_PPC_NVR_MIN && src_reg < 32) 316 - bpf_set_seen_register(ctx, insn[i].src_reg); 335 + bpf_set_seen_register(ctx, src_reg); 317 336 318 337 switch (code) { 319 338 /* ··· 1006 1025 addrs[i] = ctx->idx * 4; 1007 1026 1008 1027 return 0; 1009 - } 1010 - 1011 - /* Fix the branch target addresses for subprog calls */ 1012 - static int bpf_jit_fixup_subprog_calls(struct bpf_prog *fp, u32 *image, 1013 - struct codegen_context *ctx, u32 *addrs) 1014 - { 1015 - const struct bpf_insn *insn = fp->insnsi; 1016 - bool func_addr_fixed; 1017 - u64 func_addr; 1018 - u32 tmp_idx; 1019 - int i, ret; 1020 - 1021 - for (i = 0; i < fp->len; i++) { 1022 - /* 1023 - * During the extra pass, only the branch target addresses for 1024 - * the subprog calls need to be fixed. All other instructions 1025 - * can left untouched. 1026 - * 1027 - * The JITed image length does not change because we already 1028 - * ensure that the JITed instruction sequence for these calls 1029 - * are of fixed length by padding them with NOPs. 1030 - */ 1031 - if (insn[i].code == (BPF_JMP | BPF_CALL) && 1032 - insn[i].src_reg == BPF_PSEUDO_CALL) { 1033 - ret = bpf_jit_get_func_addr(fp, &insn[i], true, 1034 - &func_addr, 1035 - &func_addr_fixed); 1036 - if (ret < 0) 1037 - return ret; 1038 - 1039 - /* 1040 - * Save ctx->idx as this would currently point to the 1041 - * end of the JITed image and set it to the offset of 1042 - * the instruction sequence corresponding to the 1043 - * subprog call temporarily. 1044 - */ 1045 - tmp_idx = ctx->idx; 1046 - ctx->idx = addrs[i] / 4; 1047 - bpf_jit_emit_func_call_rel(image, ctx, func_addr); 1048 - 1049 - /* 1050 - * Restore ctx->idx here. This is safe as the length 1051 - * of the JITed sequence remains unchanged. 1052 - */ 1053 - ctx->idx = tmp_idx; 1054 - } 1055 - } 1056 - 1057 - return 0; 1058 - } 1059 - 1060 - struct powerpc64_jit_data { 1061 - struct bpf_binary_header *header; 1062 - u32 *addrs; 1063 - u8 *image; 1064 - u32 proglen; 1065 - struct codegen_context ctx; 1066 - }; 1067 - 1068 - bool bpf_jit_needs_zext(void) 1069 - { 1070 - return true; 1071 - } 1072 - 1073 - struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp) 1074 - { 1075 - u32 proglen; 1076 - u32 alloclen; 1077 - u8 *image = NULL; 1078 - u32 *code_base; 1079 - u32 *addrs; 1080 - struct powerpc64_jit_data *jit_data; 1081 - struct codegen_context cgctx; 1082 - int pass; 1083 - int flen; 1084 - struct bpf_binary_header *bpf_hdr; 1085 - struct bpf_prog *org_fp = fp; 1086 - struct bpf_prog *tmp_fp; 1087 - bool bpf_blinded = false; 1088 - bool extra_pass = false; 1089 - 1090 - if (!fp->jit_requested) 1091 - return org_fp; 1092 - 1093 - tmp_fp = bpf_jit_blind_constants(org_fp); 1094 - if (IS_ERR(tmp_fp)) 1095 - return org_fp; 1096 - 1097 - if (tmp_fp != org_fp) { 1098 - bpf_blinded = true; 1099 - fp = tmp_fp; 1100 - } 1101 - 1102 - jit_data = fp->aux->jit_data; 1103 - if (!jit_data) { 1104 - jit_data = kzalloc(sizeof(*jit_data), GFP_KERNEL); 1105 - if (!jit_data) { 1106 - fp = org_fp; 1107 - goto out; 1108 - } 1109 - fp->aux->jit_data = jit_data; 1110 - } 1111 - 1112 - flen = fp->len; 1113 - addrs = jit_data->addrs; 1114 - if (addrs) { 1115 - cgctx = jit_data->ctx; 1116 - image = jit_data->image; 1117 - bpf_hdr = jit_data->header; 1118 - proglen = jit_data->proglen; 1119 - alloclen = proglen + FUNCTION_DESCR_SIZE; 1120 - extra_pass = true; 1121 - goto skip_init_ctx; 1122 - } 1123 - 1124 - addrs = kcalloc(flen + 1, sizeof(*addrs), GFP_KERNEL); 1125 - if (addrs == NULL) { 1126 - fp = org_fp; 1127 - goto out_addrs; 1128 - } 1129 - 1130 - memset(&cgctx, 0, sizeof(struct codegen_context)); 1131 - 1132 - /* Make sure that the stack is quadword aligned. */ 1133 - cgctx.stack_size = round_up(fp->aux->stack_depth, 16); 1134 - 1135 - /* Scouting faux-generate pass 0 */ 1136 - if (bpf_jit_build_body(fp, 0, &cgctx, addrs, false)) { 1137 - /* We hit something illegal or unsupported. */ 1138 - fp = org_fp; 1139 - goto out_addrs; 1140 - } 1141 - 1142 - /* 1143 - * If we have seen a tail call, we need a second pass. 1144 - * This is because bpf_jit_emit_common_epilogue() is called 1145 - * from bpf_jit_emit_tail_call() with a not yet stable ctx->seen. 1146 - */ 1147 - if (cgctx.seen & SEEN_TAILCALL) { 1148 - cgctx.idx = 0; 1149 - if (bpf_jit_build_body(fp, 0, &cgctx, addrs, false)) { 1150 - fp = org_fp; 1151 - goto out_addrs; 1152 - } 1153 - } 1154 - 1155 - /* 1156 - * Pretend to build prologue, given the features we've seen. This will 1157 - * update ctgtx.idx as it pretends to output instructions, then we can 1158 - * calculate total size from idx. 1159 - */ 1160 - bpf_jit_build_prologue(0, &cgctx); 1161 - bpf_jit_build_epilogue(0, &cgctx); 1162 - 1163 - proglen = cgctx.idx * 4; 1164 - alloclen = proglen + FUNCTION_DESCR_SIZE; 1165 - 1166 - bpf_hdr = bpf_jit_binary_alloc(alloclen, &image, 4, 1167 - bpf_jit_fill_ill_insns); 1168 - if (!bpf_hdr) { 1169 - fp = org_fp; 1170 - goto out_addrs; 1171 - } 1172 - 1173 - skip_init_ctx: 1174 - code_base = (u32 *)(image + FUNCTION_DESCR_SIZE); 1175 - 1176 - if (extra_pass) { 1177 - /* 1178 - * Do not touch the prologue and epilogue as they will remain 1179 - * unchanged. Only fix the branch target address for subprog 1180 - * calls in the body. 1181 - * 1182 - * This does not change the offsets and lengths of the subprog 1183 - * call instruction sequences and hence, the size of the JITed 1184 - * image as well. 1185 - */ 1186 - bpf_jit_fixup_subprog_calls(fp, code_base, &cgctx, addrs); 1187 - 1188 - /* There is no need to perform the usual passes. */ 1189 - goto skip_codegen_passes; 1190 - } 1191 - 1192 - /* Code generation passes 1-2 */ 1193 - for (pass = 1; pass < 3; pass++) { 1194 - /* Now build the prologue, body code & epilogue for real. */ 1195 - cgctx.idx = 0; 1196 - bpf_jit_build_prologue(code_base, &cgctx); 1197 - bpf_jit_build_body(fp, code_base, &cgctx, addrs, extra_pass); 1198 - bpf_jit_build_epilogue(code_base, &cgctx); 1199 - 1200 - if (bpf_jit_enable > 1) 1201 - pr_info("Pass %d: shrink = %d, seen = 0x%x\n", pass, 1202 - proglen - (cgctx.idx * 4), cgctx.seen); 1203 - } 1204 - 1205 - skip_codegen_passes: 1206 - if (bpf_jit_enable > 1) 1207 - /* 1208 - * Note that we output the base address of the code_base 1209 - * rather than image, since opcodes are in code_base. 1210 - */ 1211 - bpf_jit_dump(flen, proglen, pass, code_base); 1212 - 1213 - #ifdef PPC64_ELF_ABI_v1 1214 - /* Function descriptor nastiness: Address + TOC */ 1215 - ((u64 *)image)[0] = (u64)code_base; 1216 - ((u64 *)image)[1] = local_paca->kernel_toc; 1217 - #endif 1218 - 1219 - fp->bpf_func = (void *)image; 1220 - fp->jited = 1; 1221 - fp->jited_len = alloclen; 1222 - 1223 - bpf_flush_icache(bpf_hdr, (u8 *)bpf_hdr + (bpf_hdr->pages * PAGE_SIZE)); 1224 - if (!fp->is_func || extra_pass) { 1225 - bpf_prog_fill_jited_linfo(fp, addrs); 1226 - out_addrs: 1227 - kfree(addrs); 1228 - kfree(jit_data); 1229 - fp->aux->jit_data = NULL; 1230 - } else { 1231 - jit_data->addrs = addrs; 1232 - jit_data->ctx = cgctx; 1233 - jit_data->proglen = proglen; 1234 - jit_data->image = image; 1235 - jit_data->header = bpf_hdr; 1236 - } 1237 - 1238 - out: 1239 - if (bpf_blinded) 1240 - bpf_jit_prog_release_other(fp, fp == org_fp ? tmp_fp : org_fp); 1241 - 1242 - return fp; 1243 - } 1244 - 1245 - /* Overriding bpf_jit_free() as we don't set images read-only. */ 1246 - void bpf_jit_free(struct bpf_prog *fp) 1247 - { 1248 - unsigned long addr = (unsigned long)fp->bpf_func & PAGE_MASK; 1249 - struct bpf_binary_header *bpf_hdr = (void *)addr; 1250 - 1251 - if (fp->jited) 1252 - bpf_jit_binary_free(bpf_hdr); 1253 - 1254 - bpf_prog_unlock_free(fp); 1255 1028 }

+16 -4

arch/powerpc/perf/core-book3s.c

··· 17 17 #include <asm/firmware.h> 18 18 #include <asm/ptrace.h> 19 19 #include <asm/code-patching.h> 20 + #include <asm/interrupt.h> 20 21 21 22 #ifdef CONFIG_PPC64 22 23 #include "internal.h" ··· 169 168 * they have not been setup using perf_read_regs() and so regs->result 170 169 * is something random. 171 170 */ 172 - return ((TRAP(regs) == 0xf00) && regs->result); 171 + return ((TRAP(regs) == INTERRUPT_PERFMON) && regs->result); 173 172 } 174 173 175 174 /* ··· 348 347 * hypervisor samples as well as samples in the kernel with 349 348 * interrupts off hence the userspace check. 350 349 */ 351 - if (TRAP(regs) != 0xf00) 350 + if (TRAP(regs) != INTERRUPT_PERFMON) 352 351 use_siar = 0; 353 352 else if ((ppmu->flags & PPMU_NO_SIAR)) 354 353 use_siar = 0; ··· 1964 1963 return -ENOENT; 1965 1964 } 1966 1965 1966 + /* 1967 + * PMU config registers have fields that are 1968 + * reserved and some specific values for bit fields are reserved. 1969 + * For ex., MMCRA[61:62] is Randome Sampling Mode (SM) 1970 + * and value of 0b11 to this field is reserved. 1971 + * Check for invalid values in attr.config. 1972 + */ 1973 + if (ppmu->check_attr_config && 1974 + ppmu->check_attr_config(event)) 1975 + return -EINVAL; 1976 + 1967 1977 event->hw.config_base = ev; 1968 1978 event->hw.idx = 0; 1969 1979 ··· 2218 2206 ppmu->get_mem_data_src) 2219 2207 ppmu->get_mem_data_src(&data.data_src, ppmu->flags, regs); 2220 2208 2221 - if (event->attr.sample_type & PERF_SAMPLE_WEIGHT && 2209 + if (event->attr.sample_type & PERF_SAMPLE_WEIGHT_TYPE && 2222 2210 ppmu->get_mem_weight) 2223 - ppmu->get_mem_weight(&data.weight.full); 2211 + ppmu->get_mem_weight(&data.weight.full, event->attr.sample_type); 2224 2212 2225 2213 if (perf_event_overflow(event, &data, regs)) 2226 2214 power_pmu_stop(event, 0);

+5 -5

arch/powerpc/perf/hv-24x7.c

··· 226 226 227 227 static struct kmem_cache *hv_page_cache; 228 228 229 - DEFINE_PER_CPU(int, hv_24x7_txn_flags); 230 - DEFINE_PER_CPU(int, hv_24x7_txn_err); 229 + static DEFINE_PER_CPU(int, hv_24x7_txn_flags); 230 + static DEFINE_PER_CPU(int, hv_24x7_txn_err); 231 231 232 232 struct hv_24x7_hw { 233 233 struct perf_event *events[255]; 234 234 }; 235 235 236 - DEFINE_PER_CPU(struct hv_24x7_hw, hv_24x7_hw); 236 + static DEFINE_PER_CPU(struct hv_24x7_hw, hv_24x7_hw); 237 237 238 238 /* 239 239 * request_buffer and result_buffer are not required to be 4k aligned, ··· 241 241 * the simplest way to ensure that. 242 242 */ 243 243 #define H24x7_DATA_BUFFER_SIZE 4096 244 - DEFINE_PER_CPU(char, hv_24x7_reqb[H24x7_DATA_BUFFER_SIZE]) __aligned(4096); 245 - DEFINE_PER_CPU(char, hv_24x7_resb[H24x7_DATA_BUFFER_SIZE]) __aligned(4096); 244 + static DEFINE_PER_CPU(char, hv_24x7_reqb[H24x7_DATA_BUFFER_SIZE]) __aligned(4096); 245 + static DEFINE_PER_CPU(char, hv_24x7_resb[H24x7_DATA_BUFFER_SIZE]) __aligned(4096); 246 246 247 247 static unsigned int max_num_requests(int interface_version) 248 248 {

+104 -11

arch/powerpc/perf/isa207-common.c

··· 21 21 PMU_FORMAT_ATTR(thresh_start, "config:36-39"); 22 22 PMU_FORMAT_ATTR(thresh_cmp, "config:40-49"); 23 23 24 - struct attribute *isa207_pmu_format_attr[] = { 24 + static struct attribute *isa207_pmu_format_attr[] = { 25 25 &format_attr_event.attr, 26 26 &format_attr_pmcxsel.attr, 27 27 &format_attr_mark.attr, ··· 275 275 276 276 sier = mfspr(SPRN_SIER); 277 277 val = (sier & ISA207_SIER_TYPE_MASK) >> ISA207_SIER_TYPE_SHIFT; 278 - if (val == 1 || val == 2) { 279 - idx = (sier & ISA207_SIER_LDST_MASK) >> ISA207_SIER_LDST_SHIFT; 280 - sub_idx = (sier & ISA207_SIER_DATA_SRC_MASK) >> ISA207_SIER_DATA_SRC_SHIFT; 278 + if (val != 1 && val != 2 && !(val == 7 && cpu_has_feature(CPU_FTR_ARCH_31))) 279 + return; 281 280 282 - dsrc->val = isa207_find_source(idx, sub_idx); 281 + idx = (sier & ISA207_SIER_LDST_MASK) >> ISA207_SIER_LDST_SHIFT; 282 + sub_idx = (sier & ISA207_SIER_DATA_SRC_MASK) >> ISA207_SIER_DATA_SRC_SHIFT; 283 + 284 + dsrc->val = isa207_find_source(idx, sub_idx); 285 + if (val == 7) { 286 + u64 mmcra; 287 + u32 op_type; 288 + 289 + /* 290 + * Type 0b111 denotes either larx or stcx instruction. Use the 291 + * MMCRA sampling bits [57:59] along with the type value 292 + * to determine the exact instruction type. If the sampling 293 + * criteria is neither load or store, set the type as default 294 + * to NA. 295 + */ 296 + mmcra = mfspr(SPRN_MMCRA); 297 + 298 + op_type = (mmcra >> MMCRA_SAMP_ELIG_SHIFT) & MMCRA_SAMP_ELIG_MASK; 299 + switch (op_type) { 300 + case 5: 301 + dsrc->val |= P(OP, LOAD); 302 + break; 303 + case 7: 304 + dsrc->val |= P(OP, STORE); 305 + break; 306 + default: 307 + dsrc->val |= P(OP, NA); 308 + break; 309 + } 310 + } else { 283 311 dsrc->val |= (val == 1) ? P(OP, LOAD) : P(OP, STORE); 284 312 } 285 313 } 286 314 287 - void isa207_get_mem_weight(u64 *weight) 315 + void isa207_get_mem_weight(u64 *weight, u64 type) 288 316 { 317 + union perf_sample_weight *weight_fields; 318 + u64 weight_lat; 289 319 u64 mmcra = mfspr(SPRN_MMCRA); 290 320 u64 exp = MMCRA_THR_CTR_EXP(mmcra); 291 321 u64 mantissa = MMCRA_THR_CTR_MANT(mmcra); ··· 325 295 if (cpu_has_feature(CPU_FTR_ARCH_31)) 326 296 mantissa = P10_MMCRA_THR_CTR_MANT(mmcra); 327 297 328 - if (val == 0 || val == 7) 329 - *weight = 0; 298 + if (val == 0 || (val == 7 && !cpu_has_feature(CPU_FTR_ARCH_31))) 299 + weight_lat = 0; 330 300 else 331 - *weight = mantissa << (2 * exp); 301 + weight_lat = mantissa << (2 * exp); 302 + 303 + /* 304 + * Use 64 bit weight field (full) if sample type is 305 + * WEIGHT. 306 + * 307 + * if sample type is WEIGHT_STRUCT: 308 + * - store memory latency in the lower 32 bits. 309 + * - For ISA v3.1, use remaining two 16 bit fields of 310 + * perf_sample_weight to store cycle counter values 311 + * from sier2. 312 + */ 313 + weight_fields = (union perf_sample_weight *)weight; 314 + if (type & PERF_SAMPLE_WEIGHT) 315 + weight_fields->full = weight_lat; 316 + else { 317 + weight_fields->var1_dw = (u32)weight_lat; 318 + if (cpu_has_feature(CPU_FTR_ARCH_31)) { 319 + weight_fields->var2_w = P10_SIER2_FINISH_CYC(mfspr(SPRN_SIER2)); 320 + weight_fields->var3_w = P10_SIER2_DISPATCH_CYC(mfspr(SPRN_SIER2)); 321 + } 322 + } 332 323 } 333 324 334 325 int isa207_get_constraint(u64 event, unsigned long *maskp, unsigned long *valp, u64 event_config1) ··· 498 447 * EBB events are pinned & exclusive, so this should never actually 499 448 * hit, but we leave it as a fallback in case. 500 449 */ 501 - mask |= CNST_EBB_VAL(ebb); 502 - value |= CNST_EBB_MASK; 450 + mask |= CNST_EBB_MASK; 451 + value |= CNST_EBB_VAL(ebb); 503 452 504 453 *maskp = mask; 505 454 *valp = value; ··· 744 693 } 745 694 746 695 return num_alt; 696 + } 697 + 698 + int isa3XX_check_attr_config(struct perf_event *ev) 699 + { 700 + u64 val, sample_mode; 701 + u64 event = ev->attr.config; 702 + 703 + val = (event >> EVENT_SAMPLE_SHIFT) & EVENT_SAMPLE_MASK; 704 + sample_mode = val & 0x3; 705 + 706 + /* 707 + * MMCRA[61:62] is Random Sampling Mode (SM). 708 + * value of 0b11 is reserved. 709 + */ 710 + if (sample_mode == 0x3) 711 + return -EINVAL; 712 + 713 + /* 714 + * Check for all reserved value 715 + * Source: Performance Monitoring Unit User Guide 716 + */ 717 + switch (val) { 718 + case 0x5: 719 + case 0x9: 720 + case 0xD: 721 + case 0x19: 722 + case 0x1D: 723 + case 0x1A: 724 + case 0x1E: 725 + return -EINVAL; 726 + } 727 + 728 + /* 729 + * MMCRA[48:51]/[52:55]) Threshold Start/Stop 730 + * Events Selection. 731 + * 0b11110000/0b00001111 is reserved. 732 + */ 733 + val = (event >> EVENT_THR_CTL_SHIFT) & EVENT_THR_CTL_MASK; 734 + if (((val & 0xF0) == 0xF0) || ((val & 0xF) == 0xF)) 735 + return -EINVAL; 736 + 737 + return 0; 747 738 }

+8 -1

arch/powerpc/perf/isa207-common.h

··· 220 220 /* Bits in MMCRA for PowerISA v2.07 */ 221 221 #define MMCRA_SAMP_MODE_SHIFT 1 222 222 #define MMCRA_SAMP_ELIG_SHIFT 4 223 + #define MMCRA_SAMP_ELIG_MASK 7 223 224 #define MMCRA_THR_CTL_SHIFT 8 224 225 #define MMCRA_THR_SEL_SHIFT 16 225 226 #define MMCRA_THR_CMP_SHIFT 32 ··· 266 265 #define ISA207_SIER_DATA_SRC_SHIFT 53 267 266 #define ISA207_SIER_DATA_SRC_MASK (0x7ull << ISA207_SIER_DATA_SRC_SHIFT) 268 267 268 + /* Bits in SIER2/SIER3 for Power10 */ 269 + #define P10_SIER2_FINISH_CYC(sier2) (((sier2) >> (63 - 37)) & 0x7fful) 270 + #define P10_SIER2_DISPATCH_CYC(sier2) (((sier2) >> (63 - 13)) & 0x7fful) 271 + 269 272 #define P(a, b) PERF_MEM_S(a, b) 270 273 #define PH(a, b) (P(LVL, HIT) | P(a, b)) 271 274 #define PM(a, b) (P(LVL, MISS) | P(a, b)) ··· 283 278 const unsigned int ev_alt[][MAX_ALT]); 284 279 void isa207_get_mem_data_src(union perf_mem_data_src *dsrc, u32 flags, 285 280 struct pt_regs *regs); 286 - void isa207_get_mem_weight(u64 *weight); 281 + void isa207_get_mem_weight(u64 *weight, u64 type); 282 + 283 + int isa3XX_check_attr_config(struct perf_event *ev); 287 284 288 285 #endif

+2 -2

arch/powerpc/perf/power10-events-list.h

··· 75 75 * thresh end (TE) 76 76 */ 77 77 78 - EVENT(MEM_LOADS, 0x34340401e0); 79 - EVENT(MEM_STORES, 0x343c0401e0); 78 + EVENT(MEM_LOADS, 0x35340401e0); 79 + EVENT(MEM_STORES, 0x353c0401e0);

+13

arch/powerpc/perf/power10-pmu.c

··· 106 106 return num_alt; 107 107 } 108 108 109 + static int power10_check_attr_config(struct perf_event *ev) 110 + { 111 + u64 val; 112 + u64 event = ev->attr.config; 113 + 114 + val = (event >> EVENT_SAMPLE_SHIFT) & EVENT_SAMPLE_MASK; 115 + if (val == 0x10 || isa3XX_check_attr_config(ev)) 116 + return -EINVAL; 117 + 118 + return 0; 119 + } 120 + 109 121 GENERIC_EVENT_ATTR(cpu-cycles, PM_RUN_CYC); 110 122 GENERIC_EVENT_ATTR(instructions, PM_RUN_INST_CMPL); 111 123 GENERIC_EVENT_ATTR(branch-instructions, PM_BR_CMPL); ··· 571 559 .attr_groups = power10_pmu_attr_groups, 572 560 .bhrb_nr = 32, 573 561 .capabilities = PERF_PMU_CAP_EXTENDED_REGS, 562 + .check_attr_config = power10_check_attr_config, 574 563 }; 575 564 576 565 int init_power10_pmu(void)

+13

arch/powerpc/perf/power9-pmu.c

··· 151 151 return num_alt; 152 152 } 153 153 154 + static int power9_check_attr_config(struct perf_event *ev) 155 + { 156 + u64 val; 157 + u64 event = ev->attr.config; 158 + 159 + val = (event >> EVENT_SAMPLE_SHIFT) & EVENT_SAMPLE_MASK; 160 + if (val == 0xC || isa3XX_check_attr_config(ev)) 161 + return -EINVAL; 162 + 163 + return 0; 164 + } 165 + 154 166 GENERIC_EVENT_ATTR(cpu-cycles, PM_CYC); 155 167 GENERIC_EVENT_ATTR(stalled-cycles-frontend, PM_ICT_NOSLOT_CYC); 156 168 GENERIC_EVENT_ATTR(stalled-cycles-backend, PM_CMPLU_STALL); ··· 449 437 .attr_groups = power9_pmu_attr_groups, 450 438 .bhrb_nr = 32, 451 439 .capabilities = PERF_PMU_CAP_EXTENDED_REGS, 440 + .check_attr_config = power9_check_attr_config, 452 441 }; 453 442 454 443 int init_power9_pmu(void)

+1 -1

arch/powerpc/platforms/44x/Kconfig

··· 5 5 select MPIC 6 6 help 7 7 This option enables support for the 47x family of processors and is 8 - not currently compatible with other 44x or 46x varients 8 + not currently compatible with other 44x or 46x variants 9 9 10 10 config BAMBOO 11 11 bool "Bamboo"

+1 -1

arch/powerpc/platforms/52xx/lite5200_sleep.S

··· 181 181 udelay: /* r11 - tb_ticks_per_usec, r12 - usecs, overwrites r13 */ 182 182 mullw r12, r12, r11 183 183 mftb r13 /* start */ 184 - addi r12, r13, r12 /* end */ 184 + add r12, r13, r12 /* end */ 185 185 1: 186 186 mftb r13 /* current */ 187 187 cmp cr0, r13, r12

+3 -2

arch/powerpc/platforms/Kconfig.cputype

··· 101 101 select ARCH_SUPPORTS_NUMA_BALANCING 102 102 select IRQ_WORK 103 103 select PPC_MM_SLICES 104 + select PPC_HAVE_KUEP 105 + select PPC_HAVE_KUAP 104 106 105 107 config PPC_BOOK3E_64 106 108 bool "Embedded processors" ··· 308 306 config ALTIVEC 309 307 bool "AltiVec Support" 310 308 depends on PPC_BOOK3S_32 || PPC_BOOK3S_64 || (PPC_E500MC && PPC64) 309 + select PPC_FPU 311 310 help 312 311 This option enables kernel support for the Altivec extensions to the 313 312 PowerPC processor. The kernel currently supports saving and restoring ··· 366 363 bool "Radix MMU Support" 367 364 depends on PPC_BOOK3S_64 368 365 select ARCH_HAS_GIGANTIC_PAGE 369 - select PPC_HAVE_KUEP 370 - select PPC_HAVE_KUAP 371 366 default y 372 367 help 373 368 Enable support for the Power ISA 3.0 Radix style MMU. Currently this

+2 -1

arch/powerpc/platforms/cell/iommu.c

··· 486 486 window->table.it_size = size >> window->table.it_page_shift; 487 487 window->table.it_ops = &cell_iommu_ops; 488 488 489 - iommu_init_table(&window->table, iommu->nid, 0, 0); 489 + if (!iommu_init_table(&window->table, iommu->nid, 0, 0)) 490 + panic("Failed to initialize iommu table"); 490 491 491 492 pr_debug("\tioid %d\n", window->ioid); 492 493 pr_debug("\tblocksize %ld\n", window->table.it_blocksize);

+1 -1

arch/powerpc/platforms/cell/spu_callbacks.c

··· 35 35 */ 36 36 37 37 static void *spu_syscall_table[] = { 38 + #define __SYSCALL_WITH_COMPAT(nr, entry, compat) __SYSCALL(nr, entry) 38 39 #define __SYSCALL(nr, entry) [nr] = entry, 39 40 #include <asm/syscall_table_spu.h> 40 - #undef __SYSCALL 41 41 }; 42 42 43 43 long spu_sys_callback(struct spu_syscall_block *s)

+1 -2

arch/powerpc/platforms/chrp/pci.c

··· 131 131 132 132 volatile struct Hydra __iomem *Hydra = NULL; 133 133 134 - int __init 135 - hydra_init(void) 134 + static int __init hydra_init(void) 136 135 { 137 136 struct device_node *np; 138 137 struct resource r;

-5

arch/powerpc/platforms/embedded6xx/Kconfig

··· 71 71 bool 72 72 select PPC_INDIRECT_PCI 73 73 74 - config MV64X60 75 - bool 76 - select PPC_INDIRECT_PCI 77 - select CHECK_CACHE_COHERENCY 78 - 79 74 config GAMECUBE_COMMON 80 75 bool 81 76

+1 -1

arch/powerpc/platforms/maple/pci.c

··· 34 34 35 35 static int __init fixup_one_level_bus_range(struct device_node *node, int higher) 36 36 { 37 - for (; node != 0;node = node->sibling) { 37 + for (; node; node = node->sibling) { 38 38 const int *bus_range; 39 39 const unsigned int *class_code; 40 40 int len;

+3 -1

arch/powerpc/platforms/pasemi/iommu.c

··· 146 146 */ 147 147 iommu_table_iobmap.it_blocksize = 4; 148 148 iommu_table_iobmap.it_ops = &iommu_table_iobmap_ops; 149 - iommu_init_table(&iommu_table_iobmap, 0, 0, 0); 149 + if (!iommu_init_table(&iommu_table_iobmap, 0, 0, 0)) 150 + panic("Failed to initialize iommu table"); 151 + 150 152 pr_debug(" <- %s\n", __func__); 151 153 } 152 154

+17 -1

arch/powerpc/platforms/powernv/memtrace.c

··· 46 46 return simple_read_from_buffer(ubuf, count, ppos, ent->mem, ent->size); 47 47 } 48 48 49 + static int memtrace_mmap(struct file *filp, struct vm_area_struct *vma) 50 + { 51 + struct memtrace_entry *ent = filp->private_data; 52 + 53 + if (ent->size < vma->vm_end - vma->vm_start) 54 + return -EINVAL; 55 + 56 + if (vma->vm_pgoff << PAGE_SHIFT >= ent->size) 57 + return -EINVAL; 58 + 59 + vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); 60 + return remap_pfn_range(vma, vma->vm_start, PHYS_PFN(ent->start) + vma->vm_pgoff, 61 + vma->vm_end - vma->vm_start, vma->vm_page_prot); 62 + } 63 + 49 64 static const struct file_operations memtrace_fops = { 50 65 .llseek = default_llseek, 51 66 .read = memtrace_read, 52 67 .open = simple_open, 68 + .mmap = memtrace_mmap, 53 69 }; 54 70 55 71 #define FLUSH_CHUNK_SIZE SZ_1G ··· 203 187 dir = debugfs_create_dir(ent->name, memtrace_debugfs_dir); 204 188 205 189 ent->dir = dir; 206 - debugfs_create_file("trace", 0400, dir, ent, &memtrace_fops); 190 + debugfs_create_file_unsafe("trace", 0600, dir, ent, &memtrace_fops); 207 191 debugfs_create_x64("start", 0400, dir, &ent->start); 208 192 debugfs_create_x64("size", 0400, dir, &ent->size); 209 193 }

+1 -1

arch/powerpc/platforms/powernv/opal-core.c

··· 71 71 static struct opalcore_config *oc_conf; 72 72 static const struct opal_mpipl_fadump *opalc_metadata; 73 73 static const struct opal_mpipl_fadump *opalc_cpu_metadata; 74 - struct kobject *mpipl_kobj; 74 + static struct kobject *mpipl_kobj; 75 75 76 76 /* 77 77 * Set crashing CPU's signal to SIGUSR1. if the kernel is triggered

+1 -4

arch/powerpc/platforms/powernv/opal-prd.c

··· 105 105 { 106 106 size_t addr, size; 107 107 pgprot_t page_prot; 108 - int rc; 109 108 110 109 pr_devel("opal_prd_mmap(0x%016lx, 0x%016lx, 0x%lx, 0x%lx)\n", 111 110 vma->vm_start, vma->vm_end, vma->vm_pgoff, ··· 120 121 page_prot = phys_mem_access_prot(file, vma->vm_pgoff, 121 122 size, vma->vm_page_prot); 122 123 123 - rc = remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff, size, 124 + return remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff, size, 124 125 page_prot); 125 - 126 - return rc; 127 126 } 128 127 129 128 static bool opal_msg_queue_empty(void)

+8 -7

arch/powerpc/platforms/powernv/pci-ioda.c

··· 1762 1762 tbl->it_ops = &pnv_ioda1_iommu_ops; 1763 1763 pe->table_group.tce32_start = tbl->it_offset << tbl->it_page_shift; 1764 1764 pe->table_group.tce32_size = tbl->it_size << tbl->it_page_shift; 1765 - iommu_init_table(tbl, phb->hose->node, 0, 0); 1765 + if (!iommu_init_table(tbl, phb->hose->node, 0, 0)) 1766 + panic("Failed to initialize iommu table"); 1766 1767 1767 1768 pe->dma_setup_done = true; 1768 1769 return; ··· 1931 1930 res_start = pe->phb->ioda.m32_pci_base >> tbl->it_page_shift; 1932 1931 res_end = min(window_size, SZ_4G) >> tbl->it_page_shift; 1933 1932 } 1934 - iommu_init_table(tbl, pe->phb->hose->node, res_start, res_end); 1935 1933 1936 - rc = pnv_pci_ioda2_set_window(&pe->table_group, 0, tbl); 1934 + if (iommu_init_table(tbl, pe->phb->hose->node, res_start, res_end)) 1935 + rc = pnv_pci_ioda2_set_window(&pe->table_group, 0, tbl); 1936 + else 1937 + rc = -ENOMEM; 1937 1938 if (rc) { 1938 - pe_err(pe, "Failed to configure 32-bit TCE table, err %ld\n", 1939 - rc); 1939 + pe_err(pe, "Failed to configure 32-bit TCE table, err %ld\n", rc); 1940 1940 iommu_tce_table_put(tbl); 1941 - return rc; 1941 + tbl = NULL; /* This clears iommu_table_base below */ 1942 1942 } 1943 - 1944 1943 if (!pnv_iommu_bypass_disabled) 1945 1944 pnv_pci_ioda2_set_bypass(pe, true); 1946 1945

+1 -1

arch/powerpc/platforms/powernv/setup.c

··· 157 157 for_each_node_by_type(dn, "cpu") { 158 158 if (of_property_match_string(dn, "status", "bad") >= 0) 159 159 bad_count++; 160 - }; 160 + } 161 161 162 162 if (bad_count) { 163 163 printk(" _ _______________\n");

+14

arch/powerpc/platforms/pseries/dlpar.c

··· 329 329 return 0; 330 330 } 331 331 332 + int dlpar_unisolate_drc(u32 drc_index) 333 + { 334 + int dr_status, rc; 335 + 336 + rc = rtas_call(rtas_token("get-sensor-state"), 2, 2, &dr_status, 337 + DR_ENTITY_SENSE, drc_index); 338 + if (rc || dr_status != DR_ENTITY_PRESENT) 339 + return -1; 340 + 341 + rtas_set_indicator(ISOLATION_STATE, drc_index, UNISOLATE); 342 + 343 + return 0; 344 + } 345 + 332 346 int handle_dlpar_errorlog(struct pseries_hp_errorlog *hp_elog) 333 347 { 334 348 int rc;

+23 -4

arch/powerpc/platforms/pseries/hotplug-cpu.c

··· 47 47 48 48 BUG_ON(rtas_stop_self_token == RTAS_UNKNOWN_SERVICE); 49 49 50 - printk("cpu %u (hwid %u) Ready to die...\n", 51 - smp_processor_id(), hard_smp_processor_id()); 52 - 53 50 rtas_call_unlocked(&args, rtas_stop_self_token, 0, 1, NULL); 54 51 55 52 panic("Alas, I survived.\n"); ··· 268 271 if (!cpu_online(cpu)) 269 272 break; 270 273 274 + /* 275 + * device_offline() will return -EBUSY (via cpu_down()) if there 276 + * is only one CPU left. Check it here to fail earlier and with a 277 + * more informative error message, while also retaining the 278 + * cpu_add_remove_lock to be sure that no CPUs are being 279 + * online/offlined during this check. 280 + */ 281 + if (num_online_cpus() == 1) { 282 + pr_warn("Unable to remove last online CPU %pOFn\n", dn); 283 + rc = -EBUSY; 284 + goto out_unlock; 285 + } 286 + 271 287 cpu_maps_update_done(); 272 288 rc = device_offline(get_cpu_device(cpu)); 273 289 if (rc) ··· 293 283 thread); 294 284 } 295 285 } 286 + out_unlock: 296 287 cpu_maps_update_done(); 297 288 298 289 out: ··· 813 802 case PSERIES_HP_ELOG_ACTION_REMOVE: 814 803 if (hp_elog->id_type == PSERIES_HP_ELOG_ID_DRC_COUNT) 815 804 rc = dlpar_cpu_remove_by_count(count); 816 - else if (hp_elog->id_type == PSERIES_HP_ELOG_ID_DRC_INDEX) 805 + else if (hp_elog->id_type == PSERIES_HP_ELOG_ID_DRC_INDEX) { 817 806 rc = dlpar_cpu_remove_by_index(drc_index); 807 + /* 808 + * Setting the isolation state of an UNISOLATED/CONFIGURED 809 + * device to UNISOLATE is a no-op, but the hypervisor can 810 + * use it as a hint that the CPU removal failed. 811 + */ 812 + if (rc) 813 + dlpar_unisolate_drc(drc_index); 814 + } 818 815 else 819 816 rc = -EINVAL; 820 817 break;

+1 -1

arch/powerpc/platforms/pseries/hvCall_inst.c

··· 26 26 }; 27 27 #define HCALL_STAT_ARRAY_SIZE ((MAX_HCALL_OPCODE >> 2) + 1) 28 28 29 - DEFINE_PER_CPU(struct hcall_stats[HCALL_STAT_ARRAY_SIZE], hcall_stats); 29 + static DEFINE_PER_CPU(struct hcall_stats[HCALL_STAT_ARRAY_SIZE], hcall_stats); 30 30 31 31 /* 32 32 * Routines for displaying the statistics in debugfs

+38 -11

arch/powerpc/platforms/pseries/iommu.c

··· 638 638 639 639 iommu_table_setparms(pci->phb, dn, tbl); 640 640 tbl->it_ops = &iommu_table_pseries_ops; 641 - iommu_init_table(tbl, pci->phb->node, 0, 0); 641 + if (!iommu_init_table(tbl, pci->phb->node, 0, 0)) 642 + panic("Failed to initialize iommu table"); 642 643 643 644 /* Divide the rest (1.75GB) among the children */ 644 645 pci->phb->dma_window_size = 0x80000000ul; ··· 721 720 iommu_table_setparms_lpar(ppci->phb, pdn, tbl, 722 721 ppci->table_group, dma_window); 723 722 tbl->it_ops = &iommu_table_lpar_multi_ops; 724 - iommu_init_table(tbl, ppci->phb->node, 0, 0); 723 + if (!iommu_init_table(tbl, ppci->phb->node, 0, 0)) 724 + panic("Failed to initialize iommu table"); 725 725 iommu_register_group(ppci->table_group, 726 726 pci_domain_nr(bus), 0); 727 727 pr_debug(" created table: %p\n", ppci->table_group); ··· 751 749 tbl = PCI_DN(dn)->table_group->tables[0]; 752 750 iommu_table_setparms(phb, dn, tbl); 753 751 tbl->it_ops = &iommu_table_pseries_ops; 754 - iommu_init_table(tbl, phb->node, 0, 0); 752 + if (!iommu_init_table(tbl, phb->node, 0, 0)) 753 + panic("Failed to initialize iommu table"); 754 + 755 755 set_iommu_table_base(&dev->dev, tbl); 756 756 return; 757 757 } ··· 1103 1099 ret); 1104 1100 } 1105 1101 1102 + /* Return largest page shift based on "IO Page Sizes" output of ibm,query-pe-dma-window. */ 1103 + static int iommu_get_page_shift(u32 query_page_size) 1104 + { 1105 + /* Supported IO page-sizes according to LoPAR */ 1106 + const int shift[] = { 1107 + __builtin_ctzll(SZ_4K), __builtin_ctzll(SZ_64K), __builtin_ctzll(SZ_16M), 1108 + __builtin_ctzll(SZ_32M), __builtin_ctzll(SZ_64M), __builtin_ctzll(SZ_128M), 1109 + __builtin_ctzll(SZ_256M), __builtin_ctzll(SZ_16G) 1110 + }; 1111 + 1112 + int i = ARRAY_SIZE(shift) - 1; 1113 + 1114 + /* 1115 + * On LoPAR, ibm,query-pe-dma-window outputs "IO Page Sizes" using a bit field: 1116 + * - bit 31 means 4k pages are supported, 1117 + * - bit 30 means 64k pages are supported, and so on. 1118 + * Larger pagesizes map more memory with the same amount of TCEs, so start probing them. 1119 + */ 1120 + for (; i >= 0 ; i--) { 1121 + if (query_page_size & (1 << i)) 1122 + return shift[i]; 1123 + } 1124 + 1125 + /* No valid page size found. */ 1126 + return 0; 1127 + } 1128 + 1106 1129 /* 1107 1130 * If the PE supports dynamic dma windows, and there is space for a table 1108 1131 * that can map all pages in a linear offset, then setup such a table, ··· 1237 1206 goto out_failed; 1238 1207 } 1239 1208 } 1240 - if (query.page_size & 4) { 1241 - page_shift = 24; /* 16MB */ 1242 - } else if (query.page_size & 2) { 1243 - page_shift = 16; /* 64kB */ 1244 - } else if (query.page_size & 1) { 1245 - page_shift = 12; /* 4kB */ 1246 - } else { 1209 + 1210 + page_shift = iommu_get_page_shift(query.page_size); 1211 + if (!page_shift) { 1247 1212 dev_dbg(&dev->dev, "no supported direct page size in mask %x", 1248 1213 query.page_size); 1249 1214 goto out_failed; ··· 1256 1229 if (pmem_present) { 1257 1230 if (query.largest_available_block >= 1258 1231 (1ULL << (MAX_PHYSMEM_BITS - page_shift))) 1259 - len = MAX_PHYSMEM_BITS - page_shift; 1232 + len = MAX_PHYSMEM_BITS; 1260 1233 else 1261 1234 dev_info(&dev->dev, "Skipping ibm,pmemory"); 1262 1235 }

+4 -2

arch/powerpc/platforms/pseries/lpar.c

··· 977 977 slot = pSeries_lpar_hpte_find(vpn, psize, ssize); 978 978 BUG_ON(slot == -1); 979 979 980 - flags = newpp & 7; 980 + flags = newpp & (HPTE_R_PP | HPTE_R_N); 981 981 if (mmu_has_feature(MMU_FTR_KERNEL_RO)) 982 982 /* Move pp0 into bit 8 (IBM 55) */ 983 983 flags |= (newpp & HPTE_R_PP0) >> 55; 984 + 985 + flags |= ((newpp & HPTE_R_KEY_HI) >> 48) | (newpp & HPTE_R_KEY_LO); 984 986 985 987 lpar_rc = plpar_pte_protect(flags, slot, 0); 986 988 ··· 1632 1630 } 1633 1631 msleep(delay); 1634 1632 rc = plpar_resize_hpt_prepare(0, shift); 1635 - }; 1633 + } 1636 1634 1637 1635 switch (rc) { 1638 1636 case H_SUCCESS:

+2

arch/powerpc/platforms/pseries/lparcfg.c

··· 537 537 parse_em_data(m); 538 538 maxmem_data(m); 539 539 540 + seq_printf(m, "security_flavor=%u\n", pseries_security_flavor); 541 + 540 542 return 0; 541 543 } 542 544

+48

arch/powerpc/platforms/pseries/papr_scm.c

··· 93 93 uint64_t block_size; 94 94 int metadata_size; 95 95 bool is_volatile; 96 + bool hcall_flush_required; 96 97 97 98 uint64_t bound_addr; 98 99 ··· 117 116 /* length of the stat buffer as expected by phyp */ 118 117 size_t stat_buffer_len; 119 118 }; 119 + 120 + static int papr_scm_pmem_flush(struct nd_region *nd_region, 121 + struct bio *bio __maybe_unused) 122 + { 123 + struct papr_scm_priv *p = nd_region_provider_data(nd_region); 124 + unsigned long ret_buf[PLPAR_HCALL_BUFSIZE], token = 0; 125 + long rc; 126 + 127 + dev_dbg(&p->pdev->dev, "flush drc 0x%x", p->drc_index); 128 + 129 + do { 130 + rc = plpar_hcall(H_SCM_FLUSH, ret_buf, p->drc_index, token); 131 + token = ret_buf[0]; 132 + 133 + /* Check if we are stalled for some time */ 134 + if (H_IS_LONG_BUSY(rc)) { 135 + msleep(get_longbusy_msecs(rc)); 136 + rc = H_BUSY; 137 + } else if (rc == H_BUSY) { 138 + cond_resched(); 139 + } 140 + } while (rc == H_BUSY); 141 + 142 + if (rc) { 143 + dev_err(&p->pdev->dev, "flush error: %ld", rc); 144 + rc = -EIO; 145 + } else { 146 + dev_dbg(&p->pdev->dev, "flush drc 0x%x complete", p->drc_index); 147 + } 148 + 149 + return rc; 150 + } 120 151 121 152 static LIST_HEAD(papr_nd_regions); 122 153 static DEFINE_MUTEX(papr_ndr_lock); ··· 947 914 dimm_flags = 0; 948 915 set_bit(NDD_LABELING, &dimm_flags); 949 916 917 + /* 918 + * Check if the nvdimm is unarmed. No locking needed as we are still 919 + * initializing. Ignore error encountered if any. 920 + */ 921 + __drc_pmem_query_health(p); 922 + 923 + if (p->health_bitmap & PAPR_PMEM_UNARMED_MASK) 924 + set_bit(NDD_UNARMED, &dimm_flags); 925 + 950 926 p->nvdimm = nvdimm_create(p->bus, p, papr_nd_attr_groups, 951 927 dimm_flags, PAPR_SCM_DIMM_CMD_MASK, 0, NULL); 952 928 if (!p->nvdimm) { ··· 984 942 ndr_desc.mapping = &mapping; 985 943 ndr_desc.num_mappings = 1; 986 944 ndr_desc.nd_set = &p->nd_set; 945 + 946 + if (p->hcall_flush_required) { 947 + set_bit(ND_REGION_ASYNC, &ndr_desc.flags); 948 + ndr_desc.flush = papr_scm_pmem_flush; 949 + } 987 950 988 951 if (p->is_volatile) 989 952 p->region = nvdimm_volatile_region_create(p->bus, &ndr_desc); ··· 1135 1088 p->block_size = block_size; 1136 1089 p->blocks = blocks; 1137 1090 p->is_volatile = !of_property_read_bool(dn, "ibm,cache-flush-required"); 1091 + p->hcall_flush_required = of_property_read_bool(dn, "ibm,hcall-flush-required"); 1138 1092 1139 1093 /* We just need to ensure that set cookies are unique across */ 1140 1094 uuid_parse(uuid_str, (uuid_t *) uuid);

+3 -1

arch/powerpc/platforms/pseries/pci_dlpar.c

··· 50 50 int remove_phb_dynamic(struct pci_controller *phb) 51 51 { 52 52 struct pci_bus *b = phb->bus; 53 + struct pci_host_bridge *host_bridge = to_pci_host_bridge(b->bridge); 53 54 struct resource *res; 54 55 int rc, i; 55 56 ··· 77 76 /* Remove the PCI bus and unregister the bridge device from sysfs */ 78 77 phb->bus = NULL; 79 78 pci_remove_bus(b); 80 - device_unregister(b->bridge); 79 + host_bridge->bus = NULL; 80 + device_unregister(&host_bridge->dev); 81 81 82 82 /* Now release the IO resource */ 83 83 if (res->flags & IORESOURCE_IO)

+1 -1

arch/powerpc/platforms/pseries/pmem.c

··· 139 139 return rc; 140 140 } 141 141 142 - const struct of_device_id drc_pmem_match[] = { 142 + static const struct of_device_id drc_pmem_match[] = { 143 143 { .type = "ibm,persistent-memory", }, 144 144 {} 145 145 };

+2 -3

arch/powerpc/platforms/pseries/pseries.h

··· 43 43 /* Poweron flag used for enabling auto ups restart */ 44 44 extern unsigned long rtas_poweron_auto; 45 45 46 - /* Provided by HVC VIO */ 47 - extern void hvc_vio_init_early(void); 48 - 49 46 /* Dynamic logical Partitioning/Mobility */ 50 47 extern void dlpar_free_cc_nodes(struct device_node *); 51 48 extern void dlpar_free_cc_property(struct property *); ··· 52 55 extern int dlpar_detach_node(struct device_node *); 53 56 extern int dlpar_acquire_drc(u32 drc_index); 54 57 extern int dlpar_release_drc(u32 drc_index); 58 + extern int dlpar_unisolate_drc(u32 drc_index); 55 59 56 60 void queue_hotplug_event(struct pseries_hp_errorlog *hp_errlog); 57 61 int handle_dlpar_errorlog(struct pseries_hp_errorlog *hp_errlog); ··· 109 111 110 112 int dlpar_workqueue_init(void); 111 113 114 + extern u32 pseries_security_flavor; 112 115 void pseries_setup_security_mitigations(void); 113 116 void pseries_lpar_read_hblkrm_characteristics(void); 114 117

+1 -1

arch/powerpc/platforms/pseries/ras.c

··· 699 699 mce_err.error_type = MCE_ERROR_TYPE_DCACHE; 700 700 break; 701 701 case MC_ERROR_TYPE_I_CACHE: 702 - mce_err.error_type = MCE_ERROR_TYPE_DCACHE; 702 + mce_err.error_type = MCE_ERROR_TYPE_ICACHE; 703 703 break; 704 704 case MC_ERROR_TYPE_UNKNOWN: 705 705 default:

+1 -1

arch/powerpc/platforms/pseries/rtas-fadump.c

··· 247 247 return i; 248 248 } 249 249 250 - void rtas_fadump_set_regval(struct pt_regs *regs, u64 reg_id, u64 reg_val) 250 + static void rtas_fadump_set_regval(struct pt_regs *regs, u64 reg_id, u64 reg_val) 251 251 { 252 252 int i; 253 253

+8

arch/powerpc/platforms/pseries/setup.c

··· 71 71 #include <asm/swiotlb.h> 72 72 #include <asm/svm.h> 73 73 #include <asm/dtl.h> 74 + #include <asm/hvconsole.h> 74 75 75 76 #include "pseries.h" 76 77 #include "../../../../drivers/pci/pci.h" ··· 86 85 87 86 int fwnmi_active; /* TRUE if an FWNMI handler is present */ 88 87 int ibm_nmi_interlock_token; 88 + u32 pseries_security_flavor; 89 89 90 90 static void pSeries_show_cpuinfo(struct seq_file *m) 91 91 { ··· 536 534 /* 537 535 * The features below are enabled by default, so we instead look to see 538 536 * if firmware has *disabled* them, and clear them if so. 537 + * H_CPU_BEHAV_FAVOUR_SECURITY_H could be set only if 538 + * H_CPU_BEHAV_FAVOUR_SECURITY is. 539 539 */ 540 540 if (!(result->behaviour & H_CPU_BEHAV_FAVOUR_SECURITY)) 541 541 security_ftr_clear(SEC_FTR_FAVOUR_SECURITY); 542 + else if (result->behaviour & H_CPU_BEHAV_FAVOUR_SECURITY_H) 543 + pseries_security_flavor = 1; 544 + else 545 + pseries_security_flavor = 2; 542 546 543 547 if (!(result->behaviour & H_CPU_BEHAV_L1D_FLUSH_PR)) 544 548 security_ftr_clear(SEC_FTR_L1D_FLUSH_PR);

+20

arch/powerpc/platforms/pseries/vio.c

··· 22 22 #include <linux/mm.h> 23 23 #include <linux/dma-map-ops.h> 24 24 #include <linux/kobject.h> 25 + #include <linux/kexec.h> 25 26 26 27 #include <asm/iommu.h> 27 28 #include <asm/dma.h> ··· 1279 1278 return 0; 1280 1279 } 1281 1280 1281 + static void vio_bus_shutdown(struct device *dev) 1282 + { 1283 + struct vio_dev *viodev = to_vio_dev(dev); 1284 + struct vio_driver *viodrv; 1285 + 1286 + if (dev->driver) { 1287 + viodrv = to_vio_driver(dev->driver); 1288 + if (viodrv->shutdown) 1289 + viodrv->shutdown(viodev); 1290 + else if (kexec_in_progress) 1291 + vio_bus_remove(dev); 1292 + } 1293 + } 1294 + 1282 1295 /** 1283 1296 * vio_register_driver: - Register a new vio driver 1284 1297 * @viodrv: The vio_driver structure to be registered. ··· 1300 1285 int __vio_register_driver(struct vio_driver *viodrv, struct module *owner, 1301 1286 const char *mod_name) 1302 1287 { 1288 + // vio_bus_type is only initialised for pseries 1289 + if (!machine_is(pseries)) 1290 + return -ENODEV; 1291 + 1303 1292 pr_debug("%s: driver %s registering\n", __func__, viodrv->name); 1304 1293 1305 1294 /* fill in 'struct driver' fields */ ··· 1632 1613 .match = vio_bus_match, 1633 1614 .probe = vio_bus_probe, 1634 1615 .remove = vio_bus_remove, 1616 + .shutdown = vio_bus_shutdown, 1635 1617 }; 1636 1618 1637 1619 /**

-1

arch/powerpc/purgatory/trampoline_64.S

··· 12 12 #include <asm/asm-compat.h> 13 13 #include <asm/crashdump-ppc64.h> 14 14 15 - .machine ppc64 16 15 .balign 256 17 16 .globl purgatory_start 18 17 purgatory_start:

+2 -1

arch/powerpc/sysdev/dart_iommu.c

··· 344 344 iommu_table_dart.it_index = 0; 345 345 iommu_table_dart.it_blocksize = 1; 346 346 iommu_table_dart.it_ops = &iommu_dart_ops; 347 - iommu_init_table(&iommu_table_dart, -1, 0, 0); 347 + if (!iommu_init_table(&iommu_table_dart, -1, 0, 0)) 348 + panic("Failed to initialize iommu table"); 348 349 349 350 /* Reserve the last page of the DART to avoid possible prefetch 350 351 * past the DART mapped area

+1 -1

arch/powerpc/sysdev/fsl_pci.c

··· 455 455 } 456 456 } 457 457 458 - static void __init setup_pci_cmd(struct pci_controller *hose) 458 + static void setup_pci_cmd(struct pci_controller *hose) 459 459 { 460 460 u16 cmd; 461 461 int cap_x;

+138 -75

arch/powerpc/sysdev/xive/common.c

··· 63 63 static struct irq_domain *xive_irq_domain; 64 64 65 65 #ifdef CONFIG_SMP 66 - /* The IPIs all use the same logical irq number */ 67 - static u32 xive_ipi_irq; 66 + /* The IPIs use the same logical irq number when on the same chip */ 67 + static struct xive_ipi_desc { 68 + unsigned int irq; 69 + char name[16]; 70 + } *xive_ipis; 71 + 72 + /* 73 + * Use early_cpu_to_node() for hot-plugged CPUs 74 + */ 75 + static unsigned int xive_ipi_cpu_to_irq(unsigned int cpu) 76 + { 77 + return xive_ipis[early_cpu_to_node(cpu)].irq; 78 + } 68 79 #endif 69 80 70 81 /* Xive state for each CPU */ ··· 264 253 xmon_printf("\n"); 265 254 } 266 255 256 + static struct irq_data *xive_get_irq_data(u32 hw_irq) 257 + { 258 + unsigned int irq = irq_find_mapping(xive_irq_domain, hw_irq); 259 + 260 + return irq ? irq_get_irq_data(irq) : NULL; 261 + } 262 + 267 263 int xmon_xive_get_irq_config(u32 hw_irq, struct irq_data *d) 268 264 { 269 - struct irq_chip *chip = irq_data_get_irq_chip(d); 270 265 int rc; 271 266 u32 target; 272 267 u8 prio; 273 268 u32 lirq; 274 - 275 - if (!is_xive_irq(chip)) 276 - return -EINVAL; 277 269 278 270 rc = xive_ops->get_irq_config(hw_irq, &target, &prio, &lirq); 279 271 if (rc) { ··· 286 272 287 273 xmon_printf("IRQ 0x%08x : target=0x%x prio=%02x lirq=0x%x ", 288 274 hw_irq, target, prio, lirq); 275 + 276 + if (!d) 277 + d = xive_get_irq_data(hw_irq); 289 278 290 279 if (d) { 291 280 struct xive_irq_data *xd = irq_data_get_irq_handler_data(d); ··· 304 287 305 288 xmon_printf("\n"); 306 289 return 0; 290 + } 291 + 292 + void xmon_xive_get_irq_all(void) 293 + { 294 + unsigned int i; 295 + struct irq_desc *desc; 296 + 297 + for_each_irq_desc(i, desc) { 298 + struct irq_data *d = irq_desc_get_irq_data(desc); 299 + unsigned int hwirq = (unsigned int)irqd_to_hwirq(d); 300 + 301 + if (d->domain == xive_irq_domain) 302 + xmon_xive_get_irq_config(hwirq, d); 303 + } 307 304 } 308 305 309 306 #endif /* CONFIG_XMON */ ··· 1098 1067 .irq_unmask = xive_ipi_do_nothing, 1099 1068 }; 1100 1069 1101 - static void __init xive_request_ipi(void) 1070 + /* 1071 + * IPIs are marked per-cpu. We use separate HW interrupts under the 1072 + * hood but associated with the same "linux" interrupt 1073 + */ 1074 + struct xive_ipi_alloc_info { 1075 + irq_hw_number_t hwirq; 1076 + }; 1077 + 1078 + static int xive_ipi_irq_domain_alloc(struct irq_domain *domain, unsigned int virq, 1079 + unsigned int nr_irqs, void *arg) 1102 1080 { 1103 - unsigned int virq; 1081 + struct xive_ipi_alloc_info *info = arg; 1082 + int i; 1104 1083 1105 - /* 1106 - * Initialization failed, move on, we might manage to 1107 - * reach the point where we display our errors before 1108 - * the system falls appart 1109 - */ 1110 - if (!xive_irq_domain) 1111 - return; 1084 + for (i = 0; i < nr_irqs; i++) { 1085 + irq_domain_set_info(domain, virq + i, info->hwirq + i, &xive_ipi_chip, 1086 + domain->host_data, handle_percpu_irq, 1087 + NULL, NULL); 1088 + } 1089 + return 0; 1090 + } 1112 1091 1113 - /* Initialize it */ 1114 - virq = irq_create_mapping(xive_irq_domain, XIVE_IPI_HW_IRQ); 1115 - xive_ipi_irq = virq; 1092 + static const struct irq_domain_ops xive_ipi_irq_domain_ops = { 1093 + .alloc = xive_ipi_irq_domain_alloc, 1094 + }; 1116 1095 1117 - WARN_ON(request_irq(virq, xive_muxed_ipi_action, 1118 - IRQF_PERCPU | IRQF_NO_THREAD, "IPI", NULL)); 1096 + static int __init xive_request_ipi(void) 1097 + { 1098 + struct fwnode_handle *fwnode; 1099 + struct irq_domain *ipi_domain; 1100 + unsigned int node; 1101 + int ret = -ENOMEM; 1102 + 1103 + fwnode = irq_domain_alloc_named_fwnode("XIVE-IPI"); 1104 + if (!fwnode) 1105 + goto out; 1106 + 1107 + ipi_domain = irq_domain_create_linear(fwnode, nr_node_ids, 1108 + &xive_ipi_irq_domain_ops, NULL); 1109 + if (!ipi_domain) 1110 + goto out_free_fwnode; 1111 + 1112 + xive_ipis = kcalloc(nr_node_ids, sizeof(*xive_ipis), GFP_KERNEL | __GFP_NOFAIL); 1113 + if (!xive_ipis) 1114 + goto out_free_domain; 1115 + 1116 + for_each_node(node) { 1117 + struct xive_ipi_desc *xid = &xive_ipis[node]; 1118 + struct xive_ipi_alloc_info info = { node }; 1119 + 1120 + /* Skip nodes without CPUs */ 1121 + if (cpumask_empty(cpumask_of_node(node))) 1122 + continue; 1123 + 1124 + /* 1125 + * Map one IPI interrupt per node for all cpus of that node. 1126 + * Since the HW interrupt number doesn't have any meaning, 1127 + * simply use the node number. 1128 + */ 1129 + xid->irq = irq_domain_alloc_irqs(ipi_domain, 1, node, &info); 1130 + if (xid->irq < 0) { 1131 + ret = xid->irq; 1132 + goto out_free_xive_ipis; 1133 + } 1134 + 1135 + snprintf(xid->name, sizeof(xid->name), "IPI-%d", node); 1136 + 1137 + ret = request_irq(xid->irq, xive_muxed_ipi_action, 1138 + IRQF_PERCPU | IRQF_NO_THREAD, xid->name, NULL); 1139 + 1140 + WARN(ret < 0, "Failed to request IPI %d: %d\n", xid->irq, ret); 1141 + } 1142 + 1143 + return ret; 1144 + 1145 + out_free_xive_ipis: 1146 + kfree(xive_ipis); 1147 + out_free_domain: 1148 + irq_domain_remove(ipi_domain); 1149 + out_free_fwnode: 1150 + irq_domain_free_fwnode(fwnode); 1151 + out: 1152 + return ret; 1119 1153 } 1120 1154 1121 1155 static int xive_setup_cpu_ipi(unsigned int cpu) 1122 1156 { 1157 + unsigned int xive_ipi_irq = xive_ipi_cpu_to_irq(cpu); 1123 1158 struct xive_cpu *xc; 1124 1159 int rc; 1125 1160 ··· 1228 1131 1229 1132 static void xive_cleanup_cpu_ipi(unsigned int cpu, struct xive_cpu *xc) 1230 1133 { 1134 + unsigned int xive_ipi_irq = xive_ipi_cpu_to_irq(cpu); 1135 + 1231 1136 /* Disable the IPI and free the IRQ data */ 1232 1137 1233 1138 /* Already cleaned up ? */ ··· 1277 1178 */ 1278 1179 irq_clear_status_flags(virq, IRQ_LEVEL); 1279 1180 1280 - #ifdef CONFIG_SMP 1281 - /* IPIs are special and come up with HW number 0 */ 1282 - if (hw == XIVE_IPI_HW_IRQ) { 1283 - /* 1284 - * IPIs are marked per-cpu. We use separate HW interrupts under 1285 - * the hood but associated with the same "linux" interrupt 1286 - */ 1287 - irq_set_chip_and_handler(virq, &xive_ipi_chip, 1288 - handle_percpu_irq); 1289 - return 0; 1290 - } 1291 - #endif 1292 - 1293 1181 rc = xive_irq_alloc_data(virq, hw); 1294 1182 if (rc) 1295 1183 return rc; ··· 1288 1202 1289 1203 static void xive_irq_domain_unmap(struct irq_domain *d, unsigned int virq) 1290 1204 { 1291 - struct irq_data *data = irq_get_irq_data(virq); 1292 - unsigned int hw_irq; 1293 - 1294 - /* XXX Assign BAD number */ 1295 - if (!data) 1296 - return; 1297 - hw_irq = (unsigned int)irqd_to_hwirq(data); 1298 - if (hw_irq != XIVE_IPI_HW_IRQ) 1299 - xive_irq_free_data(virq); 1205 + xive_irq_free_data(virq); 1300 1206 } 1301 1207 1302 1208 static int xive_irq_domain_xlate(struct irq_domain *h, struct device_node *ct, ··· 1413 1335 1414 1336 xc = per_cpu(xive_cpu, cpu); 1415 1337 if (!xc) { 1416 - struct device_node *np; 1417 - 1418 1338 xc = kzalloc_node(sizeof(struct xive_cpu), 1419 1339 GFP_KERNEL, cpu_to_node(cpu)); 1420 1340 if (!xc) 1421 1341 return -ENOMEM; 1422 - np = of_get_cpu_node(cpu, NULL); 1423 - if (np) 1424 - xc->chip_id = of_get_ibm_chip_id(np); 1425 - of_node_put(np); 1426 1342 xc->hw_ipi = XIVE_BAD_IRQ; 1343 + xc->chip_id = XIVE_INVALID_CHIP_ID; 1344 + if (xive_ops->prepare_cpu) 1345 + xive_ops->prepare_cpu(cpu, xc); 1427 1346 1428 1347 per_cpu(xive_cpu, cpu) = xc; 1429 1348 } ··· 1483 1408 struct irq_desc *desc = irq_to_desc(irq); 1484 1409 struct irq_data *d = irq_desc_get_irq_data(desc); 1485 1410 struct xive_irq_data *xd; 1486 - unsigned int hw_irq = (unsigned int)irqd_to_hwirq(d); 1487 1411 1488 1412 /* 1489 1413 * Ignore anything that isn't a XIVE irq and ignore 1490 1414 * IPIs, so can just be dropped. 1491 1415 */ 1492 - if (d->domain != xive_irq_domain || hw_irq == XIVE_IPI_HW_IRQ) 1416 + if (d->domain != xive_irq_domain) 1493 1417 continue; 1494 1418 1495 1419 /* ··· 1666 1592 seq_puts(m, "\n"); 1667 1593 } 1668 1594 1669 - static void xive_debug_show_irq(struct seq_file *m, u32 hw_irq, struct irq_data *d) 1595 + static void xive_debug_show_irq(struct seq_file *m, struct irq_data *d) 1670 1596 { 1671 - struct irq_chip *chip = irq_data_get_irq_chip(d); 1597 + unsigned int hw_irq = (unsigned int)irqd_to_hwirq(d); 1672 1598 int rc; 1673 1599 u32 target; 1674 1600 u8 prio; 1675 1601 u32 lirq; 1676 - 1677 - if (!is_xive_irq(chip)) 1678 - return; 1602 + struct xive_irq_data *xd; 1603 + u64 val; 1679 1604 1680 1605 rc = xive_ops->get_irq_config(hw_irq, &target, &prio, &lirq); 1681 1606 if (rc) { ··· 1685 1612 seq_printf(m, "IRQ 0x%08x : target=0x%x prio=%02x lirq=0x%x ", 1686 1613 hw_irq, target, prio, lirq); 1687 1614 1688 - if (d) { 1689 - struct xive_irq_data *xd = irq_data_get_irq_handler_data(d); 1690 - u64 val = xive_esb_read(xd, XIVE_ESB_GET); 1691 - 1692 - seq_printf(m, "flags=%c%c%c PQ=%c%c", 1693 - xd->flags & XIVE_IRQ_FLAG_STORE_EOI ? 'S' : ' ', 1694 - xd->flags & XIVE_IRQ_FLAG_LSI ? 'L' : ' ', 1695 - xd->flags & XIVE_IRQ_FLAG_H_INT_ESB ? 'H' : ' ', 1696 - val & XIVE_ESB_VAL_P ? 'P' : '-', 1697 - val & XIVE_ESB_VAL_Q ? 'Q' : '-'); 1698 - } 1615 + xd = irq_data_get_irq_handler_data(d); 1616 + val = xive_esb_read(xd, XIVE_ESB_GET); 1617 + seq_printf(m, "flags=%c%c%c PQ=%c%c", 1618 + xd->flags & XIVE_IRQ_FLAG_STORE_EOI ? 'S' : ' ', 1619 + xd->flags & XIVE_IRQ_FLAG_LSI ? 'L' : ' ', 1620 + xd->flags & XIVE_IRQ_FLAG_H_INT_ESB ? 'H' : ' ', 1621 + val & XIVE_ESB_VAL_P ? 'P' : '-', 1622 + val & XIVE_ESB_VAL_Q ? 'Q' : '-'); 1699 1623 seq_puts(m, "\n"); 1700 1624 } 1701 1625 ··· 1710 1640 1711 1641 for_each_irq_desc(i, desc) { 1712 1642 struct irq_data *d = irq_desc_get_irq_data(desc); 1713 - unsigned int hw_irq; 1714 1643 1715 - if (!d) 1716 - continue; 1717 - 1718 - hw_irq = (unsigned int)irqd_to_hwirq(d); 1719 - 1720 - /* IPIs are special (HW number 0) */ 1721 - if (hw_irq != XIVE_IPI_HW_IRQ) 1722 - xive_debug_show_irq(m, hw_irq, d); 1644 + if (d->domain == xive_irq_domain) 1645 + xive_debug_show_irq(m, d); 1723 1646 } 1724 1647 return 0; 1725 1648 }

+6

arch/powerpc/sysdev/xive/native.c

··· 380 380 } 381 381 } 382 382 383 + static void xive_native_prepare_cpu(unsigned int cpu, struct xive_cpu *xc) 384 + { 385 + xc->chip_id = cpu_to_chip_id(cpu); 386 + } 387 + 383 388 static void xive_native_setup_cpu(unsigned int cpu, struct xive_cpu *xc) 384 389 { 385 390 s64 rc; ··· 467 462 .match = xive_native_match, 468 463 .shutdown = xive_native_shutdown, 469 464 .update_pending = xive_native_update_pending, 465 + .prepare_cpu = xive_native_prepare_cpu, 470 466 .setup_cpu = xive_native_setup_cpu, 471 467 .teardown_cpu = xive_native_teardown_cpu, 472 468 .sync_source = xive_native_sync_source,

+1 -1

arch/powerpc/sysdev/xive/spapr.c

··· 549 549 static bool xive_spapr_match(struct device_node *node) 550 550 { 551 551 /* Ignore cascaded controllers for the moment */ 552 - return 1; 552 + return true; 553 553 } 554 554 555 555 #ifdef CONFIG_SMP

+1 -2

arch/powerpc/sysdev/xive/xive-internal.h

··· 5 5 #ifndef __XIVE_INTERNAL_H 6 6 #define __XIVE_INTERNAL_H 7 7 8 - #define XIVE_IPI_HW_IRQ 0 /* interrupt source # for IPIs */ 9 - 10 8 /* 11 9 * A "disabled" interrupt should never fire, to catch problems 12 10 * we set its logical number to this ··· 44 46 u32 *sw_irq); 45 47 int (*setup_queue)(unsigned int cpu, struct xive_cpu *xc, u8 prio); 46 48 void (*cleanup_queue)(unsigned int cpu, struct xive_cpu *xc, u8 prio); 49 + void (*prepare_cpu)(unsigned int cpu, struct xive_cpu *xc); 47 50 void (*setup_cpu)(unsigned int cpu, struct xive_cpu *xc); 48 51 void (*teardown_cpu)(unsigned int cpu, struct xive_cpu *xc); 49 52 bool (*match)(struct device_node *np);

+25 -53

arch/powerpc/xmon/xmon.c

··· 54 54 #include <asm/code-patching.h> 55 55 #include <asm/sections.h> 56 56 #include <asm/inst.h> 57 + #include <asm/interrupt.h> 57 58 58 59 #ifdef CONFIG_PPC64 59 60 #include <asm/hvcall.h> ··· 606 605 * debugger break (IPI). This is similar to 607 606 * crash_kexec_secondary(). 608 607 */ 609 - if (TRAP(regs) != 0x100 || !wait_for_other_cpus(ncpus)) 608 + if (TRAP(regs) != INTERRUPT_SYSTEM_RESET || !wait_for_other_cpus(ncpus)) 610 609 smp_send_debugger_break(); 611 610 612 611 wait_for_other_cpus(ncpus); ··· 616 615 617 616 if (!locked_down) { 618 617 /* for breakpoint or single step, print curr insn */ 619 - if (bp || TRAP(regs) == 0xd00) 618 + if (bp || TRAP(regs) == INTERRUPT_TRACE) 620 619 ppc_inst_dump(regs->nip, 1, 0); 621 620 printf("enter ? for help\n"); 622 621 } ··· 685 684 disable_surveillance(); 686 685 if (!locked_down) { 687 686 /* for breakpoint or single step, print current insn */ 688 - if (bp || TRAP(regs) == 0xd00) 687 + if (bp || TRAP(regs) == INTERRUPT_TRACE) 689 688 ppc_inst_dump(regs->nip, 1, 0); 690 689 printf("enter ? for help\n"); 691 690 } ··· 1770 1769 printf(" sp: %lx\n", fp->gpr[1]); 1771 1770 printf(" msr: %lx\n", fp->msr); 1772 1771 1773 - if (trap == 0x300 || trap == 0x380 || trap == 0x600 || trap == 0x200) { 1772 + if (trap == INTERRUPT_DATA_STORAGE || 1773 + trap == INTERRUPT_DATA_SEGMENT || 1774 + trap == INTERRUPT_ALIGNMENT || 1775 + trap == INTERRUPT_MACHINE_CHECK) { 1774 1776 printf(" dar: %lx\n", fp->dar); 1775 - if (trap != 0x380) 1777 + if (trap != INTERRUPT_DATA_SEGMENT) 1776 1778 printf(" dsisr: %lx\n", fp->dsisr); 1777 1779 } 1778 1780 ··· 1789 1785 current->pid, current->comm); 1790 1786 } 1791 1787 1792 - if (trap == 0x700) 1788 + if (trap == INTERRUPT_PROGRAM) 1793 1789 print_bug_trap(fp); 1794 1790 1795 1791 printf(linux_banner); ··· 1819 1815 } 1820 1816 1821 1817 #ifdef CONFIG_PPC64 1822 - if (FULL_REGS(fp)) { 1823 - for (n = 0; n < 16; ++n) 1824 - printf("R%.2d = "REG" R%.2d = "REG"\n", 1825 - n, fp->gpr[n], n+16, fp->gpr[n+16]); 1826 - } else { 1827 - for (n = 0; n < 7; ++n) 1828 - printf("R%.2d = "REG" R%.2d = "REG"\n", 1829 - n, fp->gpr[n], n+7, fp->gpr[n+7]); 1830 - } 1818 + #define R_PER_LINE 2 1831 1819 #else 1832 - for (n = 0; n < 32; ++n) { 1833 - printf("R%.2d = %.8lx%s", n, fp->gpr[n], 1834 - (n & 3) == 3? "\n": " "); 1835 - if (n == 12 && !FULL_REGS(fp)) { 1836 - printf("\n"); 1837 - break; 1838 - } 1839 - } 1820 + #define R_PER_LINE 4 1840 1821 #endif 1822 + 1823 + for (n = 0; n < 32; ++n) { 1824 + printf("R%.2d = "REG"%s", n, fp->gpr[n], 1825 + (n % R_PER_LINE) == R_PER_LINE - 1 ? "\n" : " "); 1826 + } 1827 + 1841 1828 printf("pc = "); 1842 1829 xmon_print_symbol(fp->nip, " ", "\n"); 1843 1830 if (!trap_is_syscall(fp) && cpu_has_feature(CPU_FTR_CFAR)) { ··· 1841 1846 printf("ctr = "REG" xer = "REG" trap = %4lx\n", 1842 1847 fp->ctr, fp->xer, fp->trap); 1843 1848 trap = TRAP(fp); 1844 - if (trap == 0x300 || trap == 0x380 || trap == 0x600) 1849 + if (trap == INTERRUPT_DATA_STORAGE || 1850 + trap == INTERRUPT_DATA_SEGMENT || 1851 + trap == INTERRUPT_ALIGNMENT) 1845 1852 printf("dar = "REG" dsisr = %.8lx\n", fp->dar, fp->dsisr); 1846 1853 } 1847 1854 ··· 2724 2727 dump_one_xive(cpu); 2725 2728 } 2726 2729 2727 - static void dump_one_xive_irq(u32 num, struct irq_data *d) 2728 - { 2729 - xmon_xive_get_irq_config(num, d); 2730 - } 2731 - 2732 - static void dump_all_xive_irq(void) 2733 - { 2734 - unsigned int i; 2735 - struct irq_desc *desc; 2736 - 2737 - for_each_irq_desc(i, desc) { 2738 - struct irq_data *d = irq_desc_get_irq_data(desc); 2739 - unsigned int hwirq; 2740 - 2741 - if (!d) 2742 - continue; 2743 - 2744 - hwirq = (unsigned int)irqd_to_hwirq(d); 2745 - /* IPIs are special (HW number 0) */ 2746 - if (hwirq) 2747 - dump_one_xive_irq(hwirq, d); 2748 - } 2749 - } 2750 - 2751 2730 static void dump_xives(void) 2752 2731 { 2753 2732 unsigned long num; ··· 2740 2767 return; 2741 2768 } else if (c == 'i') { 2742 2769 if (scanhex(&num)) 2743 - dump_one_xive_irq(num, NULL); 2770 + xmon_xive_get_irq_config(num, NULL); 2744 2771 else 2745 - dump_all_xive_irq(); 2772 + xmon_xive_get_irq_all(); 2746 2773 return; 2747 2774 } 2748 2775 ··· 2953 2980 if (!ppc_inst_prefixed(inst)) 2954 2981 dump_func(ppc_inst_val(inst), adr); 2955 2982 else 2956 - dump_func(ppc_inst_as_u64(inst), adr); 2983 + dump_func(ppc_inst_as_ulong(inst), adr); 2957 2984 printf("\n"); 2958 2985 } 2959 2986 return adr - first_adr; ··· 4185 4212 DUMP_FIELD(spu, "0x%p", pdata); 4186 4213 } 4187 4214 4188 - int 4189 - spu_inst_dump(unsigned long adr, long count, int praddr) 4215 + static int spu_inst_dump(unsigned long adr, long count, int praddr) 4190 4216 { 4191 4217 return generic_inst_dump(adr, count, praddr, print_insn_spu); 4192 4218 }

+2 -1

arch/s390/include/asm/vdso/gettimeofday.h

··· 68 68 } 69 69 70 70 #ifdef CONFIG_TIME_NS 71 - static __always_inline const struct vdso_data *__arch_get_timens_vdso_data(void) 71 + static __always_inline 72 + const struct vdso_data *__arch_get_timens_vdso_data(const struct vdso_data *vd) 72 73 { 73 74 return _timens_data; 74 75 }

+2 -1

arch/x86/include/asm/vdso/gettimeofday.h

··· 58 58 #endif 59 59 60 60 #ifdef CONFIG_TIME_NS 61 - static __always_inline const struct vdso_data *__arch_get_timens_vdso_data(void) 61 + static __always_inline 62 + const struct vdso_data *__arch_get_timens_vdso_data(const struct vdso_data *vd) 62 63 { 63 64 return __timens_vdso_data; 64 65 }

+1 -1

drivers/i2c/busses/Kconfig

··· 776 776 777 777 config I2C_MV64XXX 778 778 tristate "Marvell mv64xxx I2C Controller" 779 - depends on MV64X60 || PLAT_ORION || ARCH_SUNXI || ARCH_MVEBU || COMPILE_TEST 779 + depends on PLAT_ORION || ARCH_SUNXI || ARCH_MVEBU || COMPILE_TEST 780 780 help 781 781 If you say yes to this option, support will be included for the 782 782 built-in I2C interface on the Marvell 64xxx line of host bridges.

+2 -2

drivers/macintosh/via-pmu.c

··· 180 180 static int option_server_mode; 181 181 182 182 int pmu_battery_count; 183 - int pmu_cur_battery; 183 + static int pmu_cur_battery; 184 184 unsigned int pmu_power_flags = PMU_PWR_AC_PRESENT; 185 185 struct pmu_battery_info pmu_batteries[PMU_MAX_BATTERIES]; 186 186 static int query_batt_timer = BATTERY_POLLING_COUNT; 187 187 static struct adb_request batt_req; 188 188 static struct proc_dir_entry *proc_pmu_batt[PMU_MAX_BATTERIES]; 189 189 190 - int __fake_sleep; 191 190 int asleep; 192 191 193 192 #ifdef CONFIG_ADB ··· 1832 1833 */ 1833 1834 1834 1835 static u32 save_via[8]; 1836 + static int __fake_sleep; 1835 1837 1836 1838 static void 1837 1839 save_via_state(void)

+1 -1

drivers/macintosh/windfarm_core.c

··· 56 56 static int wf_client_count; 57 57 static unsigned int wf_overtemp; 58 58 static unsigned int wf_overtemp_counter; 59 - struct task_struct *wf_thread; 59 + static struct task_struct *wf_thread; 60 60 61 61 static struct platform_device wf_platform_device = { 62 62 .name = "windfarm",

+1 -1

drivers/macintosh/windfarm_pm121.c

··· 433 433 struct wf_pid_state pid; 434 434 }; 435 435 436 - struct pm121_sys_state *pm121_sys_state[N_LOOPS] = {}; 436 + static struct pm121_sys_state *pm121_sys_state[N_LOOPS] = {}; 437 437 438 438 /* 439 439 * ****** CPU Fans Control Loop ******

+1 -1

drivers/macintosh/windfarm_smu_controls.c

··· 94 94 return rc; 95 95 wait_for_completion(&comp); 96 96 97 - /* Handle fallback (see coment above) */ 97 + /* Handle fallback (see comment above) */ 98 98 if (cmd.status != 0 && smu_supports_new_fans_ops) { 99 99 printk(KERN_WARNING "windfarm: SMU failed new fan command " 100 100 "falling back to old method\n");

+35

include/linux/compat.h

··· 467 467 unsafe_put_user(__s->sig[0], &__c->sig[0], label); \ 468 468 } \ 469 469 } while (0) 470 + 471 + #define unsafe_get_compat_sigset(set, compat, label) do { \ 472 + const compat_sigset_t __user *__c = compat; \ 473 + compat_sigset_word hi, lo; \ 474 + sigset_t *__s = set; \ 475 + \ 476 + switch (_NSIG_WORDS) { \ 477 + case 4: \ 478 + unsafe_get_user(lo, &__c->sig[7], label); \ 479 + unsafe_get_user(hi, &__c->sig[6], label); \ 480 + __s->sig[3] = hi | (((long)lo) << 32); \ 481 + fallthrough; \ 482 + case 3: \ 483 + unsafe_get_user(lo, &__c->sig[5], label); \ 484 + unsafe_get_user(hi, &__c->sig[4], label); \ 485 + __s->sig[2] = hi | (((long)lo) << 32); \ 486 + fallthrough; \ 487 + case 2: \ 488 + unsafe_get_user(lo, &__c->sig[3], label); \ 489 + unsafe_get_user(hi, &__c->sig[2], label); \ 490 + __s->sig[1] = hi | (((long)lo) << 32); \ 491 + fallthrough; \ 492 + case 1: \ 493 + unsafe_get_user(lo, &__c->sig[1], label); \ 494 + unsafe_get_user(hi, &__c->sig[0], label); \ 495 + __s->sig[0] = hi | (((long)lo) << 32); \ 496 + } \ 497 + } while (0) 470 498 #else 471 499 #define unsafe_put_compat_sigset(compat, set, label) do { \ 472 500 compat_sigset_t __user *__c = compat; \ 473 501 const sigset_t *__s = set; \ 474 502 \ 475 503 unsafe_copy_to_user(__c, __s, sizeof(*__c), label); \ 504 + } while (0) 505 + 506 + #define unsafe_get_compat_sigset(set, compat, label) do { \ 507 + const compat_sigset_t __user *__c = compat; \ 508 + sigset_t *__s = set; \ 509 + \ 510 + unsafe_copy_from_user(__s, __c, sizeof(*__c), label); \ 476 511 } while (0) 477 512 #endif 478 513

+1

include/linux/uaccess.h

··· 397 397 #define unsafe_get_user(x,p,e) unsafe_op_wrap(__get_user(x,p),e) 398 398 #define unsafe_put_user(x,p,e) unsafe_op_wrap(__put_user(x,p),e) 399 399 #define unsafe_copy_to_user(d,s,l,e) unsafe_op_wrap(__copy_to_user(d,s,l),e) 400 + #define unsafe_copy_from_user(d,s,l,e) unsafe_op_wrap(__copy_from_user(d,s,l),e) 400 401 static inline unsigned long user_access_save(void) { return 0UL; } 401 402 static inline void user_access_restore(unsigned long flags) { } 402 403 #endif

+17 -14

lib/vdso/gettimeofday.c

··· 46 46 #endif 47 47 48 48 #ifdef CONFIG_TIME_NS 49 - static int do_hres_timens(const struct vdso_data *vdns, clockid_t clk, 50 - struct __kernel_timespec *ts) 49 + static __always_inline int do_hres_timens(const struct vdso_data *vdns, clockid_t clk, 50 + struct __kernel_timespec *ts) 51 51 { 52 - const struct vdso_data *vd = __arch_get_timens_vdso_data(); 52 + const struct vdso_data *vd; 53 53 const struct timens_offset *offs = &vdns->offset[clk]; 54 54 const struct vdso_timestamp *vdso_ts; 55 55 u64 cycles, last, ns; 56 56 u32 seq; 57 57 s64 sec; 58 58 59 + vd = vdns - (clk == CLOCK_MONOTONIC_RAW ? CS_RAW : CS_HRES_COARSE); 60 + vd = __arch_get_timens_vdso_data(vd); 59 61 if (clk != CLOCK_MONOTONIC_RAW) 60 62 vd = &vd[CS_HRES_COARSE]; 61 63 else ··· 94 92 return 0; 95 93 } 96 94 #else 97 - static __always_inline const struct vdso_data *__arch_get_timens_vdso_data(void) 95 + static __always_inline 96 + const struct vdso_data *__arch_get_timens_vdso_data(const struct vdso_data *vd) 98 97 { 99 98 return NULL; 100 99 } 101 100 102 - static int do_hres_timens(const struct vdso_data *vdns, clockid_t clk, 103 - struct __kernel_timespec *ts) 101 + static __always_inline int do_hres_timens(const struct vdso_data *vdns, clockid_t clk, 102 + struct __kernel_timespec *ts) 104 103 { 105 104 return -EINVAL; 106 105 } ··· 162 159 } 163 160 164 161 #ifdef CONFIG_TIME_NS 165 - static int do_coarse_timens(const struct vdso_data *vdns, clockid_t clk, 166 - struct __kernel_timespec *ts) 162 + static __always_inline int do_coarse_timens(const struct vdso_data *vdns, clockid_t clk, 163 + struct __kernel_timespec *ts) 167 164 { 168 - const struct vdso_data *vd = __arch_get_timens_vdso_data(); 165 + const struct vdso_data *vd = __arch_get_timens_vdso_data(vdns); 169 166 const struct vdso_timestamp *vdso_ts = &vd->basetime[clk]; 170 167 const struct timens_offset *offs = &vdns->offset[clk]; 171 168 u64 nsec; ··· 191 188 return 0; 192 189 } 193 190 #else 194 - static int do_coarse_timens(const struct vdso_data *vdns, clockid_t clk, 195 - struct __kernel_timespec *ts) 191 + static __always_inline int do_coarse_timens(const struct vdso_data *vdns, clockid_t clk, 192 + struct __kernel_timespec *ts) 196 193 { 197 194 return -1; 198 195 } ··· 313 310 if (unlikely(tz != NULL)) { 314 311 if (IS_ENABLED(CONFIG_TIME_NS) && 315 312 vd->clock_mode == VDSO_CLOCKMODE_TIMENS) 316 - vd = __arch_get_timens_vdso_data(); 313 + vd = __arch_get_timens_vdso_data(vd); 317 314 318 315 tz->tz_minuteswest = vd[CS_HRES_COARSE].tz_minuteswest; 319 316 tz->tz_dsttime = vd[CS_HRES_COARSE].tz_dsttime; ··· 336 333 337 334 if (IS_ENABLED(CONFIG_TIME_NS) && 338 335 vd->clock_mode == VDSO_CLOCKMODE_TIMENS) 339 - vd = __arch_get_timens_vdso_data(); 336 + vd = __arch_get_timens_vdso_data(vd); 340 337 341 338 t = READ_ONCE(vd[CS_HRES_COARSE].basetime[CLOCK_REALTIME].sec); 342 339 ··· 366 363 367 364 if (IS_ENABLED(CONFIG_TIME_NS) && 368 365 vd->clock_mode == VDSO_CLOCKMODE_TIMENS) 369 - vd = __arch_get_timens_vdso_data(); 366 + vd = __arch_get_timens_vdso_data(vd); 370 367 371 368 /* 372 369 * Convert the clockid to a bitmask and use it to check which

+1 -10

tools/testing/selftests/powerpc/alignment/alignment_handler.c

··· 10 10 * 11 11 * We create two sets of source and destination buffers, one in regular memory, 12 12 * the other cache-inhibited (by default we use /dev/fb0 for this, but an 13 - * alterative path for cache-inhibited memory may be provided). 14 - * 15 - * One way to get cache-inhibited memory is to use the "mem" kernel parameter 16 - * to limit the kernel to less memory than actually exists. Addresses above 17 - * the limit may still be accessed but will be treated as cache-inhibited. For 18 - * example, if there is actually 4GB of memory and the parameter "mem=3GB" is 19 - * used, memory from address 0xC0000000 onwards is treated as cache-inhibited. 20 - * To access this region /dev/mem is used. The kernel should be configured 21 - * without CONFIG_STRICT_DEVMEM. In this case use: 22 - * ./alignment_handler /dev/mem 0xc0000000 13 + * alterative path for cache-inhibited memory may be provided, e.g. memtrace). 23 14 * 24 15 * We initialise the source buffers, then use whichever set of load/store 25 16 * instructions is under test to copy bytes from the source buffers to the

+1

tools/testing/selftests/powerpc/mm/Makefile

··· 5 5 TEST_GEN_PROGS := hugetlb_vs_thp_test subpage_prot prot_sao segv_errors wild_bctr \ 6 6 large_vm_fork_separation bad_accesses pkey_exec_prot \ 7 7 pkey_siginfo stack_expansion_signal stack_expansion_ldst 8 + TEST_PROGS := stress_code_patching.sh 8 9 9 10 TEST_GEN_PROGS_EXTENDED := tlbie_test 10 11 TEST_GEN_FILES := tempfile

+49

tools/testing/selftests/powerpc/mm/stress_code_patching.sh

··· 1 + #!/bin/bash 2 + # SPDX-License-Identifier: GPL-2.0-or-later 3 + 4 + TIMEOUT=30 5 + 6 + DEBUFS_DIR=`cat /proc/mounts | grep debugfs | awk '{print $2}'` 7 + if [ ! -e "$DEBUFS_DIR" ] 8 + then 9 + echo "debugfs not found, skipping" 1>&2 10 + exit 4 11 + fi 12 + 13 + if [ ! -e "$DEBUFS_DIR/tracing/current_tracer" ] 14 + then 15 + echo "Tracing files not found, skipping" 1>&2 16 + exit 4 17 + fi 18 + 19 + 20 + echo "Testing for spurious faults when mapping kernel memory..." 21 + 22 + if grep -q "FUNCTION TRACING IS CORRUPTED" "$DEBUFS_DIR/tracing/trace" 23 + then 24 + echo "FAILED: Ftrace already dead. Probably due to a spurious fault" 1>&2 25 + exit 1 26 + fi 27 + 28 + dmesg -C 29 + START_TIME=`date +%s` 30 + END_TIME=`expr $START_TIME + $TIMEOUT` 31 + while [ `date +%s` -lt $END_TIME ] 32 + do 33 + echo function > $DEBUFS_DIR/tracing/current_tracer 34 + echo nop > $DEBUFS_DIR/tracing/current_tracer 35 + if dmesg | grep -q 'ftrace bug' 36 + then 37 + break 38 + fi 39 + done 40 + 41 + echo nop > $DEBUFS_DIR/tracing/current_tracer 42 + if dmesg | grep -q 'ftrace bug' 43 + then 44 + echo "FAILED: Mapping kernel memory causes spurious faults" 1>&2 45 + exit 1 46 + else 47 + echo "OK: Mapping kernel memory does not cause spurious faults" 48 + exit 0 49 + fi

+1 -1

tools/testing/selftests/powerpc/nx-gzip/gzfht_test.c

··· 324 324 fprintf(stderr, "error: cannot progress; "); 325 325 fprintf(stderr, "too many faults\n"); 326 326 exit(-1); 327 - }; 327 + } 328 328 } 329 329 330 330 fault_tries = NX_MAX_FAULTS; /* Reset for the next chunk */

+1

tools/testing/selftests/powerpc/ptrace/.gitignore

··· 14 14 core-pkey 15 15 ptrace-pkey 16 16 ptrace-syscall 17 + ptrace-perf-hwbreak

+1 -1

tools/testing/selftests/powerpc/ptrace/Makefile

··· 2 2 TEST_GEN_PROGS := ptrace-gpr ptrace-tm-gpr ptrace-tm-spd-gpr \ 3 3 ptrace-tar ptrace-tm-tar ptrace-tm-spd-tar ptrace-vsx ptrace-tm-vsx \ 4 4 ptrace-tm-spd-vsx ptrace-tm-spr ptrace-hwbreak ptrace-pkey core-pkey \ 5 - perf-hwbreak ptrace-syscall 5 + perf-hwbreak ptrace-syscall ptrace-perf-hwbreak 6 6 7 7 top_srcdir = ../../../../.. 8 8 include ../../lib.mk

+594 -41

tools/testing/selftests/powerpc/ptrace/perf-hwbreak.c

··· 21 21 #include <assert.h> 22 22 #include <stdio.h> 23 23 #include <stdlib.h> 24 + #include <signal.h> 24 25 #include <string.h> 25 26 #include <sys/ioctl.h> 27 + #include <sys/wait.h> 28 + #include <sys/ptrace.h> 29 + #include <sys/sysinfo.h> 30 + #include <asm/ptrace.h> 26 31 #include <elf.h> 27 32 #include <pthread.h> 28 33 #include <sys/syscall.h> ··· 35 30 #include <linux/hw_breakpoint.h> 36 31 #include "utils.h" 37 32 33 + #ifndef PPC_DEBUG_FEATURE_DATA_BP_ARCH_31 34 + #define PPC_DEBUG_FEATURE_DATA_BP_ARCH_31 0x20 35 + #endif 36 + 38 37 #define MAX_LOOPS 10000 39 38 40 39 #define DAWR_LENGTH_MAX ((0x3f + 1) * 8) 41 40 42 - static inline int sys_perf_event_open(struct perf_event_attr *attr, pid_t pid, 43 - int cpu, int group_fd, 44 - unsigned long flags) 41 + int nprocs; 42 + 43 + static volatile int a = 10; 44 + static volatile int b = 10; 45 + static volatile char c[512 + 8] __attribute__((aligned(512))); 46 + 47 + static void perf_event_attr_set(struct perf_event_attr *attr, 48 + __u32 type, __u64 addr, __u64 len, 49 + bool exclude_user) 45 50 { 46 - attr->size = sizeof(*attr); 47 - return syscall(__NR_perf_event_open, attr, pid, cpu, group_fd, flags); 51 + memset(attr, 0, sizeof(struct perf_event_attr)); 52 + attr->type = PERF_TYPE_BREAKPOINT; 53 + attr->size = sizeof(struct perf_event_attr); 54 + attr->bp_type = type; 55 + attr->bp_addr = addr; 56 + attr->bp_len = len; 57 + attr->exclude_kernel = 1; 58 + attr->exclude_hv = 1; 59 + attr->exclude_guest = 1; 60 + attr->exclude_user = exclude_user; 61 + attr->disabled = 1; 62 + } 63 + 64 + static int 65 + perf_process_event_open_exclude_user(__u32 type, __u64 addr, __u64 len, bool exclude_user) 66 + { 67 + struct perf_event_attr attr; 68 + 69 + perf_event_attr_set(&attr, type, addr, len, exclude_user); 70 + return syscall(__NR_perf_event_open, &attr, getpid(), -1, -1, 0); 71 + } 72 + 73 + static int perf_process_event_open(__u32 type, __u64 addr, __u64 len) 74 + { 75 + struct perf_event_attr attr; 76 + 77 + perf_event_attr_set(&attr, type, addr, len, 0); 78 + return syscall(__NR_perf_event_open, &attr, getpid(), -1, -1, 0); 79 + } 80 + 81 + static int perf_cpu_event_open(long cpu, __u32 type, __u64 addr, __u64 len) 82 + { 83 + struct perf_event_attr attr; 84 + 85 + perf_event_attr_set(&attr, type, addr, len, 0); 86 + return syscall(__NR_perf_event_open, &attr, -1, cpu, -1, 0); 87 + } 88 + 89 + static void close_fds(int *fd, int n) 90 + { 91 + int i; 92 + 93 + for (i = 0; i < n; i++) 94 + close(fd[i]); 95 + } 96 + 97 + static unsigned long read_fds(int *fd, int n) 98 + { 99 + int i; 100 + unsigned long c = 0; 101 + unsigned long count = 0; 102 + size_t res; 103 + 104 + for (i = 0; i < n; i++) { 105 + res = read(fd[i], &c, sizeof(c)); 106 + assert(res == sizeof(unsigned long long)); 107 + count += c; 108 + } 109 + return count; 110 + } 111 + 112 + static void reset_fds(int *fd, int n) 113 + { 114 + int i; 115 + 116 + for (i = 0; i < n; i++) 117 + ioctl(fd[i], PERF_EVENT_IOC_RESET); 118 + } 119 + 120 + static void enable_fds(int *fd, int n) 121 + { 122 + int i; 123 + 124 + for (i = 0; i < n; i++) 125 + ioctl(fd[i], PERF_EVENT_IOC_ENABLE); 126 + } 127 + 128 + static void disable_fds(int *fd, int n) 129 + { 130 + int i; 131 + 132 + for (i = 0; i < n; i++) 133 + ioctl(fd[i], PERF_EVENT_IOC_DISABLE); 134 + } 135 + 136 + static int perf_systemwide_event_open(int *fd, __u32 type, __u64 addr, __u64 len) 137 + { 138 + int i = 0; 139 + 140 + /* Assume online processors are 0 to nprocs for simplisity */ 141 + for (i = 0; i < nprocs; i++) { 142 + fd[i] = perf_cpu_event_open(i, type, addr, len); 143 + if (fd[i] < 0) { 144 + close_fds(fd, i); 145 + return fd[i]; 146 + } 147 + } 148 + return 0; 48 149 } 49 150 50 151 static inline bool breakpoint_test(int len) 51 152 { 52 - struct perf_event_attr attr; 53 153 int fd; 54 154 55 - /* setup counters */ 56 - memset(&attr, 0, sizeof(attr)); 57 - attr.disabled = 1; 58 - attr.type = PERF_TYPE_BREAKPOINT; 59 - attr.bp_type = HW_BREAKPOINT_R; 60 155 /* bp_addr can point anywhere but needs to be aligned */ 61 - attr.bp_addr = (__u64)(&attr) & 0xfffffffffffff800; 62 - attr.bp_len = len; 63 - fd = sys_perf_event_open(&attr, 0, -1, -1, 0); 156 + fd = perf_process_event_open(HW_BREAKPOINT_R, (__u64)(&fd) & 0xfffffffffffff800, len); 64 157 if (fd < 0) 65 158 return false; 66 159 close(fd); ··· 178 75 static int runtestsingle(int readwriteflag, int exclude_user, int arraytest) 179 76 { 180 77 int i,j; 181 - struct perf_event_attr attr; 182 78 size_t res; 183 79 unsigned long long breaks, needed; 184 80 int readint; ··· 187 85 int break_fd; 188 86 int loop_num = MAX_LOOPS - (rand() % 100); /* provide some variability */ 189 87 volatile int *k; 88 + __u64 len; 190 89 191 90 /* align to 0x400 boundary as required by DAWR */ 192 91 readintalign = (int *)(((unsigned long)readintarraybig + 0x7ff) & ··· 197 94 if (arraytest) 198 95 ptr = &readintalign[0]; 199 96 200 - /* setup counters */ 201 - memset(&attr, 0, sizeof(attr)); 202 - attr.disabled = 1; 203 - attr.type = PERF_TYPE_BREAKPOINT; 204 - attr.bp_type = readwriteflag; 205 - attr.bp_addr = (__u64)ptr; 206 - attr.bp_len = sizeof(int); 207 - if (arraytest) 208 - attr.bp_len = DAWR_LENGTH_MAX; 209 - attr.exclude_user = exclude_user; 210 - break_fd = sys_perf_event_open(&attr, 0, -1, -1, 0); 97 + len = arraytest ? DAWR_LENGTH_MAX : sizeof(int); 98 + break_fd = perf_process_event_open_exclude_user(readwriteflag, (__u64)ptr, 99 + len, exclude_user); 211 100 if (break_fd < 0) { 212 - perror("sys_perf_event_open"); 101 + perror("perf_process_event_open_exclude_user"); 213 102 exit(1); 214 103 } 215 104 ··· 248 153 void *target; 249 154 volatile __u16 temp16; 250 155 volatile __u64 temp64; 251 - struct perf_event_attr attr; 252 156 int break_fd; 253 157 unsigned long long breaks; 254 158 int fail = 0; ··· 259 165 exit(EXIT_FAILURE); 260 166 } 261 167 262 - /* setup counters */ 263 - memset(&attr, 0, sizeof(attr)); 264 - attr.disabled = 1; 265 - attr.type = PERF_TYPE_BREAKPOINT; 266 - attr.exclude_kernel = 1; 267 - attr.exclude_hv = 1; 268 - attr.exclude_guest = 1; 269 - attr.bp_type = HW_BREAKPOINT_RW; 270 168 /* watch middle half of target array */ 271 - attr.bp_addr = (__u64)(target + 2); 272 - attr.bp_len = 4; 273 - break_fd = sys_perf_event_open(&attr, 0, -1, -1, 0); 169 + break_fd = perf_process_event_open(HW_BREAKPOINT_RW, (__u64)(target + 2), 4); 274 170 if (break_fd < 0) { 275 171 free(target); 276 - perror("sys_perf_event_open"); 172 + perror("perf_process_event_open"); 277 173 exit(EXIT_FAILURE); 278 174 } 279 175 ··· 347 263 return fail; 348 264 } 349 265 266 + static void multi_dawr_workload(void) 267 + { 268 + a += 10; 269 + b += 10; 270 + c[512 + 1] += 'a'; 271 + } 272 + 273 + static int test_process_multi_diff_addr(void) 274 + { 275 + unsigned long long breaks1 = 0, breaks2 = 0; 276 + int fd1, fd2; 277 + char *desc = "Process specific, Two events, diff addr"; 278 + size_t res; 279 + 280 + fd1 = perf_process_event_open(HW_BREAKPOINT_RW, (__u64)&a, (__u64)sizeof(a)); 281 + if (fd1 < 0) { 282 + perror("perf_process_event_open"); 283 + exit(EXIT_FAILURE); 284 + } 285 + 286 + fd2 = perf_process_event_open(HW_BREAKPOINT_RW, (__u64)&b, (__u64)sizeof(b)); 287 + if (fd2 < 0) { 288 + close(fd1); 289 + perror("perf_process_event_open"); 290 + exit(EXIT_FAILURE); 291 + } 292 + 293 + ioctl(fd1, PERF_EVENT_IOC_RESET); 294 + ioctl(fd2, PERF_EVENT_IOC_RESET); 295 + ioctl(fd1, PERF_EVENT_IOC_ENABLE); 296 + ioctl(fd2, PERF_EVENT_IOC_ENABLE); 297 + multi_dawr_workload(); 298 + ioctl(fd1, PERF_EVENT_IOC_DISABLE); 299 + ioctl(fd2, PERF_EVENT_IOC_DISABLE); 300 + 301 + res = read(fd1, &breaks1, sizeof(breaks1)); 302 + assert(res == sizeof(unsigned long long)); 303 + res = read(fd2, &breaks2, sizeof(breaks2)); 304 + assert(res == sizeof(unsigned long long)); 305 + 306 + close(fd1); 307 + close(fd2); 308 + 309 + if (breaks1 != 2 || breaks2 != 2) { 310 + printf("FAILED: %s: %lld != 2 || %lld != 2\n", desc, breaks1, breaks2); 311 + return 1; 312 + } 313 + 314 + printf("TESTED: %s\n", desc); 315 + return 0; 316 + } 317 + 318 + static int test_process_multi_same_addr(void) 319 + { 320 + unsigned long long breaks1 = 0, breaks2 = 0; 321 + int fd1, fd2; 322 + char *desc = "Process specific, Two events, same addr"; 323 + size_t res; 324 + 325 + fd1 = perf_process_event_open(HW_BREAKPOINT_RW, (__u64)&a, (__u64)sizeof(a)); 326 + if (fd1 < 0) { 327 + perror("perf_process_event_open"); 328 + exit(EXIT_FAILURE); 329 + } 330 + 331 + fd2 = perf_process_event_open(HW_BREAKPOINT_RW, (__u64)&a, (__u64)sizeof(a)); 332 + if (fd2 < 0) { 333 + close(fd1); 334 + perror("perf_process_event_open"); 335 + exit(EXIT_FAILURE); 336 + } 337 + 338 + ioctl(fd1, PERF_EVENT_IOC_RESET); 339 + ioctl(fd2, PERF_EVENT_IOC_RESET); 340 + ioctl(fd1, PERF_EVENT_IOC_ENABLE); 341 + ioctl(fd2, PERF_EVENT_IOC_ENABLE); 342 + multi_dawr_workload(); 343 + ioctl(fd1, PERF_EVENT_IOC_DISABLE); 344 + ioctl(fd2, PERF_EVENT_IOC_DISABLE); 345 + 346 + res = read(fd1, &breaks1, sizeof(breaks1)); 347 + assert(res == sizeof(unsigned long long)); 348 + res = read(fd2, &breaks2, sizeof(breaks2)); 349 + assert(res == sizeof(unsigned long long)); 350 + 351 + close(fd1); 352 + close(fd2); 353 + 354 + if (breaks1 != 2 || breaks2 != 2) { 355 + printf("FAILED: %s: %lld != 2 || %lld != 2\n", desc, breaks1, breaks2); 356 + return 1; 357 + } 358 + 359 + printf("TESTED: %s\n", desc); 360 + return 0; 361 + } 362 + 363 + static int test_process_multi_diff_addr_ro_wo(void) 364 + { 365 + unsigned long long breaks1 = 0, breaks2 = 0; 366 + int fd1, fd2; 367 + char *desc = "Process specific, Two events, diff addr, one is RO, other is WO"; 368 + size_t res; 369 + 370 + fd1 = perf_process_event_open(HW_BREAKPOINT_W, (__u64)&a, (__u64)sizeof(a)); 371 + if (fd1 < 0) { 372 + perror("perf_process_event_open"); 373 + exit(EXIT_FAILURE); 374 + } 375 + 376 + fd2 = perf_process_event_open(HW_BREAKPOINT_R, (__u64)&b, (__u64)sizeof(b)); 377 + if (fd2 < 0) { 378 + close(fd1); 379 + perror("perf_process_event_open"); 380 + exit(EXIT_FAILURE); 381 + } 382 + 383 + ioctl(fd1, PERF_EVENT_IOC_RESET); 384 + ioctl(fd2, PERF_EVENT_IOC_RESET); 385 + ioctl(fd1, PERF_EVENT_IOC_ENABLE); 386 + ioctl(fd2, PERF_EVENT_IOC_ENABLE); 387 + multi_dawr_workload(); 388 + ioctl(fd1, PERF_EVENT_IOC_DISABLE); 389 + ioctl(fd2, PERF_EVENT_IOC_DISABLE); 390 + 391 + res = read(fd1, &breaks1, sizeof(breaks1)); 392 + assert(res == sizeof(unsigned long long)); 393 + res = read(fd2, &breaks2, sizeof(breaks2)); 394 + assert(res == sizeof(unsigned long long)); 395 + 396 + close(fd1); 397 + close(fd2); 398 + 399 + if (breaks1 != 1 || breaks2 != 1) { 400 + printf("FAILED: %s: %lld != 1 || %lld != 1\n", desc, breaks1, breaks2); 401 + return 1; 402 + } 403 + 404 + printf("TESTED: %s\n", desc); 405 + return 0; 406 + } 407 + 408 + static int test_process_multi_same_addr_ro_wo(void) 409 + { 410 + unsigned long long breaks1 = 0, breaks2 = 0; 411 + int fd1, fd2; 412 + char *desc = "Process specific, Two events, same addr, one is RO, other is WO"; 413 + size_t res; 414 + 415 + fd1 = perf_process_event_open(HW_BREAKPOINT_R, (__u64)&a, (__u64)sizeof(a)); 416 + if (fd1 < 0) { 417 + perror("perf_process_event_open"); 418 + exit(EXIT_FAILURE); 419 + } 420 + 421 + fd2 = perf_process_event_open(HW_BREAKPOINT_W, (__u64)&a, (__u64)sizeof(a)); 422 + if (fd2 < 0) { 423 + close(fd1); 424 + perror("perf_process_event_open"); 425 + exit(EXIT_FAILURE); 426 + } 427 + 428 + ioctl(fd1, PERF_EVENT_IOC_RESET); 429 + ioctl(fd2, PERF_EVENT_IOC_RESET); 430 + ioctl(fd1, PERF_EVENT_IOC_ENABLE); 431 + ioctl(fd2, PERF_EVENT_IOC_ENABLE); 432 + multi_dawr_workload(); 433 + ioctl(fd1, PERF_EVENT_IOC_DISABLE); 434 + ioctl(fd2, PERF_EVENT_IOC_DISABLE); 435 + 436 + res = read(fd1, &breaks1, sizeof(breaks1)); 437 + assert(res == sizeof(unsigned long long)); 438 + res = read(fd2, &breaks2, sizeof(breaks2)); 439 + assert(res == sizeof(unsigned long long)); 440 + 441 + close(fd1); 442 + close(fd2); 443 + 444 + if (breaks1 != 1 || breaks2 != 1) { 445 + printf("FAILED: %s: %lld != 1 || %lld != 1\n", desc, breaks1, breaks2); 446 + return 1; 447 + } 448 + 449 + printf("TESTED: %s\n", desc); 450 + return 0; 451 + } 452 + 453 + static int test_syswide_multi_diff_addr(void) 454 + { 455 + unsigned long long breaks1 = 0, breaks2 = 0; 456 + int *fd1 = malloc(nprocs * sizeof(int)); 457 + int *fd2 = malloc(nprocs * sizeof(int)); 458 + char *desc = "Systemwide, Two events, diff addr"; 459 + int ret; 460 + 461 + ret = perf_systemwide_event_open(fd1, HW_BREAKPOINT_RW, (__u64)&a, (__u64)sizeof(a)); 462 + if (ret) { 463 + perror("perf_systemwide_event_open"); 464 + exit(EXIT_FAILURE); 465 + } 466 + 467 + ret = perf_systemwide_event_open(fd2, HW_BREAKPOINT_RW, (__u64)&b, (__u64)sizeof(b)); 468 + if (ret) { 469 + close_fds(fd1, nprocs); 470 + perror("perf_systemwide_event_open"); 471 + exit(EXIT_FAILURE); 472 + } 473 + 474 + reset_fds(fd1, nprocs); 475 + reset_fds(fd2, nprocs); 476 + enable_fds(fd1, nprocs); 477 + enable_fds(fd2, nprocs); 478 + multi_dawr_workload(); 479 + disable_fds(fd1, nprocs); 480 + disable_fds(fd2, nprocs); 481 + 482 + breaks1 = read_fds(fd1, nprocs); 483 + breaks2 = read_fds(fd2, nprocs); 484 + 485 + close_fds(fd1, nprocs); 486 + close_fds(fd2, nprocs); 487 + 488 + free(fd1); 489 + free(fd2); 490 + 491 + if (breaks1 != 2 || breaks2 != 2) { 492 + printf("FAILED: %s: %lld != 2 || %lld != 2\n", desc, breaks1, breaks2); 493 + return 1; 494 + } 495 + 496 + printf("TESTED: %s\n", desc); 497 + return 0; 498 + } 499 + 500 + static int test_syswide_multi_same_addr(void) 501 + { 502 + unsigned long long breaks1 = 0, breaks2 = 0; 503 + int *fd1 = malloc(nprocs * sizeof(int)); 504 + int *fd2 = malloc(nprocs * sizeof(int)); 505 + char *desc = "Systemwide, Two events, same addr"; 506 + int ret; 507 + 508 + ret = perf_systemwide_event_open(fd1, HW_BREAKPOINT_RW, (__u64)&a, (__u64)sizeof(a)); 509 + if (ret) { 510 + perror("perf_systemwide_event_open"); 511 + exit(EXIT_FAILURE); 512 + } 513 + 514 + ret = perf_systemwide_event_open(fd2, HW_BREAKPOINT_RW, (__u64)&a, (__u64)sizeof(a)); 515 + if (ret) { 516 + close_fds(fd1, nprocs); 517 + perror("perf_systemwide_event_open"); 518 + exit(EXIT_FAILURE); 519 + } 520 + 521 + reset_fds(fd1, nprocs); 522 + reset_fds(fd2, nprocs); 523 + enable_fds(fd1, nprocs); 524 + enable_fds(fd2, nprocs); 525 + multi_dawr_workload(); 526 + disable_fds(fd1, nprocs); 527 + disable_fds(fd2, nprocs); 528 + 529 + breaks1 = read_fds(fd1, nprocs); 530 + breaks2 = read_fds(fd2, nprocs); 531 + 532 + close_fds(fd1, nprocs); 533 + close_fds(fd2, nprocs); 534 + 535 + free(fd1); 536 + free(fd2); 537 + 538 + if (breaks1 != 2 || breaks2 != 2) { 539 + printf("FAILED: %s: %lld != 2 || %lld != 2\n", desc, breaks1, breaks2); 540 + return 1; 541 + } 542 + 543 + printf("TESTED: %s\n", desc); 544 + return 0; 545 + } 546 + 547 + static int test_syswide_multi_diff_addr_ro_wo(void) 548 + { 549 + unsigned long long breaks1 = 0, breaks2 = 0; 550 + int *fd1 = malloc(nprocs * sizeof(int)); 551 + int *fd2 = malloc(nprocs * sizeof(int)); 552 + char *desc = "Systemwide, Two events, diff addr, one is RO, other is WO"; 553 + int ret; 554 + 555 + ret = perf_systemwide_event_open(fd1, HW_BREAKPOINT_W, (__u64)&a, (__u64)sizeof(a)); 556 + if (ret) { 557 + perror("perf_systemwide_event_open"); 558 + exit(EXIT_FAILURE); 559 + } 560 + 561 + ret = perf_systemwide_event_open(fd2, HW_BREAKPOINT_R, (__u64)&b, (__u64)sizeof(b)); 562 + if (ret) { 563 + close_fds(fd1, nprocs); 564 + perror("perf_systemwide_event_open"); 565 + exit(EXIT_FAILURE); 566 + } 567 + 568 + reset_fds(fd1, nprocs); 569 + reset_fds(fd2, nprocs); 570 + enable_fds(fd1, nprocs); 571 + enable_fds(fd2, nprocs); 572 + multi_dawr_workload(); 573 + disable_fds(fd1, nprocs); 574 + disable_fds(fd2, nprocs); 575 + 576 + breaks1 = read_fds(fd1, nprocs); 577 + breaks2 = read_fds(fd2, nprocs); 578 + 579 + close_fds(fd1, nprocs); 580 + close_fds(fd2, nprocs); 581 + 582 + free(fd1); 583 + free(fd2); 584 + 585 + if (breaks1 != 1 || breaks2 != 1) { 586 + printf("FAILED: %s: %lld != 1 || %lld != 1\n", desc, breaks1, breaks2); 587 + return 1; 588 + } 589 + 590 + printf("TESTED: %s\n", desc); 591 + return 0; 592 + } 593 + 594 + static int test_syswide_multi_same_addr_ro_wo(void) 595 + { 596 + unsigned long long breaks1 = 0, breaks2 = 0; 597 + int *fd1 = malloc(nprocs * sizeof(int)); 598 + int *fd2 = malloc(nprocs * sizeof(int)); 599 + char *desc = "Systemwide, Two events, same addr, one is RO, other is WO"; 600 + int ret; 601 + 602 + ret = perf_systemwide_event_open(fd1, HW_BREAKPOINT_W, (__u64)&a, (__u64)sizeof(a)); 603 + if (ret) { 604 + perror("perf_systemwide_event_open"); 605 + exit(EXIT_FAILURE); 606 + } 607 + 608 + ret = perf_systemwide_event_open(fd2, HW_BREAKPOINT_R, (__u64)&a, (__u64)sizeof(a)); 609 + if (ret) { 610 + close_fds(fd1, nprocs); 611 + perror("perf_systemwide_event_open"); 612 + exit(EXIT_FAILURE); 613 + } 614 + 615 + reset_fds(fd1, nprocs); 616 + reset_fds(fd2, nprocs); 617 + enable_fds(fd1, nprocs); 618 + enable_fds(fd2, nprocs); 619 + multi_dawr_workload(); 620 + disable_fds(fd1, nprocs); 621 + disable_fds(fd2, nprocs); 622 + 623 + breaks1 = read_fds(fd1, nprocs); 624 + breaks2 = read_fds(fd2, nprocs); 625 + 626 + close_fds(fd1, nprocs); 627 + close_fds(fd2, nprocs); 628 + 629 + free(fd1); 630 + free(fd2); 631 + 632 + if (breaks1 != 1 || breaks2 != 1) { 633 + printf("FAILED: %s: %lld != 1 || %lld != 1\n", desc, breaks1, breaks2); 634 + return 1; 635 + } 636 + 637 + printf("TESTED: %s\n", desc); 638 + return 0; 639 + } 640 + 641 + static int runtest_multi_dawr(void) 642 + { 643 + int ret = 0; 644 + 645 + ret |= test_process_multi_diff_addr(); 646 + ret |= test_process_multi_same_addr(); 647 + ret |= test_process_multi_diff_addr_ro_wo(); 648 + ret |= test_process_multi_same_addr_ro_wo(); 649 + ret |= test_syswide_multi_diff_addr(); 650 + ret |= test_syswide_multi_same_addr(); 651 + ret |= test_syswide_multi_diff_addr_ro_wo(); 652 + ret |= test_syswide_multi_same_addr_ro_wo(); 653 + 654 + return ret; 655 + } 656 + 657 + static int runtest_unaligned_512bytes(void) 658 + { 659 + unsigned long long breaks = 0; 660 + int fd; 661 + char *desc = "Process specific, 512 bytes, unaligned"; 662 + __u64 addr = (__u64)&c + 8; 663 + size_t res; 664 + 665 + fd = perf_process_event_open(HW_BREAKPOINT_RW, addr, 512); 666 + if (fd < 0) { 667 + perror("perf_process_event_open"); 668 + exit(EXIT_FAILURE); 669 + } 670 + 671 + ioctl(fd, PERF_EVENT_IOC_RESET); 672 + ioctl(fd, PERF_EVENT_IOC_ENABLE); 673 + multi_dawr_workload(); 674 + ioctl(fd, PERF_EVENT_IOC_DISABLE); 675 + 676 + res = read(fd, &breaks, sizeof(breaks)); 677 + assert(res == sizeof(unsigned long long)); 678 + 679 + close(fd); 680 + 681 + if (breaks != 2) { 682 + printf("FAILED: %s: %lld != 2\n", desc, breaks); 683 + return 1; 684 + } 685 + 686 + printf("TESTED: %s\n", desc); 687 + return 0; 688 + } 689 + 690 + /* There is no perf api to find number of available watchpoints. Use ptrace. */ 691 + static int get_nr_wps(bool *arch_31) 692 + { 693 + struct ppc_debug_info dbginfo; 694 + int child_pid; 695 + 696 + child_pid = fork(); 697 + if (!child_pid) { 698 + int ret = ptrace(PTRACE_TRACEME, 0, NULL, 0); 699 + if (ret) { 700 + perror("PTRACE_TRACEME failed\n"); 701 + exit(EXIT_FAILURE); 702 + } 703 + kill(getpid(), SIGUSR1); 704 + 705 + sleep(1); 706 + exit(EXIT_SUCCESS); 707 + } 708 + 709 + wait(NULL); 710 + if (ptrace(PPC_PTRACE_GETHWDBGINFO, child_pid, NULL, &dbginfo)) { 711 + perror("Can't get breakpoint info"); 712 + exit(EXIT_FAILURE); 713 + } 714 + 715 + *arch_31 = !!(dbginfo.features & PPC_DEBUG_FEATURE_DATA_BP_ARCH_31); 716 + return dbginfo.num_data_bps; 717 + } 718 + 350 719 static int runtest(void) 351 720 { 352 721 int rwflag; 353 722 int exclude_user; 354 723 int ret; 724 + bool dawr = dawr_supported(); 725 + bool arch_31 = false; 726 + int nr_wps = get_nr_wps(&arch_31); 355 727 356 728 /* 357 729 * perf defines rwflag as two bits read and write and at least ··· 820 280 return ret; 821 281 822 282 /* if we have the dawr, we can do an array test */ 823 - if (!dawr_supported()) 283 + if (!dawr) 824 284 continue; 825 285 ret = runtestsingle(rwflag, exclude_user, 1); 826 286 if (ret) ··· 829 289 } 830 290 831 291 ret = runtest_dar_outside(); 292 + if (ret) 293 + return ret; 294 + 295 + if (dawr && nr_wps > 1) { 296 + nprocs = get_nprocs(); 297 + ret = runtest_multi_dawr(); 298 + if (ret) 299 + return ret; 300 + } 301 + 302 + if (dawr && arch_31) 303 + ret = runtest_unaligned_512bytes(); 304 + 832 305 return ret; 833 306 } 834 307

+79

tools/testing/selftests/powerpc/ptrace/ptrace-hwbreak.c

··· 194 194 big_var[rand() % DAWR_MAX_LEN] = 'a'; 195 195 else 196 196 cvar = big_var[rand() % DAWR_MAX_LEN]; 197 + 198 + /* PPC_PTRACE_SETHWDEBUG 2, MODE_RANGE, DW ALIGNED, WO test */ 199 + gstruct.a[rand() % A_LEN] = 'a'; 200 + 201 + /* PPC_PTRACE_SETHWDEBUG 2, MODE_RANGE, DW UNALIGNED, RO test */ 202 + cvar = gstruct.b[rand() % B_LEN]; 203 + 204 + /* PPC_PTRACE_SETHWDEBUG 2, MODE_RANGE, DAWR Overlap, WO test */ 205 + gstruct.a[rand() % A_LEN] = 'a'; 206 + 207 + /* PPC_PTRACE_SETHWDEBUG 2, MODE_RANGE, DAWR Overlap, RO test */ 208 + cvar = gstruct.a[rand() % A_LEN]; 197 209 } 198 210 199 211 static void check_success(pid_t child_pid, const char *name, const char *type, ··· 429 417 ptrace_delhwdebug(child_pid, wh); 430 418 } 431 419 420 + static void test_multi_sethwdebug_range(pid_t child_pid) 421 + { 422 + struct ppc_hw_breakpoint info1, info2; 423 + unsigned long wp_addr1, wp_addr2; 424 + char *name1 = "PPC_PTRACE_SETHWDEBUG 2, MODE_RANGE, DW ALIGNED"; 425 + char *name2 = "PPC_PTRACE_SETHWDEBUG 2, MODE_RANGE, DW UNALIGNED"; 426 + int len1, len2; 427 + int wh1, wh2; 428 + 429 + wp_addr1 = (unsigned long)&gstruct.a; 430 + wp_addr2 = (unsigned long)&gstruct.b; 431 + len1 = A_LEN; 432 + len2 = B_LEN; 433 + get_ppc_hw_breakpoint(&info1, PPC_BREAKPOINT_TRIGGER_WRITE, wp_addr1, len1); 434 + get_ppc_hw_breakpoint(&info2, PPC_BREAKPOINT_TRIGGER_READ, wp_addr2, len2); 435 + 436 + /* PPC_PTRACE_SETHWDEBUG 2, MODE_RANGE, DW ALIGNED, WO test */ 437 + wh1 = ptrace_sethwdebug(child_pid, &info1); 438 + 439 + /* PPC_PTRACE_SETHWDEBUG 2, MODE_RANGE, DW UNALIGNED, RO test */ 440 + wh2 = ptrace_sethwdebug(child_pid, &info2); 441 + 442 + ptrace(PTRACE_CONT, child_pid, NULL, 0); 443 + check_success(child_pid, name1, "WO", wp_addr1, len1); 444 + 445 + ptrace(PTRACE_CONT, child_pid, NULL, 0); 446 + check_success(child_pid, name2, "RO", wp_addr2, len2); 447 + 448 + ptrace_delhwdebug(child_pid, wh1); 449 + ptrace_delhwdebug(child_pid, wh2); 450 + } 451 + 452 + static void test_multi_sethwdebug_range_dawr_overlap(pid_t child_pid) 453 + { 454 + struct ppc_hw_breakpoint info1, info2; 455 + unsigned long wp_addr1, wp_addr2; 456 + char *name = "PPC_PTRACE_SETHWDEBUG 2, MODE_RANGE, DAWR Overlap"; 457 + int len1, len2; 458 + int wh1, wh2; 459 + 460 + wp_addr1 = (unsigned long)&gstruct.a; 461 + wp_addr2 = (unsigned long)&gstruct.a; 462 + len1 = A_LEN; 463 + len2 = A_LEN; 464 + get_ppc_hw_breakpoint(&info1, PPC_BREAKPOINT_TRIGGER_WRITE, wp_addr1, len1); 465 + get_ppc_hw_breakpoint(&info2, PPC_BREAKPOINT_TRIGGER_READ, wp_addr2, len2); 466 + 467 + /* PPC_PTRACE_SETHWDEBUG 2, MODE_RANGE, DAWR Overlap, WO test */ 468 + wh1 = ptrace_sethwdebug(child_pid, &info1); 469 + 470 + /* PPC_PTRACE_SETHWDEBUG 2, MODE_RANGE, DAWR Overlap, RO test */ 471 + wh2 = ptrace_sethwdebug(child_pid, &info2); 472 + 473 + ptrace(PTRACE_CONT, child_pid, NULL, 0); 474 + check_success(child_pid, name, "WO", wp_addr1, len1); 475 + 476 + ptrace(PTRACE_CONT, child_pid, NULL, 0); 477 + check_success(child_pid, name, "RO", wp_addr2, len2); 478 + 479 + ptrace_delhwdebug(child_pid, wh1); 480 + ptrace_delhwdebug(child_pid, wh2); 481 + } 482 + 432 483 static void test_sethwdebug_range_unaligned(pid_t child_pid) 433 484 { 434 485 struct ppc_hw_breakpoint info; ··· 579 504 test_sethwdebug_range_unaligned(child_pid); 580 505 test_sethwdebug_range_unaligned_dar(child_pid); 581 506 test_sethwdebug_dawr_max_range(child_pid); 507 + if (dbginfo->num_data_bps > 1) { 508 + test_multi_sethwdebug_range(child_pid); 509 + test_multi_sethwdebug_range_dawr_overlap(child_pid); 510 + } 582 511 } 583 512 } 584 513 }

+659

tools/testing/selftests/powerpc/ptrace/ptrace-perf-hwbreak.c

··· 1 + // SPDX-License-Identifier: GPL-2.0+ 2 + #include <stdio.h> 3 + #include <string.h> 4 + #include <signal.h> 5 + #include <stdlib.h> 6 + #include <unistd.h> 7 + #include <errno.h> 8 + #include <linux/hw_breakpoint.h> 9 + #include <linux/perf_event.h> 10 + #include <asm/unistd.h> 11 + #include <sys/ptrace.h> 12 + #include <sys/wait.h> 13 + #include "ptrace.h" 14 + 15 + char data[16]; 16 + 17 + /* Overlapping address range */ 18 + volatile __u64 *ptrace_data1 = (__u64 *)&data[0]; 19 + volatile __u64 *perf_data1 = (__u64 *)&data[4]; 20 + 21 + /* Non-overlapping address range */ 22 + volatile __u64 *ptrace_data2 = (__u64 *)&data[0]; 23 + volatile __u64 *perf_data2 = (__u64 *)&data[8]; 24 + 25 + static unsigned long pid_max_addr(void) 26 + { 27 + FILE *fp; 28 + char *line, *c; 29 + char addr[100]; 30 + size_t len = 0; 31 + 32 + fp = fopen("/proc/kallsyms", "r"); 33 + if (!fp) { 34 + printf("Failed to read /proc/kallsyms. Exiting..\n"); 35 + exit(EXIT_FAILURE); 36 + } 37 + 38 + while (getline(&line, &len, fp) != -1) { 39 + if (!strstr(line, "pid_max") || strstr(line, "pid_max_max") || 40 + strstr(line, "pid_max_min")) 41 + continue; 42 + 43 + strncpy(addr, line, len < 100 ? len : 100); 44 + c = strchr(addr, ' '); 45 + *c = '\0'; 46 + return strtoul(addr, &c, 16); 47 + } 48 + fclose(fp); 49 + printf("Could not find pix_max. Exiting..\n"); 50 + exit(EXIT_FAILURE); 51 + return -1; 52 + } 53 + 54 + static void perf_user_event_attr_set(struct perf_event_attr *attr, __u64 addr, __u64 len) 55 + { 56 + memset(attr, 0, sizeof(struct perf_event_attr)); 57 + attr->type = PERF_TYPE_BREAKPOINT; 58 + attr->size = sizeof(struct perf_event_attr); 59 + attr->bp_type = HW_BREAKPOINT_R; 60 + attr->bp_addr = addr; 61 + attr->bp_len = len; 62 + attr->exclude_kernel = 1; 63 + attr->exclude_hv = 1; 64 + } 65 + 66 + static void perf_kernel_event_attr_set(struct perf_event_attr *attr) 67 + { 68 + memset(attr, 0, sizeof(struct perf_event_attr)); 69 + attr->type = PERF_TYPE_BREAKPOINT; 70 + attr->size = sizeof(struct perf_event_attr); 71 + attr->bp_type = HW_BREAKPOINT_R; 72 + attr->bp_addr = pid_max_addr(); 73 + attr->bp_len = sizeof(unsigned long); 74 + attr->exclude_user = 1; 75 + attr->exclude_hv = 1; 76 + } 77 + 78 + static int perf_cpu_event_open(int cpu, __u64 addr, __u64 len) 79 + { 80 + struct perf_event_attr attr; 81 + 82 + perf_user_event_attr_set(&attr, addr, len); 83 + return syscall(__NR_perf_event_open, &attr, -1, cpu, -1, 0); 84 + } 85 + 86 + static int perf_thread_event_open(pid_t child_pid, __u64 addr, __u64 len) 87 + { 88 + struct perf_event_attr attr; 89 + 90 + perf_user_event_attr_set(&attr, addr, len); 91 + return syscall(__NR_perf_event_open, &attr, child_pid, -1, -1, 0); 92 + } 93 + 94 + static int perf_thread_cpu_event_open(pid_t child_pid, int cpu, __u64 addr, __u64 len) 95 + { 96 + struct perf_event_attr attr; 97 + 98 + perf_user_event_attr_set(&attr, addr, len); 99 + return syscall(__NR_perf_event_open, &attr, child_pid, cpu, -1, 0); 100 + } 101 + 102 + static int perf_thread_kernel_event_open(pid_t child_pid) 103 + { 104 + struct perf_event_attr attr; 105 + 106 + perf_kernel_event_attr_set(&attr); 107 + return syscall(__NR_perf_event_open, &attr, child_pid, -1, -1, 0); 108 + } 109 + 110 + static int perf_cpu_kernel_event_open(int cpu) 111 + { 112 + struct perf_event_attr attr; 113 + 114 + perf_kernel_event_attr_set(&attr); 115 + return syscall(__NR_perf_event_open, &attr, -1, cpu, -1, 0); 116 + } 117 + 118 + static int child(void) 119 + { 120 + int ret; 121 + 122 + ret = ptrace(PTRACE_TRACEME, 0, NULL, 0); 123 + if (ret) { 124 + printf("Error: PTRACE_TRACEME failed\n"); 125 + return 0; 126 + } 127 + kill(getpid(), SIGUSR1); /* --> parent (SIGUSR1) */ 128 + 129 + return 0; 130 + } 131 + 132 + static void ptrace_ppc_hw_breakpoint(struct ppc_hw_breakpoint *info, int type, 133 + __u64 addr, int len) 134 + { 135 + info->version = 1; 136 + info->trigger_type = type; 137 + info->condition_mode = PPC_BREAKPOINT_CONDITION_NONE; 138 + info->addr = addr; 139 + info->addr2 = addr + len; 140 + info->condition_value = 0; 141 + if (!len) 142 + info->addr_mode = PPC_BREAKPOINT_MODE_EXACT; 143 + else 144 + info->addr_mode = PPC_BREAKPOINT_MODE_RANGE_INCLUSIVE; 145 + } 146 + 147 + static int ptrace_open(pid_t child_pid, __u64 wp_addr, int len) 148 + { 149 + struct ppc_hw_breakpoint info; 150 + 151 + ptrace_ppc_hw_breakpoint(&info, PPC_BREAKPOINT_TRIGGER_RW, wp_addr, len); 152 + return ptrace(PPC_PTRACE_SETHWDEBUG, child_pid, 0, &info); 153 + } 154 + 155 + static int test1(pid_t child_pid) 156 + { 157 + int perf_fd; 158 + int ptrace_fd; 159 + int ret = 0; 160 + 161 + /* Test: 162 + * if (new per thread event by ptrace) 163 + * if (existing cpu event by perf) 164 + * if (addr range overlaps) 165 + * fail; 166 + */ 167 + 168 + perf_fd = perf_cpu_event_open(0, (__u64)perf_data1, sizeof(*perf_data1)); 169 + if (perf_fd < 0) 170 + return -1; 171 + 172 + ptrace_fd = ptrace_open(child_pid, (__u64)ptrace_data1, sizeof(*ptrace_data1)); 173 + if (ptrace_fd > 0 || errno != ENOSPC) 174 + ret = -1; 175 + 176 + close(perf_fd); 177 + return ret; 178 + } 179 + 180 + static int test2(pid_t child_pid) 181 + { 182 + int perf_fd; 183 + int ptrace_fd; 184 + int ret = 0; 185 + 186 + /* Test: 187 + * if (new per thread event by ptrace) 188 + * if (existing cpu event by perf) 189 + * if (addr range does not overlaps) 190 + * allow; 191 + */ 192 + 193 + perf_fd = perf_cpu_event_open(0, (__u64)perf_data2, sizeof(*perf_data2)); 194 + if (perf_fd < 0) 195 + return -1; 196 + 197 + ptrace_fd = ptrace_open(child_pid, (__u64)ptrace_data2, sizeof(*ptrace_data2)); 198 + if (ptrace_fd < 0) { 199 + ret = -1; 200 + goto perf_close; 201 + } 202 + ptrace(PPC_PTRACE_DELHWDEBUG, child_pid, 0, ptrace_fd); 203 + 204 + perf_close: 205 + close(perf_fd); 206 + return ret; 207 + } 208 + 209 + static int test3(pid_t child_pid) 210 + { 211 + int perf_fd; 212 + int ptrace_fd; 213 + int ret = 0; 214 + 215 + /* Test: 216 + * if (new per thread event by ptrace) 217 + * if (existing thread event by perf on the same thread) 218 + * if (addr range overlaps) 219 + * fail; 220 + */ 221 + perf_fd = perf_thread_event_open(child_pid, (__u64)perf_data1, 222 + sizeof(*perf_data1)); 223 + if (perf_fd < 0) 224 + return -1; 225 + 226 + ptrace_fd = ptrace_open(child_pid, (__u64)ptrace_data1, sizeof(*ptrace_data1)); 227 + if (ptrace_fd > 0 || errno != ENOSPC) 228 + ret = -1; 229 + 230 + close(perf_fd); 231 + return ret; 232 + } 233 + 234 + static int test4(pid_t child_pid) 235 + { 236 + int perf_fd; 237 + int ptrace_fd; 238 + int ret = 0; 239 + 240 + /* Test: 241 + * if (new per thread event by ptrace) 242 + * if (existing thread event by perf on the same thread) 243 + * if (addr range does not overlaps) 244 + * fail; 245 + */ 246 + perf_fd = perf_thread_event_open(child_pid, (__u64)perf_data2, 247 + sizeof(*perf_data2)); 248 + if (perf_fd < 0) 249 + return -1; 250 + 251 + ptrace_fd = ptrace_open(child_pid, (__u64)ptrace_data2, sizeof(*ptrace_data2)); 252 + if (ptrace_fd < 0) { 253 + ret = -1; 254 + goto perf_close; 255 + } 256 + ptrace(PPC_PTRACE_DELHWDEBUG, child_pid, 0, ptrace_fd); 257 + 258 + perf_close: 259 + close(perf_fd); 260 + return ret; 261 + } 262 + 263 + static int test5(pid_t child_pid) 264 + { 265 + int perf_fd; 266 + int ptrace_fd; 267 + int cpid; 268 + int ret = 0; 269 + 270 + /* Test: 271 + * if (new per thread event by ptrace) 272 + * if (existing thread event by perf on the different thread) 273 + * allow; 274 + */ 275 + cpid = fork(); 276 + if (!cpid) { 277 + /* Temporary Child */ 278 + pause(); 279 + exit(EXIT_SUCCESS); 280 + } 281 + 282 + perf_fd = perf_thread_event_open(cpid, (__u64)perf_data1, sizeof(*perf_data1)); 283 + if (perf_fd < 0) { 284 + ret = -1; 285 + goto kill_child; 286 + } 287 + 288 + ptrace_fd = ptrace_open(child_pid, (__u64)ptrace_data1, sizeof(*ptrace_data1)); 289 + if (ptrace_fd < 0) { 290 + ret = -1; 291 + goto perf_close; 292 + } 293 + 294 + ptrace(PPC_PTRACE_DELHWDEBUG, child_pid, 0, ptrace_fd); 295 + perf_close: 296 + close(perf_fd); 297 + kill_child: 298 + kill(cpid, SIGINT); 299 + return ret; 300 + } 301 + 302 + static int test6(pid_t child_pid) 303 + { 304 + int perf_fd; 305 + int ptrace_fd; 306 + int ret = 0; 307 + 308 + /* Test: 309 + * if (new per thread kernel event by perf) 310 + * if (existing thread event by ptrace on the same thread) 311 + * allow; 312 + * -- OR -- 313 + * if (new per cpu kernel event by perf) 314 + * if (existing thread event by ptrace) 315 + * allow; 316 + */ 317 + ptrace_fd = ptrace_open(child_pid, (__u64)ptrace_data1, sizeof(*ptrace_data1)); 318 + if (ptrace_fd < 0) 319 + return -1; 320 + 321 + perf_fd = perf_thread_kernel_event_open(child_pid); 322 + if (perf_fd < 0) { 323 + ret = -1; 324 + goto ptrace_close; 325 + } 326 + close(perf_fd); 327 + 328 + perf_fd = perf_cpu_kernel_event_open(0); 329 + if (perf_fd < 0) { 330 + ret = -1; 331 + goto ptrace_close; 332 + } 333 + close(perf_fd); 334 + 335 + ptrace_close: 336 + ptrace(PPC_PTRACE_DELHWDEBUG, child_pid, 0, ptrace_fd); 337 + return ret; 338 + } 339 + 340 + static int test7(pid_t child_pid) 341 + { 342 + int perf_fd; 343 + int ptrace_fd; 344 + int ret = 0; 345 + 346 + /* Test: 347 + * if (new per thread event by perf) 348 + * if (existing thread event by ptrace on the same thread) 349 + * if (addr range overlaps) 350 + * fail; 351 + */ 352 + ptrace_fd = ptrace_open(child_pid, (__u64)ptrace_data1, sizeof(*ptrace_data1)); 353 + if (ptrace_fd < 0) 354 + return -1; 355 + 356 + perf_fd = perf_thread_event_open(child_pid, (__u64)perf_data1, 357 + sizeof(*perf_data1)); 358 + if (perf_fd > 0 || errno != ENOSPC) 359 + ret = -1; 360 + 361 + ptrace(PPC_PTRACE_DELHWDEBUG, child_pid, 0, ptrace_fd); 362 + return ret; 363 + } 364 + 365 + static int test8(pid_t child_pid) 366 + { 367 + int perf_fd; 368 + int ptrace_fd; 369 + int ret = 0; 370 + 371 + /* Test: 372 + * if (new per thread event by perf) 373 + * if (existing thread event by ptrace on the same thread) 374 + * if (addr range does not overlaps) 375 + * allow; 376 + */ 377 + ptrace_fd = ptrace_open(child_pid, (__u64)ptrace_data2, sizeof(*ptrace_data2)); 378 + if (ptrace_fd < 0) 379 + return -1; 380 + 381 + perf_fd = perf_thread_event_open(child_pid, (__u64)perf_data2, 382 + sizeof(*perf_data2)); 383 + if (perf_fd < 0) { 384 + ret = -1; 385 + goto ptrace_close; 386 + } 387 + close(perf_fd); 388 + 389 + ptrace_close: 390 + ptrace(PPC_PTRACE_DELHWDEBUG, child_pid, 0, ptrace_fd); 391 + return ret; 392 + } 393 + 394 + static int test9(pid_t child_pid) 395 + { 396 + int perf_fd; 397 + int ptrace_fd; 398 + int cpid; 399 + int ret = 0; 400 + 401 + /* Test: 402 + * if (new per thread event by perf) 403 + * if (existing thread event by ptrace on the other thread) 404 + * allow; 405 + */ 406 + ptrace_fd = ptrace_open(child_pid, (__u64)ptrace_data1, sizeof(*ptrace_data1)); 407 + if (ptrace_fd < 0) 408 + return -1; 409 + 410 + cpid = fork(); 411 + if (!cpid) { 412 + /* Temporary Child */ 413 + pause(); 414 + exit(EXIT_SUCCESS); 415 + } 416 + 417 + perf_fd = perf_thread_event_open(cpid, (__u64)perf_data1, sizeof(*perf_data1)); 418 + if (perf_fd < 0) { 419 + ret = -1; 420 + goto kill_child; 421 + } 422 + close(perf_fd); 423 + 424 + kill_child: 425 + kill(cpid, SIGINT); 426 + ptrace(PPC_PTRACE_DELHWDEBUG, child_pid, 0, ptrace_fd); 427 + return ret; 428 + } 429 + 430 + static int test10(pid_t child_pid) 431 + { 432 + int perf_fd; 433 + int ptrace_fd; 434 + int ret = 0; 435 + 436 + /* Test: 437 + * if (new per cpu event by perf) 438 + * if (existing thread event by ptrace on the same thread) 439 + * if (addr range overlaps) 440 + * fail; 441 + */ 442 + ptrace_fd = ptrace_open(child_pid, (__u64)ptrace_data1, sizeof(*ptrace_data1)); 443 + if (ptrace_fd < 0) 444 + return -1; 445 + 446 + perf_fd = perf_cpu_event_open(0, (__u64)perf_data1, sizeof(*perf_data1)); 447 + if (perf_fd > 0 || errno != ENOSPC) 448 + ret = -1; 449 + 450 + ptrace(PPC_PTRACE_DELHWDEBUG, child_pid, 0, ptrace_fd); 451 + return ret; 452 + } 453 + 454 + static int test11(pid_t child_pid) 455 + { 456 + int perf_fd; 457 + int ptrace_fd; 458 + int ret = 0; 459 + 460 + /* Test: 461 + * if (new per cpu event by perf) 462 + * if (existing thread event by ptrace on the same thread) 463 + * if (addr range does not overlap) 464 + * allow; 465 + */ 466 + ptrace_fd = ptrace_open(child_pid, (__u64)ptrace_data2, sizeof(*ptrace_data2)); 467 + if (ptrace_fd < 0) 468 + return -1; 469 + 470 + perf_fd = perf_cpu_event_open(0, (__u64)perf_data2, sizeof(*perf_data2)); 471 + if (perf_fd < 0) { 472 + ret = -1; 473 + goto ptrace_close; 474 + } 475 + close(perf_fd); 476 + 477 + ptrace_close: 478 + ptrace(PPC_PTRACE_DELHWDEBUG, child_pid, 0, ptrace_fd); 479 + return ret; 480 + } 481 + 482 + static int test12(pid_t child_pid) 483 + { 484 + int perf_fd; 485 + int ptrace_fd; 486 + int ret = 0; 487 + 488 + /* Test: 489 + * if (new per thread and per cpu event by perf) 490 + * if (existing thread event by ptrace on the same thread) 491 + * if (addr range overlaps) 492 + * fail; 493 + */ 494 + ptrace_fd = ptrace_open(child_pid, (__u64)ptrace_data1, sizeof(*ptrace_data1)); 495 + if (ptrace_fd < 0) 496 + return -1; 497 + 498 + perf_fd = perf_thread_cpu_event_open(child_pid, 0, (__u64)perf_data1, sizeof(*perf_data1)); 499 + if (perf_fd > 0 || errno != ENOSPC) 500 + ret = -1; 501 + 502 + ptrace(PPC_PTRACE_DELHWDEBUG, child_pid, 0, ptrace_fd); 503 + return ret; 504 + } 505 + 506 + static int test13(pid_t child_pid) 507 + { 508 + int perf_fd; 509 + int ptrace_fd; 510 + int ret = 0; 511 + 512 + /* Test: 513 + * if (new per thread and per cpu event by perf) 514 + * if (existing thread event by ptrace on the same thread) 515 + * if (addr range does not overlap) 516 + * allow; 517 + */ 518 + ptrace_fd = ptrace_open(child_pid, (__u64)ptrace_data2, sizeof(*ptrace_data2)); 519 + if (ptrace_fd < 0) 520 + return -1; 521 + 522 + perf_fd = perf_thread_cpu_event_open(child_pid, 0, (__u64)perf_data2, sizeof(*perf_data2)); 523 + if (perf_fd < 0) { 524 + ret = -1; 525 + goto ptrace_close; 526 + } 527 + close(perf_fd); 528 + 529 + ptrace_close: 530 + ptrace(PPC_PTRACE_DELHWDEBUG, child_pid, 0, ptrace_fd); 531 + return ret; 532 + } 533 + 534 + static int test14(pid_t child_pid) 535 + { 536 + int perf_fd; 537 + int ptrace_fd; 538 + int cpid; 539 + int ret = 0; 540 + 541 + /* Test: 542 + * if (new per thread and per cpu event by perf) 543 + * if (existing thread event by ptrace on the other thread) 544 + * allow; 545 + */ 546 + ptrace_fd = ptrace_open(child_pid, (__u64)ptrace_data1, sizeof(*ptrace_data1)); 547 + if (ptrace_fd < 0) 548 + return -1; 549 + 550 + cpid = fork(); 551 + if (!cpid) { 552 + /* Temporary Child */ 553 + pause(); 554 + exit(EXIT_SUCCESS); 555 + } 556 + 557 + perf_fd = perf_thread_cpu_event_open(cpid, 0, (__u64)perf_data1, 558 + sizeof(*perf_data1)); 559 + if (perf_fd < 0) { 560 + ret = -1; 561 + goto kill_child; 562 + } 563 + close(perf_fd); 564 + 565 + kill_child: 566 + kill(cpid, SIGINT); 567 + ptrace(PPC_PTRACE_DELHWDEBUG, child_pid, 0, ptrace_fd); 568 + return ret; 569 + } 570 + 571 + static int do_test(const char *msg, int (*fun)(pid_t arg), pid_t arg) 572 + { 573 + int ret; 574 + 575 + ret = fun(arg); 576 + if (ret) 577 + printf("%s: Error\n", msg); 578 + else 579 + printf("%s: Ok\n", msg); 580 + return ret; 581 + } 582 + 583 + char *desc[14] = { 584 + "perf cpu event -> ptrace thread event (Overlapping)", 585 + "perf cpu event -> ptrace thread event (Non-overlapping)", 586 + "perf thread event -> ptrace same thread event (Overlapping)", 587 + "perf thread event -> ptrace same thread event (Non-overlapping)", 588 + "perf thread event -> ptrace other thread event", 589 + "ptrace thread event -> perf kernel event", 590 + "ptrace thread event -> perf same thread event (Overlapping)", 591 + "ptrace thread event -> perf same thread event (Non-overlapping)", 592 + "ptrace thread event -> perf other thread event", 593 + "ptrace thread event -> perf cpu event (Overlapping)", 594 + "ptrace thread event -> perf cpu event (Non-overlapping)", 595 + "ptrace thread event -> perf same thread & cpu event (Overlapping)", 596 + "ptrace thread event -> perf same thread & cpu event (Non-overlapping)", 597 + "ptrace thread event -> perf other thread & cpu event", 598 + }; 599 + 600 + static int test(pid_t child_pid) 601 + { 602 + int ret = TEST_PASS; 603 + 604 + ret |= do_test(desc[0], test1, child_pid); 605 + ret |= do_test(desc[1], test2, child_pid); 606 + ret |= do_test(desc[2], test3, child_pid); 607 + ret |= do_test(desc[3], test4, child_pid); 608 + ret |= do_test(desc[4], test5, child_pid); 609 + ret |= do_test(desc[5], test6, child_pid); 610 + ret |= do_test(desc[6], test7, child_pid); 611 + ret |= do_test(desc[7], test8, child_pid); 612 + ret |= do_test(desc[8], test9, child_pid); 613 + ret |= do_test(desc[9], test10, child_pid); 614 + ret |= do_test(desc[10], test11, child_pid); 615 + ret |= do_test(desc[11], test12, child_pid); 616 + ret |= do_test(desc[12], test13, child_pid); 617 + ret |= do_test(desc[13], test14, child_pid); 618 + 619 + return ret; 620 + } 621 + 622 + static void get_dbginfo(pid_t child_pid, struct ppc_debug_info *dbginfo) 623 + { 624 + if (ptrace(PPC_PTRACE_GETHWDBGINFO, child_pid, NULL, dbginfo)) { 625 + perror("Can't get breakpoint info"); 626 + exit(-1); 627 + } 628 + } 629 + 630 + static int ptrace_perf_hwbreak(void) 631 + { 632 + int ret; 633 + pid_t child_pid; 634 + struct ppc_debug_info dbginfo; 635 + 636 + child_pid = fork(); 637 + if (!child_pid) 638 + return child(); 639 + 640 + /* parent */ 641 + wait(NULL); /* <-- child (SIGUSR1) */ 642 + 643 + get_dbginfo(child_pid, &dbginfo); 644 + SKIP_IF(dbginfo.num_data_bps <= 1); 645 + 646 + ret = perf_cpu_event_open(0, (__u64)perf_data1, sizeof(*perf_data1)); 647 + SKIP_IF(ret < 0); 648 + close(ret); 649 + 650 + ret = test(child_pid); 651 + 652 + ptrace(PTRACE_CONT, child_pid, NULL, 0); 653 + return ret; 654 + } 655 + 656 + int main(int argc, char *argv[]) 657 + { 658 + return test_harness(ptrace_perf_hwbreak, "ptrace-perf-hwbreak"); 659 + }

+2 -1

tools/testing/selftests/powerpc/security/Makefile

··· 1 1 # SPDX-License-Identifier: GPL-2.0+ 2 2 3 - TEST_GEN_PROGS := rfi_flush entry_flush spectre_v2 3 + TEST_GEN_PROGS := rfi_flush entry_flush uaccess_flush spectre_v2 4 4 top_srcdir = ../../../../.. 5 5 6 6 CFLAGS += -I../../../../../usr/include ··· 13 13 $(OUTPUT)/spectre_v2: ../pmu/event.c branch_loops.S 14 14 $(OUTPUT)/rfi_flush: flush_utils.c 15 15 $(OUTPUT)/entry_flush: flush_utils.c 16 + $(OUTPUT)/uaccess_flush: flush_utils.c

+1 -1

tools/testing/selftests/powerpc/security/entry_flush.c

··· 53 53 54 54 entry_flush = entry_flush_orig; 55 55 56 - fd = perf_event_open_counter(PERF_TYPE_RAW, /* L1d miss */ 0x400f0, -1); 56 + fd = perf_event_open_counter(PERF_TYPE_HW_CACHE, PERF_L1D_READ_MISS_CONFIG, -1); 57 57 FAIL_IF(fd < 0); 58 58 59 59 p = (char *)memalign(zero_size, CACHELINE_SIZE);

+13

tools/testing/selftests/powerpc/security/flush_utils.c

··· 13 13 #include <stdlib.h> 14 14 #include <string.h> 15 15 #include <stdio.h> 16 + #include <sys/utsname.h> 16 17 #include "utils.h" 17 18 #include "flush_utils.h" 18 19 ··· 33 32 for (unsigned long j = 0; j < zero_size; j += CACHELINE_SIZE) 34 33 load(p + j); 35 34 getppid(); 35 + } 36 + } 37 + 38 + void syscall_loop_uaccess(char *p, unsigned long iterations, 39 + unsigned long zero_size) 40 + { 41 + struct utsname utsname; 42 + 43 + for (unsigned long i = 0; i < iterations; i++) { 44 + for (unsigned long j = 0; j < zero_size; j += CACHELINE_SIZE) 45 + load(p + j); 46 + uname(&utsname); 36 47 } 37 48 } 38 49

+7

tools/testing/selftests/powerpc/security/flush_utils.h

··· 9 9 10 10 #define CACHELINE_SIZE 128 11 11 12 + #define PERF_L1D_READ_MISS_CONFIG ((PERF_COUNT_HW_CACHE_L1D) | \ 13 + (PERF_COUNT_HW_CACHE_OP_READ << 8) | \ 14 + (PERF_COUNT_HW_CACHE_RESULT_MISS << 16)) 15 + 12 16 void syscall_loop(char *p, unsigned long iterations, 13 17 unsigned long zero_size); 18 + 19 + void syscall_loop_uaccess(char *p, unsigned long iterations, 20 + unsigned long zero_size); 14 21 15 22 void set_dscr(unsigned long val); 16 23

+1 -1

tools/testing/selftests/powerpc/security/rfi_flush.c

··· 54 54 55 55 rfi_flush = rfi_flush_orig; 56 56 57 - fd = perf_event_open_counter(PERF_TYPE_RAW, /* L1d miss */ 0x400f0, -1); 57 + fd = perf_event_open_counter(PERF_TYPE_HW_CACHE, PERF_L1D_READ_MISS_CONFIG, -1); 58 58 FAIL_IF(fd < 0); 59 59 60 60 p = (char *)memalign(zero_size, CACHELINE_SIZE);

+158

tools/testing/selftests/powerpc/security/uaccess_flush.c

··· 1 + // SPDX-License-Identifier: GPL-2.0+ 2 + 3 + /* 4 + * Copyright 2018 IBM Corporation. 5 + * Copyright 2020 Canonical Ltd. 6 + */ 7 + 8 + #define __SANE_USERSPACE_TYPES__ 9 + 10 + #include <sys/types.h> 11 + #include <stdint.h> 12 + #include <malloc.h> 13 + #include <unistd.h> 14 + #include <signal.h> 15 + #include <stdlib.h> 16 + #include <string.h> 17 + #include <stdio.h> 18 + #include "utils.h" 19 + #include "flush_utils.h" 20 + 21 + int uaccess_flush_test(void) 22 + { 23 + char *p; 24 + int repetitions = 10; 25 + int fd, passes = 0, iter, rc = 0; 26 + struct perf_event_read v; 27 + __u64 l1d_misses_total = 0; 28 + unsigned long iterations = 100000, zero_size = 24 * 1024; 29 + unsigned long l1d_misses_expected; 30 + int rfi_flush_orig; 31 + int entry_flush_orig; 32 + int uaccess_flush, uaccess_flush_orig; 33 + 34 + SKIP_IF(geteuid() != 0); 35 + 36 + // The PMU event we use only works on Power7 or later 37 + SKIP_IF(!have_hwcap(PPC_FEATURE_ARCH_2_06)); 38 + 39 + if (read_debugfs_file("powerpc/rfi_flush", &rfi_flush_orig) < 0) { 40 + perror("Unable to read powerpc/rfi_flush debugfs file"); 41 + SKIP_IF(1); 42 + } 43 + 44 + if (read_debugfs_file("powerpc/entry_flush", &entry_flush_orig) < 0) { 45 + perror("Unable to read powerpc/entry_flush debugfs file"); 46 + SKIP_IF(1); 47 + } 48 + 49 + if (read_debugfs_file("powerpc/uaccess_flush", &uaccess_flush_orig) < 0) { 50 + perror("Unable to read powerpc/entry_flush debugfs file"); 51 + SKIP_IF(1); 52 + } 53 + 54 + if (rfi_flush_orig != 0) { 55 + if (write_debugfs_file("powerpc/rfi_flush", 0) < 0) { 56 + perror("error writing to powerpc/rfi_flush debugfs file"); 57 + FAIL_IF(1); 58 + } 59 + } 60 + 61 + if (entry_flush_orig != 0) { 62 + if (write_debugfs_file("powerpc/entry_flush", 0) < 0) { 63 + perror("error writing to powerpc/entry_flush debugfs file"); 64 + FAIL_IF(1); 65 + } 66 + } 67 + 68 + uaccess_flush = uaccess_flush_orig; 69 + 70 + fd = perf_event_open_counter(PERF_TYPE_HW_CACHE, PERF_L1D_READ_MISS_CONFIG, -1); 71 + FAIL_IF(fd < 0); 72 + 73 + p = (char *)memalign(zero_size, CACHELINE_SIZE); 74 + 75 + FAIL_IF(perf_event_enable(fd)); 76 + 77 + // disable L1 prefetching 78 + set_dscr(1); 79 + 80 + iter = repetitions; 81 + 82 + /* 83 + * We expect to see l1d miss for each cacheline access when entry_flush 84 + * is set. Allow a small variation on this. 85 + */ 86 + l1d_misses_expected = iterations * (zero_size / CACHELINE_SIZE - 2); 87 + 88 + again: 89 + FAIL_IF(perf_event_reset(fd)); 90 + 91 + syscall_loop_uaccess(p, iterations, zero_size); 92 + 93 + FAIL_IF(read(fd, &v, sizeof(v)) != sizeof(v)); 94 + 95 + if (uaccess_flush && v.l1d_misses >= l1d_misses_expected) 96 + passes++; 97 + else if (!uaccess_flush && v.l1d_misses < (l1d_misses_expected / 2)) 98 + passes++; 99 + 100 + l1d_misses_total += v.l1d_misses; 101 + 102 + while (--iter) 103 + goto again; 104 + 105 + if (passes < repetitions) { 106 + printf("FAIL (L1D misses with uaccess_flush=%d: %llu %c %lu) [%d/%d failures]\n", 107 + uaccess_flush, l1d_misses_total, uaccess_flush ? '<' : '>', 108 + uaccess_flush ? repetitions * l1d_misses_expected : 109 + repetitions * l1d_misses_expected / 2, 110 + repetitions - passes, repetitions); 111 + rc = 1; 112 + } else { 113 + printf("PASS (L1D misses with uaccess_flush=%d: %llu %c %lu) [%d/%d pass]\n", 114 + uaccess_flush, l1d_misses_total, uaccess_flush ? '>' : '<', 115 + uaccess_flush ? repetitions * l1d_misses_expected : 116 + repetitions * l1d_misses_expected / 2, 117 + passes, repetitions); 118 + } 119 + 120 + if (uaccess_flush == uaccess_flush_orig) { 121 + uaccess_flush = !uaccess_flush_orig; 122 + if (write_debugfs_file("powerpc/uaccess_flush", uaccess_flush) < 0) { 123 + perror("error writing to powerpc/uaccess_flush debugfs file"); 124 + return 1; 125 + } 126 + iter = repetitions; 127 + l1d_misses_total = 0; 128 + passes = 0; 129 + goto again; 130 + } 131 + 132 + perf_event_disable(fd); 133 + close(fd); 134 + 135 + set_dscr(0); 136 + 137 + if (write_debugfs_file("powerpc/rfi_flush", rfi_flush_orig) < 0) { 138 + perror("unable to restore original value of powerpc/rfi_flush debugfs file"); 139 + return 1; 140 + } 141 + 142 + if (write_debugfs_file("powerpc/entry_flush", entry_flush_orig) < 0) { 143 + perror("unable to restore original value of powerpc/entry_flush debugfs file"); 144 + return 1; 145 + } 146 + 147 + if (write_debugfs_file("powerpc/uaccess_flush", uaccess_flush_orig) < 0) { 148 + perror("unable to restore original value of powerpc/uaccess_flush debugfs file"); 149 + return 1; 150 + } 151 + 152 + return rc; 153 + } 154 + 155 + int main(int argc, char *argv[]) 156 + { 157 + return test_harness(uaccess_flush_test, "uaccess_flush_test"); 158 + }

+2 -2

tools/testing/selftests/powerpc/tm/tm-trap.c

··· 66 66 /* Get thread endianness: extract bit LE from MSR */ 67 67 thread_endianness = MSR_LE & ucp->uc_mcontext.gp_regs[PT_MSR]; 68 68 69 - /*** 69 + /* 70 70 * Little-Endian Machine 71 71 */ 72 72 ··· 126 126 } 127 127 } 128 128 129 - /*** 129 + /* 130 130 * Big-Endian Machine 131 131 */ 132 132

+8

tools/testing/selftests/timens/gettime_perf.c

··· 25 25 if (!vdso) 26 26 vdso = dlopen("linux-gate.so.1", 27 27 RTLD_LAZY | RTLD_LOCAL | RTLD_NOLOAD); 28 + if (!vdso) 29 + vdso = dlopen("linux-vdso32.so.1", 30 + RTLD_LAZY | RTLD_LOCAL | RTLD_NOLOAD); 31 + if (!vdso) 32 + vdso = dlopen("linux-vdso64.so.1", 33 + RTLD_LAZY | RTLD_LOCAL | RTLD_NOLOAD); 28 34 if (!vdso) { 29 35 pr_err("[WARN]\tfailed to find vDSO\n"); 30 36 return; 31 37 } 32 38 33 39 vdso_clock_gettime = (vgettime_t)dlsym(vdso, "__vdso_clock_gettime"); 40 + if (!vdso_clock_gettime) 41 + vdso_clock_gettime = (vgettime_t)dlsym(vdso, "__kernel_clock_gettime"); 34 42 if (!vdso_clock_gettime) 35 43 pr_err("Warning: failed to find clock_gettime in vDSO\n"); 36 44

Configure Feed

Configure Feed