Merge tag 's390-6.7-1' of git://git.kernel.org/pub/scm/linux/kernel/git/s390/linux

+1

arch/s390/Kconfig

··· 131 131 select ARCH_WANT_OPTIMIZE_HUGETLB_VMEMMAP 132 132 select BUILDTIME_TABLE_SORT 133 133 select CLONE_BACKWARDS2 134 + select DCACHE_WORD_ACCESS if !KMSAN 134 135 select DMA_OPS if PCI 135 136 select DYNAMIC_FTRACE if FUNCTION_TRACER 136 137 select FUNCTION_ALIGNMENT_8B if CC_IS_GCC

+1 -1

arch/s390/boot/startup.c

··· 49 49 { 50 50 if (test_facility(8)) { 51 51 machine.has_edat1 = 1; 52 - __ctl_set_bit(0, 23); 52 + local_ctl_set_bit(0, CR0_EDAT_BIT); 53 53 } 54 54 if (test_facility(78)) 55 55 machine.has_edat2 = 1;

+8 -9

arch/s390/boot/vmem.c

··· 5 5 #include <asm/pgalloc.h> 6 6 #include <asm/facility.h> 7 7 #include <asm/sections.h> 8 + #include <asm/ctlreg.h> 8 9 #include <asm/physmem_info.h> 9 10 #include <asm/maccess.h> 10 11 #include <asm/abs_lowcore.h> 11 12 #include "decompressor.h" 12 13 #include "boot.h" 13 14 14 - unsigned long __bootdata_preserved(s390_invalid_asce); 15 + struct ctlreg __bootdata_preserved(s390_invalid_asce); 15 16 16 17 #ifdef CONFIG_PROC_FS 17 18 atomic_long_t __bootdata_preserved(direct_pages_count[PG_DIRECT_MAP_MAX]); ··· 167 166 168 167 static bool kasan_pte_populate_zero_shadow(pte_t *pte, enum populate_mode mode) 169 168 { 170 - pte_t entry; 171 - 172 169 if (mode == POPULATE_KASAN_ZERO_SHADOW) { 173 170 set_pte(pte, pte_z); 174 171 return true; ··· 425 426 asce_type = _REGION3_ENTRY_EMPTY; 426 427 asce_bits = _ASCE_TYPE_REGION3 | _ASCE_TABLE_LENGTH; 427 428 } 428 - s390_invalid_asce = invalid_pg_dir | _ASCE_TYPE_REGION3 | _ASCE_TABLE_LENGTH; 429 + s390_invalid_asce.val = invalid_pg_dir | _ASCE_TYPE_REGION3 | _ASCE_TABLE_LENGTH; 429 430 430 431 crst_table_init((unsigned long *)swapper_pg_dir, asce_type); 431 432 crst_table_init((unsigned long *)invalid_pg_dir, _REGION3_ENTRY_EMPTY); ··· 446 447 447 448 kasan_populate_shadow(); 448 449 449 - S390_lowcore.kernel_asce = swapper_pg_dir | asce_bits; 450 + S390_lowcore.kernel_asce.val = swapper_pg_dir | asce_bits; 450 451 S390_lowcore.user_asce = s390_invalid_asce; 451 452 452 - __ctl_load(S390_lowcore.kernel_asce, 1, 1); 453 - __ctl_load(S390_lowcore.user_asce, 7, 7); 454 - __ctl_load(S390_lowcore.kernel_asce, 13, 13); 453 + local_ctl_load(1, &S390_lowcore.kernel_asce); 454 + local_ctl_load(7, &S390_lowcore.user_asce); 455 + local_ctl_load(13, &S390_lowcore.kernel_asce); 455 456 456 - init_mm.context.asce = S390_lowcore.kernel_asce; 457 + init_mm.context.asce = S390_lowcore.kernel_asce.val; 457 458 }

+11 -16

arch/s390/include/asm/asm-extable.h

··· 13 13 #define EX_TYPE_UA_LOAD_MEM 4 14 14 #define EX_TYPE_UA_LOAD_REG 5 15 15 #define EX_TYPE_UA_LOAD_REGPAIR 6 16 + #define EX_TYPE_ZEROPAD 7 16 17 17 18 #define EX_DATA_REG_ERR_SHIFT 0 18 19 #define EX_DATA_REG_ERR GENMASK(3, 0) ··· 24 23 #define EX_DATA_LEN_SHIFT 8 25 24 #define EX_DATA_LEN GENMASK(11, 8) 26 25 27 - #define __EX_TABLE(_section, _fault, _target, _type) \ 28 - stringify_in_c(.section _section,"a";) \ 29 - stringify_in_c(.balign 4;) \ 30 - stringify_in_c(.long (_fault) - .;) \ 31 - stringify_in_c(.long (_target) - .;) \ 32 - stringify_in_c(.short (_type);) \ 33 - stringify_in_c(.short 0;) \ 34 - stringify_in_c(.previous) 35 - 36 - #define __EX_TABLE_UA(_section, _fault, _target, _type, _regerr, _regaddr, _len)\ 26 + #define __EX_TABLE(_section, _fault, _target, _type, _regerr, _regaddr, _len) \ 37 27 stringify_in_c(.section _section,"a";) \ 38 28 stringify_in_c(.balign 4;) \ 39 29 stringify_in_c(.long (_fault) - .;) \ ··· 64 72 stringify_in_c(.previous) 65 73 66 74 #define EX_TABLE(_fault, _target) \ 67 - __EX_TABLE(__ex_table, _fault, _target, EX_TYPE_FIXUP) 75 + __EX_TABLE(__ex_table, _fault, _target, EX_TYPE_FIXUP, __stringify(%%r0), __stringify(%%r0), 0) 68 76 69 77 #define EX_TABLE_AMODE31(_fault, _target) \ 70 - __EX_TABLE(.amode31.ex_table, _fault, _target, EX_TYPE_FIXUP) 78 + __EX_TABLE(.amode31.ex_table, _fault, _target, EX_TYPE_FIXUP, __stringify(%%r0), __stringify(%%r0), 0) 71 79 72 80 #define EX_TABLE_UA_STORE(_fault, _target, _regerr) \ 73 - __EX_TABLE_UA(__ex_table, _fault, _target, EX_TYPE_UA_STORE, _regerr, _regerr, 0) 81 + __EX_TABLE(__ex_table, _fault, _target, EX_TYPE_UA_STORE, _regerr, _regerr, 0) 74 82 75 83 #define EX_TABLE_UA_LOAD_MEM(_fault, _target, _regerr, _regmem, _len) \ 76 - __EX_TABLE_UA(__ex_table, _fault, _target, EX_TYPE_UA_LOAD_MEM, _regerr, _regmem, _len) 84 + __EX_TABLE(__ex_table, _fault, _target, EX_TYPE_UA_LOAD_MEM, _regerr, _regmem, _len) 77 85 78 86 #define EX_TABLE_UA_LOAD_REG(_fault, _target, _regerr, _regzero) \ 79 - __EX_TABLE_UA(__ex_table, _fault, _target, EX_TYPE_UA_LOAD_REG, _regerr, _regzero, 0) 87 + __EX_TABLE(__ex_table, _fault, _target, EX_TYPE_UA_LOAD_REG, _regerr, _regzero, 0) 80 88 81 89 #define EX_TABLE_UA_LOAD_REGPAIR(_fault, _target, _regerr, _regzero) \ 82 - __EX_TABLE_UA(__ex_table, _fault, _target, EX_TYPE_UA_LOAD_REGPAIR, _regerr, _regzero, 0) 90 + __EX_TABLE(__ex_table, _fault, _target, EX_TYPE_UA_LOAD_REGPAIR, _regerr, _regzero, 0) 91 + 92 + #define EX_TABLE_ZEROPAD(_fault, _target, _regdata, _regaddr) \ 93 + __EX_TABLE(__ex_table, _fault, _target, EX_TYPE_ZEROPAD, _regdata, _regaddr, 0) 83 94 84 95 #endif /* __ASM_EXTABLE_H */

-146

arch/s390/include/asm/ctl_reg.h

··· 1 - /* SPDX-License-Identifier: GPL-2.0 */ 2 - /* 3 - * Copyright IBM Corp. 1999, 2009 4 - * 5 - * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com> 6 - */ 7 - 8 - #ifndef __ASM_CTL_REG_H 9 - #define __ASM_CTL_REG_H 10 - 11 - #include <linux/bits.h> 12 - 13 - #define CR0_CLOCK_COMPARATOR_SIGN BIT(63 - 10) 14 - #define CR0_LOW_ADDRESS_PROTECTION BIT(63 - 35) 15 - #define CR0_FETCH_PROTECTION_OVERRIDE BIT(63 - 38) 16 - #define CR0_STORAGE_PROTECTION_OVERRIDE BIT(63 - 39) 17 - #define CR0_EMERGENCY_SIGNAL_SUBMASK BIT(63 - 49) 18 - #define CR0_EXTERNAL_CALL_SUBMASK BIT(63 - 50) 19 - #define CR0_CLOCK_COMPARATOR_SUBMASK BIT(63 - 52) 20 - #define CR0_CPU_TIMER_SUBMASK BIT(63 - 53) 21 - #define CR0_SERVICE_SIGNAL_SUBMASK BIT(63 - 54) 22 - #define CR0_UNUSED_56 BIT(63 - 56) 23 - #define CR0_INTERRUPT_KEY_SUBMASK BIT(63 - 57) 24 - #define CR0_MEASUREMENT_ALERT_SUBMASK BIT(63 - 58) 25 - 26 - #define CR14_UNUSED_32 BIT(63 - 32) 27 - #define CR14_UNUSED_33 BIT(63 - 33) 28 - #define CR14_CHANNEL_REPORT_SUBMASK BIT(63 - 35) 29 - #define CR14_RECOVERY_SUBMASK BIT(63 - 36) 30 - #define CR14_DEGRADATION_SUBMASK BIT(63 - 37) 31 - #define CR14_EXTERNAL_DAMAGE_SUBMASK BIT(63 - 38) 32 - #define CR14_WARNING_SUBMASK BIT(63 - 39) 33 - 34 - #ifndef __ASSEMBLY__ 35 - 36 - #include <linux/bug.h> 37 - 38 - #define __ctl_load(array, low, high) do { \ 39 - typedef struct { char _[sizeof(array)]; } addrtype; \ 40 - \ 41 - BUILD_BUG_ON(sizeof(addrtype) != (high - low + 1) * sizeof(long));\ 42 - asm volatile( \ 43 - " lctlg %1,%2,%0\n" \ 44 - : \ 45 - : "Q" (*(addrtype *)(&array)), "i" (low), "i" (high) \ 46 - : "memory"); \ 47 - } while (0) 48 - 49 - #define __ctl_store(array, low, high) do { \ 50 - typedef struct { char _[sizeof(array)]; } addrtype; \ 51 - \ 52 - BUILD_BUG_ON(sizeof(addrtype) != (high - low + 1) * sizeof(long));\ 53 - asm volatile( \ 54 - " stctg %1,%2,%0\n" \ 55 - : "=Q" (*(addrtype *)(&array)) \ 56 - : "i" (low), "i" (high)); \ 57 - } while (0) 58 - 59 - static __always_inline void __ctl_set_bit(unsigned int cr, unsigned int bit) 60 - { 61 - unsigned long reg; 62 - 63 - __ctl_store(reg, cr, cr); 64 - reg |= 1UL << bit; 65 - __ctl_load(reg, cr, cr); 66 - } 67 - 68 - static __always_inline void __ctl_clear_bit(unsigned int cr, unsigned int bit) 69 - { 70 - unsigned long reg; 71 - 72 - __ctl_store(reg, cr, cr); 73 - reg &= ~(1UL << bit); 74 - __ctl_load(reg, cr, cr); 75 - } 76 - 77 - void smp_ctl_set_clear_bit(int cr, int bit, bool set); 78 - 79 - static inline void ctl_set_bit(int cr, int bit) 80 - { 81 - smp_ctl_set_clear_bit(cr, bit, true); 82 - } 83 - 84 - static inline void ctl_clear_bit(int cr, int bit) 85 - { 86 - smp_ctl_set_clear_bit(cr, bit, false); 87 - } 88 - 89 - union ctlreg0 { 90 - unsigned long val; 91 - struct { 92 - unsigned long : 8; 93 - unsigned long tcx : 1; /* Transactional-Execution control */ 94 - unsigned long pifo : 1; /* Transactional-Execution Program- 95 - Interruption-Filtering Override */ 96 - unsigned long : 3; 97 - unsigned long ccc : 1; /* Cryptography counter control */ 98 - unsigned long pec : 1; /* PAI extension control */ 99 - unsigned long : 17; 100 - unsigned long : 3; 101 - unsigned long lap : 1; /* Low-address-protection control */ 102 - unsigned long : 4; 103 - unsigned long edat : 1; /* Enhanced-DAT-enablement control */ 104 - unsigned long : 2; 105 - unsigned long iep : 1; /* Instruction-Execution-Protection */ 106 - unsigned long : 1; 107 - unsigned long afp : 1; /* AFP-register control */ 108 - unsigned long vx : 1; /* Vector enablement control */ 109 - unsigned long : 7; 110 - unsigned long sssm : 1; /* Service signal subclass mask */ 111 - unsigned long : 9; 112 - }; 113 - }; 114 - 115 - union ctlreg2 { 116 - unsigned long val; 117 - struct { 118 - unsigned long : 33; 119 - unsigned long ducto : 25; 120 - unsigned long : 1; 121 - unsigned long gse : 1; 122 - unsigned long : 1; 123 - unsigned long tds : 1; 124 - unsigned long tdc : 2; 125 - }; 126 - }; 127 - 128 - union ctlreg5 { 129 - unsigned long val; 130 - struct { 131 - unsigned long : 33; 132 - unsigned long pasteo: 25; 133 - unsigned long : 6; 134 - }; 135 - }; 136 - 137 - union ctlreg15 { 138 - unsigned long val; 139 - struct { 140 - unsigned long lsea : 61; 141 - unsigned long : 3; 142 - }; 143 - }; 144 - 145 - #endif /* __ASSEMBLY__ */ 146 - #endif /* __ASM_CTL_REG_H */

+251

arch/s390/include/asm/ctlreg.h

··· 1 + /* SPDX-License-Identifier: GPL-2.0 */ 2 + /* 3 + * Copyright IBM Corp. 1999, 2009 4 + * 5 + * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com> 6 + */ 7 + 8 + #ifndef __ASM_S390_CTLREG_H 9 + #define __ASM_S390_CTLREG_H 10 + 11 + #include <linux/bits.h> 12 + 13 + #define CR0_TRANSACTIONAL_EXECUTION_BIT (63 - 8) 14 + #define CR0_CLOCK_COMPARATOR_SIGN_BIT (63 - 10) 15 + #define CR0_CRYPTOGRAPHY_COUNTER_BIT (63 - 13) 16 + #define CR0_PAI_EXTENSION_BIT (63 - 14) 17 + #define CR0_CPUMF_EXTRACTION_AUTH_BIT (63 - 15) 18 + #define CR0_WARNING_TRACK_BIT (63 - 30) 19 + #define CR0_LOW_ADDRESS_PROTECTION_BIT (63 - 35) 20 + #define CR0_FETCH_PROTECTION_OVERRIDE_BIT (63 - 38) 21 + #define CR0_STORAGE_PROTECTION_OVERRIDE_BIT (63 - 39) 22 + #define CR0_EDAT_BIT (63 - 40) 23 + #define CR0_INSTRUCTION_EXEC_PROTECTION_BIT (63 - 43) 24 + #define CR0_VECTOR_BIT (63 - 46) 25 + #define CR0_MALFUNCTION_ALERT_SUBMASK_BIT (63 - 48) 26 + #define CR0_EMERGENCY_SIGNAL_SUBMASK_BIT (63 - 49) 27 + #define CR0_EXTERNAL_CALL_SUBMASK_BIT (63 - 50) 28 + #define CR0_CLOCK_COMPARATOR_SUBMASK_BIT (63 - 52) 29 + #define CR0_CPU_TIMER_SUBMASK_BIT (63 - 53) 30 + #define CR0_SERVICE_SIGNAL_SUBMASK_BIT (63 - 54) 31 + #define CR0_UNUSED_56_BIT (63 - 56) 32 + #define CR0_INTERRUPT_KEY_SUBMASK_BIT (63 - 57) 33 + #define CR0_MEASUREMENT_ALERT_SUBMASK_BIT (63 - 58) 34 + #define CR0_ETR_SUBMASK_BIT (63 - 59) 35 + #define CR0_IUCV_BIT (63 - 62) 36 + 37 + #define CR0_TRANSACTIONAL_EXECUTION BIT(CR0_TRANSACTIONAL_EXECUTION_BIT) 38 + #define CR0_CLOCK_COMPARATOR_SIGN BIT(CR0_CLOCK_COMPARATOR_SIGN_BIT) 39 + #define CR0_CRYPTOGRAPHY_COUNTER BIT(CR0_CRYPTOGRAPHY_COUNTER_BIT) 40 + #define CR0_PAI_EXTENSION BIT(CR0_PAI_EXTENSION_BIT) 41 + #define CR0_CPUMF_EXTRACTION_AUTH BIT(CR0_CPUMF_EXTRACTION_AUTH_BIT) 42 + #define CR0_WARNING_TRACK BIT(CR0_WARNING_TRACK_BIT) 43 + #define CR0_LOW_ADDRESS_PROTECTION BIT(CR0_LOW_ADDRESS_PROTECTION_BIT) 44 + #define CR0_FETCH_PROTECTION_OVERRIDE BIT(CR0_FETCH_PROTECTION_OVERRIDE_BIT) 45 + #define CR0_STORAGE_PROTECTION_OVERRIDE BIT(CR0_STORAGE_PROTECTION_OVERRIDE_BIT) 46 + #define CR0_EDAT BIT(CR0_EDAT_BIT) 47 + #define CR0_INSTRUCTION_EXEC_PROTECTION BIT(CR0_INSTRUCTION_EXEC_PROTECTION_BIT) 48 + #define CR0_VECTOR BIT(CR0_VECTOR_BIT) 49 + #define CR0_MALFUNCTION_ALERT_SUBMASK BIT(CR0_MALFUNCTION_ALERT_SUBMASK_BIT) 50 + #define CR0_EMERGENCY_SIGNAL_SUBMASK BIT(CR0_EMERGENCY_SIGNAL_SUBMASK_BIT) 51 + #define CR0_EXTERNAL_CALL_SUBMASK BIT(CR0_EXTERNAL_CALL_SUBMASK_BIT) 52 + #define CR0_CLOCK_COMPARATOR_SUBMASK BIT(CR0_CLOCK_COMPARATOR_SUBMASK_BIT) 53 + #define CR0_CPU_TIMER_SUBMASK BIT(CR0_CPU_TIMER_SUBMASK_BIT) 54 + #define CR0_SERVICE_SIGNAL_SUBMASK BIT(CR0_SERVICE_SIGNAL_SUBMASK_BIT) 55 + #define CR0_UNUSED_56 BIT(CR0_UNUSED_56_BIT) 56 + #define CR0_INTERRUPT_KEY_SUBMASK BIT(CR0_INTERRUPT_KEY_SUBMASK_BIT) 57 + #define CR0_MEASUREMENT_ALERT_SUBMASK BIT(CR0_MEASUREMENT_ALERT_SUBMASK_BIT) 58 + #define CR0_ETR_SUBMASK BIT(CR0_ETR_SUBMASK_BIT) 59 + #define CR0_IUCV BIT(CR0_IUCV_BIT) 60 + 61 + #define CR2_MIO_ADDRESSING_BIT (63 - 58) 62 + #define CR2_GUARDED_STORAGE_BIT (63 - 59) 63 + 64 + #define CR2_MIO_ADDRESSING BIT(CR2_MIO_ADDRESSING_BIT) 65 + #define CR2_GUARDED_STORAGE BIT(CR2_GUARDED_STORAGE_BIT) 66 + 67 + #define CR14_UNUSED_32_BIT (63 - 32) 68 + #define CR14_UNUSED_33_BIT (63 - 33) 69 + #define CR14_CHANNEL_REPORT_SUBMASK_BIT (63 - 35) 70 + #define CR14_RECOVERY_SUBMASK_BIT (63 - 36) 71 + #define CR14_DEGRADATION_SUBMASK_BIT (63 - 37) 72 + #define CR14_EXTERNAL_DAMAGE_SUBMASK_BIT (63 - 38) 73 + #define CR14_WARNING_SUBMASK_BIT (63 - 39) 74 + 75 + #define CR14_UNUSED_32 BIT(CR14_UNUSED_32_BIT) 76 + #define CR14_UNUSED_33 BIT(CR14_UNUSED_33_BIT) 77 + #define CR14_CHANNEL_REPORT_SUBMASK BIT(CR14_CHANNEL_REPORT_SUBMASK_BIT) 78 + #define CR14_RECOVERY_SUBMASK BIT(CR14_RECOVERY_SUBMASK_BIT) 79 + #define CR14_DEGRADATION_SUBMASK BIT(CR14_DEGRADATION_SUBMASK_BIT) 80 + #define CR14_EXTERNAL_DAMAGE_SUBMASK BIT(CR14_EXTERNAL_DAMAGE_SUBMASK_BIT) 81 + #define CR14_WARNING_SUBMASK BIT(CR14_WARNING_SUBMASK_BIT) 82 + 83 + #ifndef __ASSEMBLY__ 84 + 85 + #include <linux/bug.h> 86 + 87 + struct ctlreg { 88 + unsigned long val; 89 + }; 90 + 91 + #define __local_ctl_load(low, high, array) do { \ 92 + struct addrtype { \ 93 + char _[sizeof(array)]; \ 94 + }; \ 95 + int _high = high; \ 96 + int _low = low; \ 97 + int _esize; \ 98 + \ 99 + _esize = (_high - _low + 1) * sizeof(struct ctlreg); \ 100 + BUILD_BUG_ON(sizeof(struct addrtype) != _esize); \ 101 + typecheck(struct ctlreg, array[0]); \ 102 + asm volatile( \ 103 + " lctlg %[_low],%[_high],%[_arr]\n" \ 104 + : \ 105 + : [_arr] "Q" (*(struct addrtype *)(&array)), \ 106 + [_low] "i" (low), [_high] "i" (high) \ 107 + : "memory"); \ 108 + } while (0) 109 + 110 + #define __local_ctl_store(low, high, array) do { \ 111 + struct addrtype { \ 112 + char _[sizeof(array)]; \ 113 + }; \ 114 + int _high = high; \ 115 + int _low = low; \ 116 + int _esize; \ 117 + \ 118 + _esize = (_high - _low + 1) * sizeof(struct ctlreg); \ 119 + BUILD_BUG_ON(sizeof(struct addrtype) != _esize); \ 120 + typecheck(struct ctlreg, array[0]); \ 121 + asm volatile( \ 122 + " stctg %[_low],%[_high],%[_arr]\n" \ 123 + : [_arr] "=Q" (*(struct addrtype *)(&array)) \ 124 + : [_low] "i" (low), [_high] "i" (high)); \ 125 + } while (0) 126 + 127 + static __always_inline void local_ctl_load(unsigned int cr, struct ctlreg *reg) 128 + { 129 + asm volatile( 130 + " lctlg %[cr],%[cr],%[reg]\n" 131 + : 132 + : [reg] "Q" (*reg), [cr] "i" (cr) 133 + : "memory"); 134 + } 135 + 136 + static __always_inline void local_ctl_store(unsigned int cr, struct ctlreg *reg) 137 + { 138 + asm volatile( 139 + " stctg %[cr],%[cr],%[reg]\n" 140 + : [reg] "=Q" (*reg) 141 + : [cr] "i" (cr)); 142 + } 143 + 144 + static __always_inline void local_ctl_set_bit(unsigned int cr, unsigned int bit) 145 + { 146 + struct ctlreg reg; 147 + 148 + local_ctl_store(cr, &reg); 149 + reg.val |= 1UL << bit; 150 + local_ctl_load(cr, &reg); 151 + } 152 + 153 + static __always_inline void local_ctl_clear_bit(unsigned int cr, unsigned int bit) 154 + { 155 + struct ctlreg reg; 156 + 157 + local_ctl_store(cr, &reg); 158 + reg.val &= ~(1UL << bit); 159 + local_ctl_load(cr, &reg); 160 + } 161 + 162 + struct lowcore; 163 + 164 + void system_ctlreg_lock(void); 165 + void system_ctlreg_unlock(void); 166 + void system_ctlreg_init_save_area(struct lowcore *lc); 167 + void system_ctlreg_modify(unsigned int cr, unsigned long data, int request); 168 + 169 + enum { 170 + CTLREG_SET_BIT, 171 + CTLREG_CLEAR_BIT, 172 + CTLREG_LOAD, 173 + }; 174 + 175 + static inline void system_ctl_set_bit(unsigned int cr, unsigned int bit) 176 + { 177 + system_ctlreg_modify(cr, bit, CTLREG_SET_BIT); 178 + } 179 + 180 + static inline void system_ctl_clear_bit(unsigned int cr, unsigned int bit) 181 + { 182 + system_ctlreg_modify(cr, bit, CTLREG_CLEAR_BIT); 183 + } 184 + 185 + static inline void system_ctl_load(unsigned int cr, struct ctlreg *reg) 186 + { 187 + system_ctlreg_modify(cr, reg->val, CTLREG_LOAD); 188 + } 189 + 190 + union ctlreg0 { 191 + unsigned long val; 192 + struct ctlreg reg; 193 + struct { 194 + unsigned long : 8; 195 + unsigned long tcx : 1; /* Transactional-Execution control */ 196 + unsigned long pifo : 1; /* Transactional-Execution Program- 197 + Interruption-Filtering Override */ 198 + unsigned long : 3; 199 + unsigned long ccc : 1; /* Cryptography counter control */ 200 + unsigned long pec : 1; /* PAI extension control */ 201 + unsigned long : 17; 202 + unsigned long : 3; 203 + unsigned long lap : 1; /* Low-address-protection control */ 204 + unsigned long : 4; 205 + unsigned long edat : 1; /* Enhanced-DAT-enablement control */ 206 + unsigned long : 2; 207 + unsigned long iep : 1; /* Instruction-Execution-Protection */ 208 + unsigned long : 1; 209 + unsigned long afp : 1; /* AFP-register control */ 210 + unsigned long vx : 1; /* Vector enablement control */ 211 + unsigned long : 7; 212 + unsigned long sssm : 1; /* Service signal subclass mask */ 213 + unsigned long : 9; 214 + }; 215 + }; 216 + 217 + union ctlreg2 { 218 + unsigned long val; 219 + struct ctlreg reg; 220 + struct { 221 + unsigned long : 33; 222 + unsigned long ducto : 25; 223 + unsigned long : 1; 224 + unsigned long gse : 1; 225 + unsigned long : 1; 226 + unsigned long tds : 1; 227 + unsigned long tdc : 2; 228 + }; 229 + }; 230 + 231 + union ctlreg5 { 232 + unsigned long val; 233 + struct ctlreg reg; 234 + struct { 235 + unsigned long : 33; 236 + unsigned long pasteo: 25; 237 + unsigned long : 6; 238 + }; 239 + }; 240 + 241 + union ctlreg15 { 242 + unsigned long val; 243 + struct ctlreg reg; 244 + struct { 245 + unsigned long lsea : 61; 246 + unsigned long : 3; 247 + }; 248 + }; 249 + 250 + #endif /* __ASSEMBLY__ */ 251 + #endif /* __ASM_S390_CTLREG_H */

+28

arch/s390/include/asm/fault.h

··· 1 + /* SPDX-License-Identifier: GPL-2.0 */ 2 + /* 3 + * Copyright IBM Corp. 1999, 2023 4 + */ 5 + #ifndef _ASM_S390_FAULT_H 6 + #define _ASM_S390_FAULT_H 7 + 8 + union teid { 9 + unsigned long val; 10 + struct { 11 + unsigned long addr : 52; /* Translation-exception Address */ 12 + unsigned long fsi : 2; /* Access Exception Fetch/Store Indication */ 13 + unsigned long : 2; 14 + unsigned long b56 : 1; 15 + unsigned long : 3; 16 + unsigned long b60 : 1; 17 + unsigned long b61 : 1; 18 + unsigned long as : 2; /* ASCE Identifier */ 19 + }; 20 + }; 21 + 22 + enum { 23 + TEID_FSI_UNKNOWN = 0, /* Unknown whether fetch or store */ 24 + TEID_FSI_STORE = 1, /* Exception was due to store operation */ 25 + TEID_FSI_FETCH = 2 /* Exception was due to fetch operation */ 26 + }; 27 + 28 + #endif /* _ASM_S390_FAULT_H */

-1

arch/s390/include/asm/fpu/internal.h

··· 10 10 #define _ASM_S390_FPU_INTERNAL_H 11 11 12 12 #include <linux/string.h> 13 - #include <asm/ctl_reg.h> 14 13 #include <asm/fpu/types.h> 15 14 16 15 static inline void save_vx_regs(__vector128 *vxrs)

+12 -11

arch/s390/include/asm/irq.h

··· 31 31 #include <linux/percpu.h> 32 32 #include <linux/cache.h> 33 33 #include <linux/types.h> 34 + #include <asm/ctlreg.h> 34 35 35 36 enum interruption_class { 36 37 IRQEXT_CLK, ··· 102 101 }; 103 102 104 103 #define CR0_IRQ_SUBCLASS_MASK \ 105 - ((1UL << (63 - 30)) /* Warning Track */ | \ 106 - (1UL << (63 - 48)) /* Malfunction Alert */ | \ 107 - (1UL << (63 - 49)) /* Emergency Signal */ | \ 108 - (1UL << (63 - 50)) /* External Call */ | \ 109 - (1UL << (63 - 52)) /* Clock Comparator */ | \ 110 - (1UL << (63 - 53)) /* CPU Timer */ | \ 111 - (1UL << (63 - 54)) /* Service Signal */ | \ 112 - (1UL << (63 - 57)) /* Interrupt Key */ | \ 113 - (1UL << (63 - 58)) /* Measurement Alert */ | \ 114 - (1UL << (63 - 59)) /* Timing Alert */ | \ 115 - (1UL << (63 - 62))) /* IUCV */ 104 + (CR0_WARNING_TRACK | \ 105 + CR0_MALFUNCTION_ALERT_SUBMASK | \ 106 + CR0_EMERGENCY_SIGNAL_SUBMASK | \ 107 + CR0_EXTERNAL_CALL_SUBMASK | \ 108 + CR0_CLOCK_COMPARATOR_SUBMASK | \ 109 + CR0_CPU_TIMER_SUBMASK | \ 110 + CR0_SERVICE_SIGNAL_SUBMASK | \ 111 + CR0_INTERRUPT_KEY_SUBMASK | \ 112 + CR0_MEASUREMENT_ALERT_SUBMASK | \ 113 + CR0_ETR_SUBMASK | \ 114 + CR0_IUCV) 116 115 117 116 void irq_subclass_register(enum irq_subclass subclass); 118 117 void irq_subclass_unregister(enum irq_subclass subclass);

+2 -1

arch/s390/include/asm/kprobes.h

··· 15 15 * <grundym@us.ibm.com> 16 16 */ 17 17 #include <linux/types.h> 18 + #include <asm/ctlreg.h> 18 19 #include <asm-generic/kprobes.h> 19 20 20 21 #define BREAKPOINT_INSTRUCTION 0x0002 ··· 66 65 struct kprobe_ctlblk { 67 66 unsigned long kprobe_status; 68 67 unsigned long kprobe_saved_imask; 69 - unsigned long kprobe_saved_ctl[3]; 68 + struct ctlreg kprobe_saved_ctl[3]; 70 69 struct prev_kprobe prev_kprobe; 71 70 }; 72 71

+4 -11

arch/s390/include/asm/lowcore.h

··· 11 11 12 12 #include <linux/types.h> 13 13 #include <asm/ptrace.h> 14 + #include <asm/ctlreg.h> 14 15 #include <asm/cpu.h> 15 16 #include <asm/types.h> 16 17 ··· 140 139 __u32 restart_flags; /* 0x0384 */ 141 140 142 141 /* Address space pointer. */ 143 - __u64 kernel_asce; /* 0x0388 */ 144 - __u64 user_asce; /* 0x0390 */ 142 + struct ctlreg kernel_asce; /* 0x0388 */ 143 + struct ctlreg user_asce; /* 0x0390 */ 145 144 146 145 /* 147 146 * The lpp and current_pid fields form a ··· 200 199 __u32 clock_comp_save_area[2]; /* 0x1330 */ 201 200 __u64 last_break_save_area; /* 0x1338 */ 202 201 __u32 access_regs_save_area[16]; /* 0x1340 */ 203 - __u64 cregs_save_area[16]; /* 0x1380 */ 202 + struct ctlreg cregs_save_area[16]; /* 0x1380 */ 204 203 __u8 pad_0x1400[0x1500-0x1400]; /* 0x1400 */ 205 204 /* Cryptography-counter designation */ 206 205 __u64 ccd; /* 0x1500 */ ··· 220 219 static inline void set_prefix(__u32 address) 221 220 { 222 221 asm volatile("spx %0" : : "Q" (address) : "memory"); 223 - } 224 - 225 - static inline __u32 store_prefix(void) 226 - { 227 - __u32 address; 228 - 229 - asm volatile("stpx %0" : "=Q" (address)); 230 - return address; 231 222 } 232 223 233 224 #endif /* _ASM_S390_LOWCORE_H */

+5 -5

arch/s390/include/asm/mmu_context.h

··· 12 12 #include <linux/uaccess.h> 13 13 #include <linux/mm_types.h> 14 14 #include <asm/tlbflush.h> 15 - #include <asm/ctl_reg.h> 15 + #include <asm/ctlreg.h> 16 16 #include <asm-generic/mm_hooks.h> 17 17 18 18 #define init_new_context init_new_context ··· 78 78 if (next == &init_mm) 79 79 S390_lowcore.user_asce = s390_invalid_asce; 80 80 else 81 - S390_lowcore.user_asce = next->context.asce; 81 + S390_lowcore.user_asce.val = next->context.asce; 82 82 cpumask_set_cpu(cpu, &next->context.cpu_attach_mask); 83 83 /* Clear previous user-ASCE from CR7 */ 84 - __ctl_load(s390_invalid_asce, 7, 7); 84 + local_ctl_load(7, &s390_invalid_asce); 85 85 if (prev != next) 86 86 cpumask_clear_cpu(cpu, &prev->context.cpu_attach_mask); 87 87 } ··· 111 111 __tlb_flush_mm_lazy(mm); 112 112 preempt_enable(); 113 113 } 114 - __ctl_load(S390_lowcore.user_asce, 7, 7); 114 + local_ctl_load(7, &S390_lowcore.user_asce); 115 115 } 116 116 117 117 #define activate_mm activate_mm ··· 120 120 { 121 121 switch_mm(prev, next, current); 122 122 cpumask_set_cpu(smp_processor_id(), mm_cpumask(next)); 123 - __ctl_load(S390_lowcore.user_asce, 7, 7); 123 + local_ctl_load(7, &S390_lowcore.user_asce); 124 124 } 125 125 126 126 #include <asm-generic/mmu_context.h>

+2 -1

arch/s390/include/asm/pgtable.h

··· 18 18 #include <linux/radix-tree.h> 19 19 #include <linux/atomic.h> 20 20 #include <asm/sections.h> 21 + #include <asm/ctlreg.h> 21 22 #include <asm/bug.h> 22 23 #include <asm/page.h> 23 24 #include <asm/uv.h> ··· 26 25 extern pgd_t swapper_pg_dir[]; 27 26 extern pgd_t invalid_pg_dir[]; 28 27 extern void paging_init(void); 29 - extern unsigned long s390_invalid_asce; 28 + extern struct ctlreg s390_invalid_asce; 30 29 31 30 enum { 32 31 PG_DIRECT_MAP_4K = 0,

-1

arch/s390/include/asm/smp.h

··· 59 59 { 60 60 } 61 61 62 - extern int smp_reinit_ipl_cpu(void); 63 62 extern int smp_rescan_cpus(void); 64 63 extern void __noreturn cpu_die(void); 65 64 extern void __cpu_die(unsigned int cpu);

-1

arch/s390/include/asm/uaccess.h

··· 15 15 */ 16 16 #include <asm/asm-extable.h> 17 17 #include <asm/processor.h> 18 - #include <asm/ctl_reg.h> 19 18 #include <asm/extable.h> 20 19 #include <asm/facility.h> 21 20 #include <asm-generic/access_ok.h>

+64

arch/s390/include/asm/word-at-a-time.h

··· 1 + /* SPDX-License-Identifier: GPL-2.0 */ 2 + #ifndef _ASM_WORD_AT_A_TIME_H 3 + #define _ASM_WORD_AT_A_TIME_H 4 + 5 + #include <linux/kernel.h> 6 + #include <asm/asm-extable.h> 7 + #include <asm/bitsperlong.h> 8 + 9 + struct word_at_a_time { 10 + const unsigned long bits; 11 + }; 12 + 13 + #define WORD_AT_A_TIME_CONSTANTS { REPEAT_BYTE(0x7f) } 14 + 15 + static inline unsigned long prep_zero_mask(unsigned long val, unsigned long data, const struct word_at_a_time *c) 16 + { 17 + return data; 18 + } 19 + 20 + static inline unsigned long create_zero_mask(unsigned long data) 21 + { 22 + return __fls(data); 23 + } 24 + 25 + static inline unsigned long find_zero(unsigned long data) 26 + { 27 + return (data ^ (BITS_PER_LONG - 1)) >> 3; 28 + } 29 + 30 + static inline unsigned long has_zero(unsigned long val, unsigned long *data, const struct word_at_a_time *c) 31 + { 32 + unsigned long mask = (val & c->bits) + c->bits; 33 + 34 + *data = ~(mask | val | c->bits); 35 + return *data; 36 + } 37 + 38 + static inline unsigned long zero_bytemask(unsigned long data) 39 + { 40 + return ~1UL << data; 41 + } 42 + 43 + /* 44 + * Load an unaligned word from kernel space. 45 + * 46 + * In the (very unlikely) case of the word being a page-crosser 47 + * and the next page not being mapped, take the exception and 48 + * return zeroes in the non-existing part. 49 + */ 50 + static inline unsigned long load_unaligned_zeropad(const void *addr) 51 + { 52 + unsigned long data; 53 + 54 + asm volatile( 55 + "0: lg %[data],0(%[addr])\n" 56 + "1: nopr %%r7\n" 57 + EX_TABLE_ZEROPAD(0b, 1b, %[data], %[addr]) 58 + EX_TABLE_ZEROPAD(1b, 1b, %[data], %[addr]) 59 + : [data] "=d" (data) 60 + : [addr] "a" (addr), "m" (*(unsigned long *)addr)); 61 + return data; 62 + } 63 + 64 + #endif /* _ASM_WORD_AT_A_TIME_H */

+1 -1

arch/s390/kernel/Makefile

··· 37 37 obj-y := head64.o traps.o time.o process.o earlypgm.o early.o setup.o idle.o vtime.o 38 38 obj-y += processor.o syscall.o ptrace.o signal.o cpcmd.o ebcdic.o nmi.o 39 39 obj-y += debug.o irq.o ipl.o dis.o diag.o vdso.o cpufeature.o 40 - obj-y += sysinfo.o lgr.o os_info.o 40 + obj-y += sysinfo.o lgr.o os_info.o ctlreg.o 41 41 obj-y += runtime_instr.o cache.o fpu.o dumpstack.o guarded_storage.o sthyi.o 42 42 obj-y += entry.o reipl.o kdebugfs.o alternative.o 43 43 obj-y += nospec-branch.o ipl_vmparm.o machine_kexec_reloc.o unwind_bc.o

+3 -3

arch/s390/kernel/crash_dump.c

··· 498 498 /* 499 499 * Initialize ELF loads (new kernel) 500 500 */ 501 - static void loads_init(Elf64_Phdr *phdr, u64 loads_offset) 501 + static void loads_init(Elf64_Phdr *phdr) 502 502 { 503 503 phys_addr_t start, end; 504 504 u64 idx; ··· 507 507 phdr->p_filesz = end - start; 508 508 phdr->p_type = PT_LOAD; 509 509 phdr->p_offset = start; 510 - phdr->p_vaddr = start; 510 + phdr->p_vaddr = (unsigned long)__va(start); 511 511 phdr->p_paddr = start; 512 512 phdr->p_memsz = end - start; 513 513 phdr->p_flags = PF_R | PF_W | PF_X; ··· 612 612 ptr = notes_init(phdr_notes, ptr, ((unsigned long) hdr) + hdr_off); 613 613 /* Init loads */ 614 614 hdr_off = PTR_DIFF(ptr, hdr); 615 - loads_init(phdr_loads, hdr_off); 615 + loads_init(phdr_loads); 616 616 *addr = (unsigned long long) hdr; 617 617 *size = (unsigned long long) hdr_off; 618 618 BUG_ON(elfcorehdr_size > alloc_size);

+121

arch/s390/kernel/ctlreg.c

··· 1 + // SPDX-License-Identifier: GPL-2.0 2 + /* 3 + * Copyright IBM Corp. 1999, 2023 4 + */ 5 + 6 + #include <linux/irqflags.h> 7 + #include <linux/spinlock.h> 8 + #include <linux/kernel.h> 9 + #include <linux/init.h> 10 + #include <linux/smp.h> 11 + #include <linux/cache.h> 12 + #include <asm/abs_lowcore.h> 13 + #include <asm/ctlreg.h> 14 + 15 + /* 16 + * ctl_lock guards access to global control register contents which 17 + * are kept in the control register save area within absolute lowcore 18 + * at physical address zero. 19 + */ 20 + static DEFINE_SPINLOCK(system_ctl_lock); 21 + 22 + void system_ctlreg_lock(void) 23 + __acquires(&system_ctl_lock) 24 + { 25 + spin_lock(&system_ctl_lock); 26 + } 27 + 28 + void system_ctlreg_unlock(void) 29 + __releases(&system_ctl_lock) 30 + { 31 + spin_unlock(&system_ctl_lock); 32 + } 33 + 34 + static bool system_ctlreg_area_init __ro_after_init; 35 + 36 + void __init system_ctlreg_init_save_area(struct lowcore *lc) 37 + { 38 + struct lowcore *abs_lc; 39 + 40 + abs_lc = get_abs_lowcore(); 41 + __local_ctl_store(0, 15, lc->cregs_save_area); 42 + __local_ctl_store(0, 15, abs_lc->cregs_save_area); 43 + put_abs_lowcore(abs_lc); 44 + system_ctlreg_area_init = true; 45 + } 46 + 47 + struct ctlreg_parms { 48 + unsigned long andval; 49 + unsigned long orval; 50 + unsigned long val; 51 + int request; 52 + int cr; 53 + }; 54 + 55 + static void ctlreg_callback(void *info) 56 + { 57 + struct ctlreg_parms *pp = info; 58 + struct ctlreg regs[16]; 59 + 60 + __local_ctl_store(0, 15, regs); 61 + if (pp->request == CTLREG_LOAD) { 62 + regs[pp->cr].val = pp->val; 63 + } else { 64 + regs[pp->cr].val &= pp->andval; 65 + regs[pp->cr].val |= pp->orval; 66 + } 67 + __local_ctl_load(0, 15, regs); 68 + } 69 + 70 + static void system_ctlreg_update(void *info) 71 + { 72 + unsigned long flags; 73 + 74 + if (system_state == SYSTEM_BOOTING) { 75 + /* 76 + * For very early calls do not call on_each_cpu() 77 + * since not everything might be setup. 78 + */ 79 + local_irq_save(flags); 80 + ctlreg_callback(info); 81 + local_irq_restore(flags); 82 + } else { 83 + on_each_cpu(ctlreg_callback, info, 1); 84 + } 85 + } 86 + 87 + void system_ctlreg_modify(unsigned int cr, unsigned long data, int request) 88 + { 89 + struct ctlreg_parms pp = { .cr = cr, .request = request, }; 90 + struct lowcore *abs_lc; 91 + 92 + switch (request) { 93 + case CTLREG_SET_BIT: 94 + pp.orval = 1UL << data; 95 + pp.andval = -1UL; 96 + break; 97 + case CTLREG_CLEAR_BIT: 98 + pp.orval = 0; 99 + pp.andval = ~(1UL << data); 100 + break; 101 + case CTLREG_LOAD: 102 + pp.val = data; 103 + break; 104 + } 105 + if (system_ctlreg_area_init) { 106 + system_ctlreg_lock(); 107 + abs_lc = get_abs_lowcore(); 108 + if (request == CTLREG_LOAD) { 109 + abs_lc->cregs_save_area[cr].val = pp.val; 110 + } else { 111 + abs_lc->cregs_save_area[cr].val &= pp.andval; 112 + abs_lc->cregs_save_area[cr].val |= pp.orval; 113 + } 114 + put_abs_lowcore(abs_lc); 115 + system_ctlreg_update(&pp); 116 + system_ctlreg_unlock(); 117 + } else { 118 + system_ctlreg_update(&pp); 119 + } 120 + } 121 + EXPORT_SYMBOL(system_ctlreg_modify);

+2 -1

arch/s390/kernel/diag.c

··· 245 245 246 246 int diag224(void *ptr) 247 247 { 248 + unsigned long addr = __pa(ptr); 248 249 int rc = -EOPNOTSUPP; 249 250 250 251 diag_stat_inc(DIAG_STAT_X224); ··· 254 253 "0: lhi %0,0x0\n" 255 254 "1:\n" 256 255 EX_TABLE(0b,1b) 257 - : "+d" (rc) :"d" (0), "d" (ptr) : "memory"); 256 + : "+d" (rc) :"d" (0), "d" (addr) : "memory"); 258 257 return rc; 259 258 } 260 259 EXPORT_SYMBOL(diag224);

+8 -14

arch/s390/kernel/early.c

··· 216 216 { 217 217 if (test_facility(8)) { 218 218 S390_lowcore.machine_flags |= MACHINE_FLAG_EDAT1; 219 - __ctl_set_bit(0, 23); 219 + system_ctl_set_bit(0, CR0_EDAT_BIT); 220 220 } 221 221 if (test_facility(78)) 222 222 S390_lowcore.machine_flags |= MACHINE_FLAG_EDAT2; ··· 224 224 S390_lowcore.machine_flags |= MACHINE_FLAG_IDTE; 225 225 if (test_facility(50) && test_facility(73)) { 226 226 S390_lowcore.machine_flags |= MACHINE_FLAG_TE; 227 - __ctl_set_bit(0, 55); 227 + system_ctl_set_bit(0, CR0_TRANSACTIONAL_EXECUTION_BIT); 228 228 } 229 229 if (test_facility(51)) 230 230 S390_lowcore.machine_flags |= MACHINE_FLAG_TLB_LC; 231 231 if (test_facility(129)) { 232 232 S390_lowcore.machine_flags |= MACHINE_FLAG_VX; 233 - __ctl_set_bit(0, 17); 233 + system_ctl_set_bit(0, CR0_VECTOR_BIT); 234 234 } 235 235 if (test_facility(130)) 236 236 S390_lowcore.machine_flags |= MACHINE_FLAG_NX; ··· 240 240 /* Enabled signed clock comparator comparisons */ 241 241 S390_lowcore.machine_flags |= MACHINE_FLAG_SCC; 242 242 clock_comparator_max = -1ULL >> 1; 243 - __ctl_set_bit(0, 53); 243 + system_ctl_set_bit(0, CR0_CLOCK_COMPARATOR_SIGN_BIT); 244 244 } 245 245 if (IS_ENABLED(CONFIG_PCI) && test_facility(153)) { 246 246 S390_lowcore.machine_flags |= MACHINE_FLAG_PCI_MIO; ··· 258 258 #endif 259 259 } 260 260 261 - static inline void setup_control_registers(void) 261 + static inline void setup_low_address_protection(void) 262 262 { 263 - unsigned long reg; 264 - 265 - __ctl_store(reg, 0, 0); 266 - reg |= CR0_LOW_ADDRESS_PROTECTION; 267 - reg |= CR0_EMERGENCY_SIGNAL_SUBMASK; 268 - reg |= CR0_EXTERNAL_CALL_SUBMASK; 269 - __ctl_load(reg, 0, 0); 263 + system_ctl_set_bit(0, CR0_LOW_ADDRESS_PROTECTION_BIT); 270 264 } 271 265 272 266 static inline void setup_access_registers(void) ··· 273 279 static int __init disable_vector_extension(char *str) 274 280 { 275 281 S390_lowcore.machine_flags &= ~MACHINE_FLAG_VX; 276 - __ctl_clear_bit(0, 17); 282 + system_ctl_clear_bit(0, CR0_VECTOR_BIT); 277 283 return 0; 278 284 } 279 285 early_param("novx", disable_vector_extension); ··· 308 314 save_vector_registers(); 309 315 setup_topology(); 310 316 sclp_early_detect(); 311 - setup_control_registers(); 317 + setup_low_address_protection(); 312 318 setup_access_registers(); 313 319 lockdep_on(); 314 320 }

+3 -3

arch/s390/kernel/guarded_storage.c

··· 28 28 return -ENOMEM; 29 29 gs_cb->gsd = 25; 30 30 preempt_disable(); 31 - __ctl_set_bit(2, 4); 31 + local_ctl_set_bit(2, CR2_GUARDED_STORAGE_BIT); 32 32 load_gs_cb(gs_cb); 33 33 current->thread.gs_cb = gs_cb; 34 34 preempt_enable(); ··· 42 42 preempt_disable(); 43 43 kfree(current->thread.gs_cb); 44 44 current->thread.gs_cb = NULL; 45 - __ctl_clear_bit(2, 4); 45 + local_ctl_clear_bit(2, CR2_GUARDED_STORAGE_BIT); 46 46 preempt_enable(); 47 47 } 48 48 return 0; ··· 84 84 if (gs_cb) { 85 85 kfree(current->thread.gs_cb); 86 86 current->thread.gs_bc_cb = NULL; 87 - __ctl_set_bit(2, 4); 87 + local_ctl_set_bit(2, CR2_GUARDED_STORAGE_BIT); 88 88 load_gs_cb(gs_cb); 89 89 current->thread.gs_cb = gs_cb; 90 90 }

+1 -1

arch/s390/kernel/ipl.c

··· 2381 2381 set_prefix(0); 2382 2382 2383 2383 /* Disable lowcore protection */ 2384 - __ctl_clear_bit(0, 28); 2384 + local_ctl_clear_bit(0, CR0_LOW_ADDRESS_PROTECTION_BIT); 2385 2385 diag_amode31_ops.diag308_reset(); 2386 2386 } 2387 2387

+2 -2

arch/s390/kernel/irq.c

··· 385 385 { 386 386 spin_lock(&irq_subclass_lock); 387 387 if (!irq_subclass_refcount[subclass]) 388 - ctl_set_bit(0, subclass); 388 + system_ctl_set_bit(0, subclass); 389 389 irq_subclass_refcount[subclass]++; 390 390 spin_unlock(&irq_subclass_lock); 391 391 } ··· 396 396 spin_lock(&irq_subclass_lock); 397 397 irq_subclass_refcount[subclass]--; 398 398 if (!irq_subclass_refcount[subclass]) 399 - ctl_clear_bit(0, subclass); 399 + system_ctl_clear_bit(0, subclass); 400 400 spin_unlock(&irq_subclass_lock); 401 401 } 402 402 EXPORT_SYMBOL(irq_subclass_unregister);

+14 -7

arch/s390/kernel/kprobes.c

··· 224 224 struct pt_regs *regs, 225 225 unsigned long ip) 226 226 { 227 - struct per_regs per_kprobe; 227 + union { 228 + struct ctlreg regs[3]; 229 + struct { 230 + struct ctlreg control; 231 + struct ctlreg start; 232 + struct ctlreg end; 233 + }; 234 + } per_kprobe; 228 235 229 236 /* Set up the PER control registers %cr9-%cr11 */ 230 - per_kprobe.control = PER_EVENT_IFETCH; 231 - per_kprobe.start = ip; 232 - per_kprobe.end = ip; 237 + per_kprobe.control.val = PER_EVENT_IFETCH; 238 + per_kprobe.start.val = ip; 239 + per_kprobe.end.val = ip; 233 240 234 241 /* Save control regs and psw mask */ 235 - __ctl_store(kcb->kprobe_saved_ctl, 9, 11); 242 + __local_ctl_store(9, 11, kcb->kprobe_saved_ctl); 236 243 kcb->kprobe_saved_imask = regs->psw.mask & 237 244 (PSW_MASK_PER | PSW_MASK_IO | PSW_MASK_EXT); 238 245 239 246 /* Set PER control regs, turns on single step for the given address */ 240 - __ctl_load(per_kprobe, 9, 11); 247 + __local_ctl_load(9, 11, per_kprobe.regs); 241 248 regs->psw.mask |= PSW_MASK_PER; 242 249 regs->psw.mask &= ~(PSW_MASK_IO | PSW_MASK_EXT); 243 250 regs->psw.addr = ip; ··· 256 249 unsigned long ip) 257 250 { 258 251 /* Restore control regs and psw mask, set new psw address */ 259 - __ctl_load(kcb->kprobe_saved_ctl, 9, 11); 252 + __local_ctl_load(9, 11, kcb->kprobe_saved_ctl); 260 253 regs->psw.mask &= ~PSW_MASK_PER; 261 254 regs->psw.mask |= kcb->kprobe_saved_imask; 262 255 regs->psw.addr = ip;

+3 -3

arch/s390/kernel/machine_kexec.c

··· 94 94 if (MACHINE_HAS_VX) 95 95 save_vx_regs((__vector128 *) mcesa->vector_save_area); 96 96 if (MACHINE_HAS_GS) { 97 - __ctl_store(cr2_old.val, 2, 2); 97 + local_ctl_store(2, &cr2_old.reg); 98 98 cr2_new = cr2_old; 99 99 cr2_new.gse = 1; 100 - __ctl_load(cr2_new.val, 2, 2); 100 + local_ctl_load(2, &cr2_new.reg); 101 101 save_gs_cb((struct gs_cb *) mcesa->guarded_storage_save_area); 102 - __ctl_load(cr2_old.val, 2, 2); 102 + local_ctl_load(2, &cr2_old.reg); 103 103 } 104 104 /* 105 105 * To create a good backchain for this CPU in the dump store_status

+12 -12

arch/s390/kernel/nmi.c

··· 22 22 #include <linux/kvm_host.h> 23 23 #include <linux/export.h> 24 24 #include <asm/lowcore.h> 25 + #include <asm/ctlreg.h> 25 26 #include <asm/smp.h> 26 27 #include <asm/stp.h> 27 28 #include <asm/cputime.h> 28 29 #include <asm/nmi.h> 29 30 #include <asm/crw.h> 30 31 #include <asm/switch_to.h> 31 - #include <asm/ctl_reg.h> 32 32 #include <asm/asm-offsets.h> 33 33 #include <asm/pai.h> 34 34 #include <asm/vx-insn.h> ··· 131 131 * Disable low address protection and make machine check new PSW a 132 132 * disabled wait PSW. Any additional machine check cannot be handled. 133 133 */ 134 - __ctl_store(cr0.val, 0, 0); 134 + local_ctl_store(0, &cr0.reg); 135 135 cr0_new = cr0; 136 136 cr0_new.lap = 0; 137 - __ctl_load(cr0_new.val, 0, 0); 137 + local_ctl_load(0, &cr0_new.reg); 138 138 psw_save = S390_lowcore.mcck_new_psw; 139 139 psw_bits(S390_lowcore.mcck_new_psw).io = 0; 140 140 psw_bits(S390_lowcore.mcck_new_psw).ext = 0; ··· 146 146 * values. This makes possible system dump analysis easier. 147 147 */ 148 148 S390_lowcore.mcck_new_psw = psw_save; 149 - __ctl_load(cr0.val, 0, 0); 149 + local_ctl_load(0, &cr0.reg); 150 150 disabled_wait(); 151 151 while (1); 152 152 } ··· 185 185 static int mchchk_wng_posted = 0; 186 186 187 187 /* Use single cpu clear, as we cannot handle smp here. */ 188 - __ctl_clear_bit(14, 24); /* Disable WARNING MCH */ 188 + local_ctl_clear_bit(14, CR14_WARNING_SUBMASK_BIT); 189 189 if (xchg(&mchchk_wng_posted, 1) == 0) 190 190 kill_cad_pid(SIGPWR, 1); 191 191 } ··· 269 269 */ 270 270 if (!mci.vr && !test_cpu_flag(CIF_MCCK_GUEST)) 271 271 kill_task = 1; 272 - cr0.val = S390_lowcore.cregs_save_area[0]; 272 + cr0.reg = S390_lowcore.cregs_save_area[0]; 273 273 cr0.afp = cr0.vx = 1; 274 - __ctl_load(cr0.val, 0, 0); 274 + local_ctl_load(0, &cr0.reg); 275 275 asm volatile( 276 276 " la 1,%0\n" 277 277 " VLM 0,15,0,1\n" ··· 279 279 : 280 280 : "Q" (*(struct vx_array *)mcesa->vector_save_area) 281 281 : "1"); 282 - __ctl_load(S390_lowcore.cregs_save_area[0], 0, 0); 282 + local_ctl_load(0, &S390_lowcore.cregs_save_area[0]); 283 283 } 284 284 /* Validate access registers */ 285 285 asm volatile( ··· 290 290 if (!mci.ar) 291 291 kill_task = 1; 292 292 /* Validate guarded storage registers */ 293 - cr2.val = S390_lowcore.cregs_save_area[2]; 293 + cr2.reg = S390_lowcore.cregs_save_area[2]; 294 294 if (cr2.gse) { 295 295 if (!mci.gs) { 296 296 /* ··· 505 505 506 506 static int __init machine_check_init(void) 507 507 { 508 - ctl_set_bit(14, 25); /* enable external damage MCH */ 509 - ctl_set_bit(14, 27); /* enable system recovery MCH */ 510 - ctl_set_bit(14, 24); /* enable warning MCH */ 508 + system_ctl_set_bit(14, CR14_EXTERNAL_DAMAGE_SUBMASK_BIT); 509 + system_ctl_set_bit(14, CR14_RECOVERY_SUBMASK_BIT); 510 + system_ctl_set_bit(14, CR14_WARNING_SUBMASK_BIT); 511 511 return 0; 512 512 } 513 513 early_initcall(machine_check_init);

+1 -1

arch/s390/kernel/perf_cpum_cf.c

··· 1193 1193 * Clear bit 15 of cr0 to unauthorize problem-state to 1194 1194 * extract measurement counters 1195 1195 */ 1196 - ctl_clear_bit(0, 48); 1196 + system_ctl_clear_bit(0, CR0_CPUMF_EXTRACTION_AUTH_BIT); 1197 1197 1198 1198 /* register handler for measurement-alert interruptions */ 1199 1199 rc = register_external_irq(EXT_IRQ_MEASURE_ALERT,

+104 -34

arch/s390/kernel/perf_pai_crypto.c

··· 16 16 #include <linux/export.h> 17 17 #include <linux/io.h> 18 18 #include <linux/perf_event.h> 19 - 20 - #include <asm/ctl_reg.h> 19 + #include <asm/ctlreg.h> 21 20 #include <asm/pai.h> 22 21 #include <asm/debug.h> 23 22 ··· 40 41 struct perf_event *event; /* Perf event for sampling */ 41 42 }; 42 43 43 - static DEFINE_PER_CPU(struct paicrypt_map, paicrypt_map); 44 + struct paicrypt_mapptr { 45 + struct paicrypt_map *mapptr; 46 + }; 47 + 48 + static struct paicrypt_root { /* Anchor to per CPU data */ 49 + refcount_t refcnt; /* Overall active events */ 50 + struct paicrypt_mapptr __percpu *mapptr; 51 + } paicrypt_root; 52 + 53 + /* Free per CPU data when the last event is removed. */ 54 + static void paicrypt_root_free(void) 55 + { 56 + if (refcount_dec_and_test(&paicrypt_root.refcnt)) { 57 + free_percpu(paicrypt_root.mapptr); 58 + paicrypt_root.mapptr = NULL; 59 + } 60 + debug_sprintf_event(cfm_dbg, 5, "%s root.refcount %d\n", __func__, 61 + refcount_read(&paicrypt_root.refcnt)); 62 + } 63 + 64 + /* 65 + * On initialization of first event also allocate per CPU data dynamically. 66 + * Start with an array of pointers, the array size is the maximum number of 67 + * CPUs possible, which might be larger than the number of CPUs currently 68 + * online. 69 + */ 70 + static int paicrypt_root_alloc(void) 71 + { 72 + if (!refcount_inc_not_zero(&paicrypt_root.refcnt)) { 73 + /* The memory is already zeroed. */ 74 + paicrypt_root.mapptr = alloc_percpu(struct paicrypt_mapptr); 75 + if (!paicrypt_root.mapptr) 76 + return -ENOMEM; 77 + refcount_set(&paicrypt_root.refcnt, 1); 78 + } 79 + return 0; 80 + } 44 81 45 82 /* Release the PMU if event is the last perf event */ 46 83 static DEFINE_MUTEX(pai_reserve_mutex); ··· 86 51 */ 87 52 static void paicrypt_event_destroy(struct perf_event *event) 88 53 { 89 - struct paicrypt_map *cpump = per_cpu_ptr(&paicrypt_map, event->cpu); 54 + struct paicrypt_mapptr *mp = per_cpu_ptr(paicrypt_root.mapptr, 55 + event->cpu); 56 + struct paicrypt_map *cpump = mp->mapptr; 90 57 91 58 cpump->event = NULL; 92 59 static_branch_dec(&pai_key); ··· 103 66 __func__, (unsigned long)cpump->page, 104 67 cpump->save); 105 68 free_page((unsigned long)cpump->page); 106 - cpump->page = NULL; 107 69 kvfree(cpump->save); 108 - cpump->save = NULL; 109 - cpump->mode = PAI_MODE_NONE; 70 + kfree(cpump); 71 + mp->mapptr = NULL; 110 72 } 73 + paicrypt_root_free(); 111 74 mutex_unlock(&pai_reserve_mutex); 112 75 } 113 76 ··· 123 86 */ 124 87 static u64 paicrypt_getdata(struct perf_event *event, bool kernel) 125 88 { 126 - struct paicrypt_map *cpump = this_cpu_ptr(&paicrypt_map); 89 + struct paicrypt_mapptr *mp = this_cpu_ptr(paicrypt_root.mapptr); 90 + struct paicrypt_map *cpump = mp->mapptr; 127 91 u64 sum = 0; 128 92 int i; 129 93 ··· 170 132 * 171 133 * Allocate the memory for the event. 172 134 */ 173 - static int paicrypt_busy(struct perf_event_attr *a, struct paicrypt_map *cpump) 135 + static struct paicrypt_map *paicrypt_busy(struct perf_event *event) 174 136 { 175 - int rc = 0; 137 + struct perf_event_attr *a = &event->attr; 138 + struct paicrypt_map *cpump = NULL; 139 + struct paicrypt_mapptr *mp; 140 + int rc; 176 141 177 142 mutex_lock(&pai_reserve_mutex); 143 + 144 + /* Allocate root node */ 145 + rc = paicrypt_root_alloc(); 146 + if (rc) 147 + goto unlock; 148 + 149 + /* Allocate node for this event */ 150 + mp = per_cpu_ptr(paicrypt_root.mapptr, event->cpu); 151 + cpump = mp->mapptr; 152 + if (!cpump) { /* Paicrypt_map allocated? */ 153 + cpump = kzalloc(sizeof(*cpump), GFP_KERNEL); 154 + if (!cpump) { 155 + rc = -ENOMEM; 156 + goto free_root; 157 + } 158 + } 159 + 178 160 if (a->sample_period) { /* Sampling requested */ 179 161 if (cpump->mode != PAI_MODE_NONE) 180 162 rc = -EBUSY; /* ... sampling/counting active */ ··· 202 144 if (cpump->mode == PAI_MODE_SAMPLING) 203 145 rc = -EBUSY; /* ... and sampling active */ 204 146 } 147 + /* 148 + * This error case triggers when there is a conflict: 149 + * Either sampling requested and counting already active, or visa 150 + * versa. Therefore the struct paicrypto_map for this CPU is 151 + * needed or the error could not have occurred. Only adjust root 152 + * node refcount. 153 + */ 205 154 if (rc) 206 - goto unlock; 155 + goto free_root; 207 156 208 157 /* Allocate memory for counter page and counter extraction. 209 158 * Only the first counting event has to allocate a page. ··· 223 158 rc = -ENOMEM; 224 159 cpump->page = (unsigned long *)get_zeroed_page(GFP_KERNEL); 225 160 if (!cpump->page) 226 - goto unlock; 161 + goto free_paicrypt_map; 227 162 cpump->save = kvmalloc_array(paicrypt_cnt + 1, 228 163 sizeof(struct pai_userdata), GFP_KERNEL); 229 164 if (!cpump->save) { 230 165 free_page((unsigned long)cpump->page); 231 166 cpump->page = NULL; 232 - goto unlock; 167 + goto free_paicrypt_map; 233 168 } 169 + 170 + /* Set mode and reference count */ 234 171 rc = 0; 235 172 refcount_set(&cpump->refcnt, 1); 236 - 237 - unlock: 238 - /* If rc is non-zero, do not set mode and reference count */ 239 - if (!rc) { 240 - cpump->mode = a->sample_period ? PAI_MODE_SAMPLING 241 - : PAI_MODE_COUNTING; 242 - } 173 + cpump->mode = a->sample_period ? PAI_MODE_SAMPLING : PAI_MODE_COUNTING; 174 + mp->mapptr = cpump; 243 175 debug_sprintf_event(cfm_dbg, 5, "%s sample_period %#llx users %d" 244 176 " mode %d refcnt %u page %#lx save %p rc %d\n", 245 177 __func__, a->sample_period, cpump->active_events, 246 178 cpump->mode, refcount_read(&cpump->refcnt), 247 179 (unsigned long)cpump->page, cpump->save, rc); 180 + goto unlock; 181 + 182 + free_paicrypt_map: 183 + kfree(cpump); 184 + mp->mapptr = NULL; 185 + free_root: 186 + paicrypt_root_free(); 187 + 188 + unlock: 248 189 mutex_unlock(&pai_reserve_mutex); 249 - return rc; 190 + return rc ? ERR_PTR(rc) : cpump; 250 191 } 251 192 252 193 /* Might be called on different CPU than the one the event is intended for. */ ··· 260 189 { 261 190 struct perf_event_attr *a = &event->attr; 262 191 struct paicrypt_map *cpump; 263 - int rc; 264 192 265 193 /* PAI crypto PMU registered as PERF_TYPE_RAW, check event type */ 266 194 if (a->type != PERF_TYPE_RAW && event->pmu->type != a->type) ··· 269 199 a->config > PAI_CRYPTO_BASE + paicrypt_cnt) 270 200 return -EINVAL; 271 201 /* Allow only CPU wide operation, no process context for now. */ 272 - if (event->hw.target || event->cpu == -1) 202 + if ((event->attach_state & PERF_ATTACH_TASK) || event->cpu == -1) 273 203 return -ENOENT; 274 204 /* Allow only CRYPTO_ALL for sampling. */ 275 205 if (a->sample_period && a->config != PAI_CRYPTO_BASE) 276 206 return -EINVAL; 277 207 278 - cpump = per_cpu_ptr(&paicrypt_map, event->cpu); 279 - rc = paicrypt_busy(a, cpump); 280 - if (rc) 281 - return rc; 208 + cpump = paicrypt_busy(event); 209 + if (IS_ERR(cpump)) 210 + return PTR_ERR(cpump); 282 211 283 212 /* Event initialization sets last_tag to 0. When later on the events 284 213 * are deleted and re-added, do not reset the event count value to zero. ··· 285 216 * are active at the same time. 286 217 */ 287 218 event->hw.last_tag = 0; 288 - cpump->event = event; 289 219 event->destroy = paicrypt_event_destroy; 290 220 291 221 if (a->sample_period) { ··· 321 253 if (!event->hw.last_tag) { 322 254 event->hw.last_tag = 1; 323 255 sum = paicrypt_getall(event); /* Get current value */ 324 - local64_set(&event->count, 0); 325 256 local64_set(&event->hw.prev_count, sum); 326 257 } 327 258 } 328 259 329 260 static int paicrypt_add(struct perf_event *event, int flags) 330 261 { 331 - struct paicrypt_map *cpump = this_cpu_ptr(&paicrypt_map); 262 + struct paicrypt_mapptr *mp = this_cpu_ptr(paicrypt_root.mapptr); 263 + struct paicrypt_map *cpump = mp->mapptr; 332 264 unsigned long ccd; 333 265 334 266 if (++cpump->active_events == 1) { 335 267 ccd = virt_to_phys(cpump->page) | PAI_CRYPTO_KERNEL_OFFSET; 336 268 WRITE_ONCE(S390_lowcore.ccd, ccd); 337 - __ctl_set_bit(0, 50); 269 + local_ctl_set_bit(0, CR0_CRYPTOGRAPHY_COUNTER_BIT); 338 270 } 339 271 cpump->event = event; 340 272 if (flags & PERF_EF_START && !event->attr.sample_period) { ··· 355 287 356 288 static void paicrypt_del(struct perf_event *event, int flags) 357 289 { 358 - struct paicrypt_map *cpump = this_cpu_ptr(&paicrypt_map); 290 + struct paicrypt_mapptr *mp = this_cpu_ptr(paicrypt_root.mapptr); 291 + struct paicrypt_map *cpump = mp->mapptr; 359 292 360 293 if (event->attr.sample_period) 361 294 perf_sched_cb_dec(event->pmu); ··· 364 295 /* Only counting needs to read counter */ 365 296 paicrypt_stop(event, PERF_EF_UPDATE); 366 297 if (--cpump->active_events == 0) { 367 - __ctl_clear_bit(0, 50); 298 + local_ctl_clear_bit(0, CR0_CRYPTOGRAPHY_COUNTER_BIT); 368 299 WRITE_ONCE(S390_lowcore.ccd, 0); 369 300 } 370 301 } ··· 398 329 399 330 static int paicrypt_push_sample(void) 400 331 { 401 - struct paicrypt_map *cpump = this_cpu_ptr(&paicrypt_map); 332 + struct paicrypt_mapptr *mp = this_cpu_ptr(paicrypt_root.mapptr); 333 + struct paicrypt_map *cpump = mp->mapptr; 402 334 struct perf_event *event = cpump->event; 403 335 struct perf_sample_data data; 404 336 struct perf_raw_record raw;

+4 -6

arch/s390/kernel/perf_pai_ext.c

··· 17 17 #include <linux/export.h> 18 18 #include <linux/io.h> 19 19 #include <linux/perf_event.h> 20 - 21 - #include <asm/ctl_reg.h> 20 + #include <asm/ctlreg.h> 22 21 #include <asm/pai.h> 23 22 #include <asm/debug.h> 24 23 ··· 248 249 if (rc) 249 250 return rc; 250 251 /* Allow only CPU wide operation, no process context for now. */ 251 - if (event->hw.target || event->cpu == -1) 252 + if ((event->attach_state & PERF_ATTACH_TASK) || event->cpu == -1) 252 253 return -ENOENT; 253 254 /* Allow only event NNPA_ALL for sampling. */ 254 255 if (a->sample_period && a->config != PAI_NNPA_BASE) ··· 326 327 event->hw.last_tag = 1; 327 328 sum = paiext_getall(event); /* Get current value */ 328 329 local64_set(&event->hw.prev_count, sum); 329 - local64_set(&event->count, 0); 330 330 } 331 331 332 332 static int paiext_add(struct perf_event *event, int flags) ··· 338 340 S390_lowcore.aicd = virt_to_phys(cpump->paiext_cb); 339 341 pcb->acc = virt_to_phys(cpump->area) | 0x1; 340 342 /* Enable CPU instruction lookup for PAIE1 control block */ 341 - __ctl_set_bit(0, 49); 343 + local_ctl_set_bit(0, CR0_PAI_EXTENSION_BIT); 342 344 debug_sprintf_event(paiext_dbg, 4, "%s 1508 %llx acc %llx\n", 343 345 __func__, S390_lowcore.aicd, pcb->acc); 344 346 } ··· 374 376 } 375 377 if (--cpump->active_events == 0) { 376 378 /* Disable CPU instruction lookup for PAIE1 control block */ 377 - __ctl_clear_bit(0, 49); 379 + local_ctl_clear_bit(0, CR0_PAI_EXTENSION_BIT); 378 380 pcb->acc = 0; 379 381 S390_lowcore.aicd = 0; 380 382 debug_sprintf_event(paiext_dbg, 4, "%s 1508 %llx acc %llx\n",

+26 -19

arch/s390/kernel/ptrace.c

··· 41 41 { 42 42 struct pt_regs *regs = task_pt_regs(task); 43 43 struct thread_struct *thread = &task->thread; 44 - struct per_regs old, new; 45 44 union ctlreg0 cr0_old, cr0_new; 46 45 union ctlreg2 cr2_old, cr2_new; 47 46 int cr0_changed, cr2_changed; 47 + union { 48 + struct ctlreg regs[3]; 49 + struct { 50 + struct ctlreg control; 51 + struct ctlreg start; 52 + struct ctlreg end; 53 + }; 54 + } old, new; 48 55 49 - __ctl_store(cr0_old.val, 0, 0); 50 - __ctl_store(cr2_old.val, 2, 2); 56 + local_ctl_store(0, &cr0_old.reg); 57 + local_ctl_store(2, &cr2_old.reg); 51 58 cr0_new = cr0_old; 52 59 cr2_new = cr2_old; 53 60 /* Take care of the enable/disable of transactional execution. */ ··· 82 75 cr0_changed = cr0_new.val != cr0_old.val; 83 76 cr2_changed = cr2_new.val != cr2_old.val; 84 77 if (cr0_changed) 85 - __ctl_load(cr0_new.val, 0, 0); 78 + local_ctl_load(0, &cr0_new.reg); 86 79 if (cr2_changed) 87 - __ctl_load(cr2_new.val, 2, 2); 80 + local_ctl_load(2, &cr2_new.reg); 88 81 /* Copy user specified PER registers */ 89 - new.control = thread->per_user.control; 90 - new.start = thread->per_user.start; 91 - new.end = thread->per_user.end; 82 + new.control.val = thread->per_user.control; 83 + new.start.val = thread->per_user.start; 84 + new.end.val = thread->per_user.end; 92 85 93 86 /* merge TIF_SINGLE_STEP into user specified PER registers. */ 94 87 if (test_tsk_thread_flag(task, TIF_SINGLE_STEP) || 95 88 test_tsk_thread_flag(task, TIF_UPROBE_SINGLESTEP)) { 96 89 if (test_tsk_thread_flag(task, TIF_BLOCK_STEP)) 97 - new.control |= PER_EVENT_BRANCH; 90 + new.control.val |= PER_EVENT_BRANCH; 98 91 else 99 - new.control |= PER_EVENT_IFETCH; 100 - new.control |= PER_CONTROL_SUSPENSION; 101 - new.control |= PER_EVENT_TRANSACTION_END; 92 + new.control.val |= PER_EVENT_IFETCH; 93 + new.control.val |= PER_CONTROL_SUSPENSION; 94 + new.control.val |= PER_EVENT_TRANSACTION_END; 102 95 if (test_tsk_thread_flag(task, TIF_UPROBE_SINGLESTEP)) 103 - new.control |= PER_EVENT_IFETCH; 104 - new.start = 0; 105 - new.end = -1UL; 96 + new.control.val |= PER_EVENT_IFETCH; 97 + new.start.val = 0; 98 + new.end.val = -1UL; 106 99 } 107 100 108 101 /* Take care of the PER enablement bit in the PSW. */ 109 - if (!(new.control & PER_EVENT_MASK)) { 102 + if (!(new.control.val & PER_EVENT_MASK)) { 110 103 regs->psw.mask &= ~PSW_MASK_PER; 111 104 return; 112 105 } 113 106 regs->psw.mask |= PSW_MASK_PER; 114 - __ctl_store(old, 9, 11); 107 + __local_ctl_store(9, 11, old.regs); 115 108 if (memcmp(&new, &old, sizeof(struct per_regs)) != 0) 116 - __ctl_load(new, 9, 11); 109 + __local_ctl_load(9, 11, new.regs); 117 110 } 118 111 119 112 void user_enable_single_step(struct task_struct *task) ··· 1114 1107 target->thread.gs_cb = data; 1115 1108 *target->thread.gs_cb = gs_cb; 1116 1109 if (target == current) { 1117 - __ctl_set_bit(2, 4); 1110 + local_ctl_set_bit(2, CR2_GUARDED_STORAGE_BIT); 1118 1111 restore_gs_cb(target->thread.gs_cb); 1119 1112 } 1120 1113 preempt_enable();

+8 -15

arch/s390/kernel/setup.c

··· 305 305 return; 306 306 if (oldmem_data.start) 307 307 return; 308 - strcat(boot_command_line, " cio_ignore=all,!ipldev,!condev"); 308 + strlcat(boot_command_line, " cio_ignore=all,!ipldev,!condev", COMMAND_LINE_SIZE); 309 309 console_loglevel = 2; 310 310 } 311 311 #else ··· 381 381 #endif 382 382 } 383 383 384 - void __init __noreturn arch_call_rest_init(void) 385 - { 386 - smp_reinit_ipl_cpu(); 387 - rest_init(); 388 - } 389 - 390 384 static unsigned long __init stack_alloc_early(void) 391 385 { 392 386 unsigned long stack; ··· 449 455 lc->restart_fn = (unsigned long) do_restart; 450 456 lc->restart_data = 0; 451 457 lc->restart_source = -1U; 452 - __ctl_store(lc->cregs_save_area, 0, 15); 453 458 lc->spinlock_lockval = arch_spin_lockval(0); 454 459 lc->spinlock_index = 0; 455 460 arch_spin_lock_setup(0); ··· 458 465 lc->kernel_asce = S390_lowcore.kernel_asce; 459 466 lc->user_asce = S390_lowcore.user_asce; 460 467 468 + system_ctlreg_init_save_area(lc); 461 469 abs_lc = get_abs_lowcore(); 462 470 abs_lc->restart_stack = lc->restart_stack; 463 471 abs_lc->restart_fn = lc->restart_fn; ··· 466 472 abs_lc->restart_source = lc->restart_source; 467 473 abs_lc->restart_psw = lc->restart_psw; 468 474 abs_lc->restart_flags = RESTART_FLAG_CTLREGS; 469 - memcpy(abs_lc->cregs_save_area, lc->cregs_save_area, sizeof(abs_lc->cregs_save_area)); 470 475 abs_lc->program_new_psw = lc->program_new_psw; 471 476 abs_lc->mcesad = lc->mcesad; 472 477 put_abs_lowcore(abs_lc); ··· 790 797 __ctl_duct[4] = (unsigned long)__ctl_duald; 791 798 792 799 /* Update control registers CR2, CR5 and CR15 */ 793 - __ctl_store(cr2.val, 2, 2); 794 - __ctl_store(cr5.val, 5, 5); 795 - __ctl_store(cr15.val, 15, 15); 800 + local_ctl_store(2, &cr2.reg); 801 + local_ctl_store(5, &cr5.reg); 802 + local_ctl_store(15, &cr15.reg); 796 803 cr2.ducto = (unsigned long)__ctl_duct >> 6; 797 804 cr5.pasteo = (unsigned long)__ctl_duct >> 6; 798 805 cr15.lsea = (unsigned long)__ctl_linkage_stack >> 3; 799 - __ctl_load(cr2.val, 2, 2); 800 - __ctl_load(cr5.val, 5, 5); 801 - __ctl_load(cr15.val, 15, 15); 806 + system_ctl_load(2, &cr2.reg); 807 + system_ctl_load(5, &cr5.reg); 808 + system_ctl_load(15, &cr15.reg); 802 809 } 803 810 804 811 /*

+13 -119

arch/s390/kernel/smp.c

··· 37 37 #include <linux/crash_dump.h> 38 38 #include <linux/kprobes.h> 39 39 #include <asm/asm-offsets.h> 40 + #include <asm/ctlreg.h> 40 41 #include <asm/pfault.h> 41 42 #include <asm/diag.h> 42 43 #include <asm/switch_to.h> ··· 568 567 } 569 568 #endif 570 569 571 - /* 572 - * parameter area for the set/clear control bit callbacks 573 - */ 574 - struct ec_creg_mask_parms { 575 - unsigned long orval; 576 - unsigned long andval; 577 - int cr; 578 - }; 579 - 580 - /* 581 - * callback for setting/clearing control bits 582 - */ 583 - static void smp_ctl_bit_callback(void *info) 584 - { 585 - struct ec_creg_mask_parms *pp = info; 586 - unsigned long cregs[16]; 587 - 588 - __ctl_store(cregs, 0, 15); 589 - cregs[pp->cr] = (cregs[pp->cr] & pp->andval) | pp->orval; 590 - __ctl_load(cregs, 0, 15); 591 - } 592 - 593 - static DEFINE_SPINLOCK(ctl_lock); 594 - 595 - void smp_ctl_set_clear_bit(int cr, int bit, bool set) 596 - { 597 - struct ec_creg_mask_parms parms = { .cr = cr, }; 598 - struct lowcore *abs_lc; 599 - u64 ctlreg; 600 - 601 - if (set) { 602 - parms.orval = 1UL << bit; 603 - parms.andval = -1UL; 604 - } else { 605 - parms.orval = 0; 606 - parms.andval = ~(1UL << bit); 607 - } 608 - spin_lock(&ctl_lock); 609 - abs_lc = get_abs_lowcore(); 610 - ctlreg = abs_lc->cregs_save_area[cr]; 611 - ctlreg = (ctlreg & parms.andval) | parms.orval; 612 - abs_lc->cregs_save_area[cr] = ctlreg; 613 - put_abs_lowcore(abs_lc); 614 - on_each_cpu(smp_ctl_bit_callback, &parms, 1); 615 - spin_unlock(&ctl_lock); 616 - } 617 - EXPORT_SYMBOL(smp_ctl_set_clear_bit); 618 - 619 570 #ifdef CONFIG_CRASH_DUMP 620 571 621 572 int smp_store_status(int cpu) ··· 888 935 * Make sure global control register contents do not change 889 936 * until new CPU has initialized control registers. 890 937 */ 891 - spin_lock(&ctl_lock); 938 + system_ctlreg_lock(); 892 939 pcpu_prepare_secondary(pcpu, cpu); 893 940 pcpu_attach_task(pcpu, tidle); 894 941 pcpu_start_fn(pcpu, smp_start_secondary, NULL); 895 942 /* Wait until cpu puts itself in the online & active maps */ 896 943 while (!cpu_online(cpu)) 897 944 cpu_relax(); 898 - spin_unlock(&ctl_lock); 945 + system_ctlreg_unlock(); 899 946 return 0; 900 947 } 901 948 ··· 910 957 911 958 int __cpu_disable(void) 912 959 { 913 - unsigned long cregs[16]; 960 + struct ctlreg cregs[16]; 914 961 int cpu; 915 962 916 963 /* Handle possible pending IPIs */ ··· 922 969 /* Disable pseudo page faults on this cpu. */ 923 970 pfault_fini(); 924 971 /* Disable interrupt sources via control register. */ 925 - __ctl_store(cregs, 0, 15); 926 - cregs[0] &= ~0x0000ee70UL; /* disable all external interrupts */ 927 - cregs[6] &= ~0xff000000UL; /* disable all I/O interrupts */ 928 - cregs[14] &= ~0x1f000000UL; /* disable most machine checks */ 929 - __ctl_load(cregs, 0, 15); 972 + __local_ctl_store(0, 15, cregs); 973 + cregs[0].val &= ~0x0000ee70UL; /* disable all external interrupts */ 974 + cregs[6].val &= ~0xff000000UL; /* disable all I/O interrupts */ 975 + cregs[14].val &= ~0x1f000000UL; /* disable most machine checks */ 976 + __local_ctl_load(0, 15, cregs); 930 977 clear_cpu_flag(CIF_NOHZ_DELAY); 931 978 return 0; 932 979 } ··· 966 1013 967 1014 void __init smp_prepare_cpus(unsigned int max_cpus) 968 1015 { 969 - /* request the 0x1201 emergency signal external interrupt */ 970 1016 if (register_external_irq(EXT_IRQ_EMERGENCY_SIG, do_ext_call_interrupt)) 971 1017 panic("Couldn't request external interrupt 0x1201"); 972 - /* request the 0x1202 external call external interrupt */ 1018 + system_ctl_set_bit(0, 14); 973 1019 if (register_external_irq(EXT_IRQ_EXTERNAL_CALL, do_ext_call_interrupt)) 974 1020 panic("Couldn't request external interrupt 0x1202"); 1021 + system_ctl_set_bit(0, 13); 975 1022 } 976 1023 977 1024 void __init smp_prepare_boot_cpu(void) ··· 1029 1076 cpus_read_lock(); 1030 1077 mutex_lock(&smp_cpu_state_mutex); 1031 1078 rc = -EBUSY; 1032 - /* disallow configuration changes of online cpus and cpu 0 */ 1079 + /* disallow configuration changes of online cpus */ 1033 1080 cpu = dev->id; 1034 1081 cpu = smp_get_base_cpu(cpu); 1035 - if (cpu == 0) 1036 - goto out; 1037 1082 for (i = 0; i <= smp_cpu_mtid; i++) 1038 1083 if (cpu_online(cpu + i)) 1039 1084 goto out; ··· 1131 1180 return -ENOMEM; 1132 1181 per_cpu(cpu_device, cpu) = c; 1133 1182 s = &c->dev; 1134 - c->hotpluggable = 1; 1183 + c->hotpluggable = !!cpu; 1135 1184 rc = register_cpu(c, cpu); 1136 1185 if (rc) 1137 1186 goto out; ··· 1209 1258 return rc; 1210 1259 } 1211 1260 subsys_initcall(s390_smp_init); 1212 - 1213 - static __always_inline void set_new_lowcore(struct lowcore *lc) 1214 - { 1215 - union register_pair dst, src; 1216 - u32 pfx; 1217 - 1218 - src.even = (unsigned long) &S390_lowcore; 1219 - src.odd = sizeof(S390_lowcore); 1220 - dst.even = (unsigned long) lc; 1221 - dst.odd = sizeof(*lc); 1222 - pfx = __pa(lc); 1223 - 1224 - asm volatile( 1225 - " mvcl %[dst],%[src]\n" 1226 - " spx %[pfx]\n" 1227 - : [dst] "+&d" (dst.pair), [src] "+&d" (src.pair) 1228 - : [pfx] "Q" (pfx) 1229 - : "memory", "cc"); 1230 - } 1231 - 1232 - int __init smp_reinit_ipl_cpu(void) 1233 - { 1234 - unsigned long async_stack, nodat_stack, mcck_stack; 1235 - struct lowcore *lc, *lc_ipl; 1236 - unsigned long flags, cr0; 1237 - u64 mcesad; 1238 - 1239 - lc_ipl = lowcore_ptr[0]; 1240 - lc = (struct lowcore *) __get_free_pages(GFP_KERNEL | GFP_DMA, LC_ORDER); 1241 - nodat_stack = __get_free_pages(GFP_KERNEL, THREAD_SIZE_ORDER); 1242 - async_stack = stack_alloc(); 1243 - mcck_stack = stack_alloc(); 1244 - if (!lc || !nodat_stack || !async_stack || !mcck_stack || nmi_alloc_mcesa(&mcesad)) 1245 - panic("Couldn't allocate memory"); 1246 - 1247 - local_irq_save(flags); 1248 - local_mcck_disable(); 1249 - set_new_lowcore(lc); 1250 - S390_lowcore.nodat_stack = nodat_stack + STACK_INIT_OFFSET; 1251 - S390_lowcore.async_stack = async_stack + STACK_INIT_OFFSET; 1252 - S390_lowcore.mcck_stack = mcck_stack + STACK_INIT_OFFSET; 1253 - __ctl_store(cr0, 0, 0); 1254 - __ctl_clear_bit(0, 28); /* disable lowcore protection */ 1255 - S390_lowcore.mcesad = mcesad; 1256 - __ctl_load(cr0, 0, 0); 1257 - if (abs_lowcore_map(0, lc, false)) 1258 - panic("Couldn't remap absolute lowcore"); 1259 - lowcore_ptr[0] = lc; 1260 - local_mcck_enable(); 1261 - local_irq_restore(flags); 1262 - 1263 - memblock_free_late(__pa(lc_ipl->mcck_stack - STACK_INIT_OFFSET), THREAD_SIZE); 1264 - memblock_free_late(__pa(lc_ipl->async_stack - STACK_INIT_OFFSET), THREAD_SIZE); 1265 - memblock_free_late(__pa(lc_ipl->nodat_stack - STACK_INIT_OFFSET), THREAD_SIZE); 1266 - memblock_free_late(__pa(lc_ipl), sizeof(*lc_ipl)); 1267 - return 0; 1268 - }

+2 -2

arch/s390/kernel/time.c

··· 173 173 clockevents_register_device(cd); 174 174 175 175 /* Enable clock comparator timer interrupt. */ 176 - __ctl_set_bit(0,11); 176 + local_ctl_set_bit(0, CR0_CLOCK_COMPARATOR_SUBMASK_BIT); 177 177 178 178 /* Always allow the timing alert external interrupt. */ 179 - __ctl_set_bit(0, 4); 179 + local_ctl_set_bit(0, CR0_ETR_SUBMASK_BIT); 180 180 } 181 181 182 182 static void clock_comparator_interrupt(struct ext_code ext_code,

+14 -31

arch/s390/kvm/gaccess.c

··· 11 11 #include <linux/err.h> 12 12 #include <linux/pgtable.h> 13 13 #include <linux/bitfield.h> 14 - 14 + #include <asm/fault.h> 15 15 #include <asm/gmap.h> 16 16 #include "kvm-s390.h" 17 17 #include "gaccess.h" ··· 466 466 return 0; 467 467 } 468 468 469 - struct trans_exc_code_bits { 470 - unsigned long addr : 52; /* Translation-exception Address */ 471 - unsigned long fsi : 2; /* Access Exception Fetch/Store Indication */ 472 - unsigned long : 2; 473 - unsigned long b56 : 1; 474 - unsigned long : 3; 475 - unsigned long b60 : 1; 476 - unsigned long b61 : 1; 477 - unsigned long as : 2; /* ASCE Identifier */ 478 - }; 479 - 480 - enum { 481 - FSI_UNKNOWN = 0, /* Unknown whether fetch or store */ 482 - FSI_STORE = 1, /* Exception was due to store operation */ 483 - FSI_FETCH = 2 /* Exception was due to fetch operation */ 484 - }; 485 - 486 469 enum prot_type { 487 470 PROT_TYPE_LA = 0, 488 471 PROT_TYPE_KEYC = 1, ··· 480 497 enum gacc_mode mode, enum prot_type prot, bool terminate) 481 498 { 482 499 struct kvm_s390_pgm_info *pgm = &vcpu->arch.pgm; 483 - struct trans_exc_code_bits *tec; 500 + union teid *teid; 484 501 485 502 memset(pgm, 0, sizeof(*pgm)); 486 503 pgm->code = code; 487 - tec = (struct trans_exc_code_bits *)&pgm->trans_exc_code; 504 + teid = (union teid *)&pgm->trans_exc_code; 488 505 489 506 switch (code) { 490 507 case PGM_PROTECTION: ··· 494 511 WARN_ON_ONCE(1); 495 512 break; 496 513 case PROT_TYPE_IEP: 497 - tec->b61 = 1; 514 + teid->b61 = 1; 498 515 fallthrough; 499 516 case PROT_TYPE_LA: 500 - tec->b56 = 1; 517 + teid->b56 = 1; 501 518 break; 502 519 case PROT_TYPE_KEYC: 503 - tec->b60 = 1; 520 + teid->b60 = 1; 504 521 break; 505 522 case PROT_TYPE_ALC: 506 - tec->b60 = 1; 523 + teid->b60 = 1; 507 524 fallthrough; 508 525 case PROT_TYPE_DAT: 509 - tec->b61 = 1; 526 + teid->b61 = 1; 510 527 break; 511 528 } 512 529 if (terminate) { 513 - tec->b56 = 0; 514 - tec->b60 = 0; 515 - tec->b61 = 0; 530 + teid->b56 = 0; 531 + teid->b60 = 0; 532 + teid->b61 = 0; 516 533 } 517 534 fallthrough; 518 535 case PGM_ASCE_TYPE: ··· 526 543 * exc_access_id has to be set to 0 for some instructions. Both 527 544 * cases have to be handled by the caller. 528 545 */ 529 - tec->addr = gva >> PAGE_SHIFT; 530 - tec->fsi = mode == GACC_STORE ? FSI_STORE : FSI_FETCH; 531 - tec->as = psw_bits(vcpu->arch.sie_block->gpsw).as; 546 + teid->addr = gva >> PAGE_SHIFT; 547 + teid->fsi = mode == GACC_STORE ? TEID_FSI_STORE : TEID_FSI_FETCH; 548 + teid->as = psw_bits(vcpu->arch.sie_block->gpsw).as; 532 549 fallthrough; 533 550 case PGM_ALEN_TRANSLATION: 534 551 case PGM_ALE_SEQUENCE:

+3 -3

arch/s390/kvm/kvm-s390.c

··· 4927 4927 } 4928 4928 if (MACHINE_HAS_GS) { 4929 4929 preempt_disable(); 4930 - __ctl_set_bit(2, 4); 4930 + local_ctl_set_bit(2, CR2_GUARDED_STORAGE_BIT); 4931 4931 if (current->thread.gs_cb) { 4932 4932 vcpu->arch.host_gscb = current->thread.gs_cb; 4933 4933 save_gs_cb(vcpu->arch.host_gscb); ··· 5004 5004 kvm_run->s.regs.diag318 = vcpu->arch.diag318_info.val; 5005 5005 if (MACHINE_HAS_GS) { 5006 5006 preempt_disable(); 5007 - __ctl_set_bit(2, 4); 5007 + local_ctl_set_bit(2, CR2_GUARDED_STORAGE_BIT); 5008 5008 if (vcpu->arch.gs_enabled) 5009 5009 save_gs_cb(current->thread.gs_cb); 5010 5010 current->thread.gs_cb = vcpu->arch.host_gscb; 5011 5011 restore_gs_cb(vcpu->arch.host_gscb); 5012 5012 if (!vcpu->arch.host_gscb) 5013 - __ctl_clear_bit(2, 4); 5013 + local_ctl_clear_bit(2, CR2_GUARDED_STORAGE_BIT); 5014 5014 vcpu->arch.host_gscb = NULL; 5015 5015 preempt_enable(); 5016 5016 }

+1 -1

arch/s390/kvm/priv.c

··· 57 57 if (test_kvm_facility(vcpu->kvm, 133)) { 58 58 VCPU_EVENT(vcpu, 3, "%s", "ENABLE: GS (lazy)"); 59 59 preempt_disable(); 60 - __ctl_set_bit(2, 4); 60 + local_ctl_set_bit(2, CR2_GUARDED_STORAGE_BIT); 61 61 current->thread.gs_cb = (struct gs_cb *)&vcpu->run->s.regs.gscb; 62 62 restore_gs_cb(current->thread.gs_cb); 63 63 preempt_enable();

+8 -7

arch/s390/lib/uaccess.c

··· 12 12 #include <linux/export.h> 13 13 #include <linux/mm.h> 14 14 #include <asm/asm-extable.h> 15 + #include <asm/ctlreg.h> 15 16 16 17 #ifdef CONFIG_DEBUG_ENTRY 17 18 void debug_user_asce(int exit) 18 19 { 19 - unsigned long cr1, cr7; 20 + struct ctlreg cr1, cr7; 20 21 21 - __ctl_store(cr1, 1, 1); 22 - __ctl_store(cr7, 7, 7); 23 - if (cr1 == S390_lowcore.kernel_asce && cr7 == S390_lowcore.user_asce) 22 + local_ctl_store(1, &cr1); 23 + local_ctl_store(7, &cr7); 24 + if (cr1.val == S390_lowcore.kernel_asce.val && cr7.val == S390_lowcore.user_asce.val) 24 25 return; 25 26 panic("incorrect ASCE on kernel %s\n" 26 27 "cr1: %016lx cr7: %016lx\n" 27 - "kernel: %016llx user: %016llx\n", 28 - exit ? "exit" : "entry", cr1, cr7, 29 - S390_lowcore.kernel_asce, S390_lowcore.user_asce); 28 + "kernel: %016lx user: %016lx\n", 29 + exit ? "exit" : "entry", cr1.val, cr7.val, 30 + S390_lowcore.kernel_asce.val, S390_lowcore.user_asce.val); 30 31 } 31 32 #endif /*CONFIG_DEBUG_ENTRY */ 32 33

+1 -1

arch/s390/mm/dump_pagetables.c

··· 287 287 * kernel ASCE. We need this to keep the page table walker functions 288 288 * from accessing non-existent entries. 289 289 */ 290 - max_addr = (S390_lowcore.kernel_asce & _REGION_ENTRY_TYPE_MASK) >> 2; 290 + max_addr = (S390_lowcore.kernel_asce.val & _REGION_ENTRY_TYPE_MASK) >> 2; 291 291 max_addr = 1UL << (max_addr * 11 + 31); 292 292 address_markers[IDENTITY_AFTER_END_NR].start_address = ident_map_size; 293 293 address_markers[AMODE31_START_NR].start_address = (unsigned long)__samode31;

+18

arch/s390/mm/extable.c

··· 61 61 return true; 62 62 } 63 63 64 + static bool ex_handler_zeropad(const struct exception_table_entry *ex, struct pt_regs *regs) 65 + { 66 + unsigned int reg_addr = FIELD_GET(EX_DATA_REG_ADDR, ex->data); 67 + unsigned int reg_data = FIELD_GET(EX_DATA_REG_ERR, ex->data); 68 + unsigned long data, addr, offset; 69 + 70 + addr = regs->gprs[reg_addr]; 71 + offset = addr & (sizeof(unsigned long) - 1); 72 + addr &= ~(sizeof(unsigned long) - 1); 73 + data = *(unsigned long *)addr; 74 + data <<= BITS_PER_BYTE * offset; 75 + regs->gprs[reg_data] = data; 76 + regs->psw.addr = extable_fixup(ex); 77 + return true; 78 + } 79 + 64 80 bool fixup_exception(struct pt_regs *regs) 65 81 { 66 82 const struct exception_table_entry *ex; ··· 97 81 return ex_handler_ua_load_reg(ex, false, regs); 98 82 case EX_TYPE_UA_LOAD_REGPAIR: 99 83 return ex_handler_ua_load_reg(ex, true, regs); 84 + case EX_TYPE_ZEROPAD: 85 + return ex_handler_zeropad(ex, regs); 100 86 } 101 87 panic("invalid exception table entry"); 102 88 }

+173 -287

arch/s390/mm/fault.c

··· 3 3 * S390 version 4 4 * Copyright IBM Corp. 1999 5 5 * Author(s): Hartmut Penner (hp@de.ibm.com) 6 - * Ulrich Weigand (uweigand@de.ibm.com) 6 + * Ulrich Weigand (uweigand@de.ibm.com) 7 7 * 8 8 * Derived from "arch/i386/mm/fault.c" 9 9 * Copyright (C) 1995 Linus Torvalds 10 10 */ 11 11 12 12 #include <linux/kernel_stat.h> 13 + #include <linux/mmu_context.h> 13 14 #include <linux/perf_event.h> 14 15 #include <linux/signal.h> 15 16 #include <linux/sched.h> 16 17 #include <linux/sched/debug.h> 18 + #include <linux/jump_label.h> 17 19 #include <linux/kernel.h> 18 20 #include <linux/errno.h> 19 21 #include <linux/string.h> ··· 36 34 #include <linux/kfence.h> 37 35 #include <asm/asm-extable.h> 38 36 #include <asm/asm-offsets.h> 37 + #include <asm/ptrace.h> 38 + #include <asm/fault.h> 39 39 #include <asm/diag.h> 40 40 #include <asm/gmap.h> 41 41 #include <asm/irq.h> 42 - #include <asm/mmu_context.h> 43 42 #include <asm/facility.h> 44 43 #include <asm/uv.h> 45 44 #include "../kernel/entry.h" 46 - 47 - #define __FAIL_ADDR_MASK -4096L 48 - 49 - /* 50 - * Allocate private vm_fault_reason from top. Please make sure it won't 51 - * collide with vm_fault_reason. 52 - */ 53 - #define VM_FAULT_BADCONTEXT ((__force vm_fault_t)0x80000000) 54 - #define VM_FAULT_BADMAP ((__force vm_fault_t)0x40000000) 55 - #define VM_FAULT_BADACCESS ((__force vm_fault_t)0x20000000) 56 - #define VM_FAULT_SIGNAL ((__force vm_fault_t)0x10000000) 57 - #define VM_FAULT_PFAULT ((__force vm_fault_t)0x8000000) 58 45 59 46 enum fault_type { 60 47 KERNEL_FAULT, ··· 51 60 GMAP_FAULT, 52 61 }; 53 62 54 - static unsigned long store_indication __read_mostly; 63 + static DEFINE_STATIC_KEY_FALSE(have_store_indication); 55 64 56 65 static int __init fault_init(void) 57 66 { 58 67 if (test_facility(75)) 59 - store_indication = 0xc00; 68 + static_branch_enable(&have_store_indication); 60 69 return 0; 61 70 } 62 71 early_initcall(fault_init); ··· 66 75 */ 67 76 static enum fault_type get_fault_type(struct pt_regs *regs) 68 77 { 69 - unsigned long trans_exc_code; 78 + union teid teid = { .val = regs->int_parm_long }; 70 79 71 - trans_exc_code = regs->int_parm_long & 3; 72 - if (likely(trans_exc_code == 0)) { 73 - /* primary space exception */ 80 + if (likely(teid.as == PSW_BITS_AS_PRIMARY)) { 74 81 if (user_mode(regs)) 75 82 return USER_FAULT; 76 83 if (!IS_ENABLED(CONFIG_PGSTE)) ··· 77 88 return GMAP_FAULT; 78 89 return KERNEL_FAULT; 79 90 } 80 - if (trans_exc_code == 2) 91 + if (teid.as == PSW_BITS_AS_SECONDARY) 81 92 return USER_FAULT; 82 - if (trans_exc_code == 1) { 83 - /* access register mode, not used in the kernel */ 93 + /* Access register mode, not used in the kernel */ 94 + if (teid.as == PSW_BITS_AS_ACCREG) 84 95 return USER_FAULT; 85 - } 86 - /* home space exception -> access via kernel ASCE */ 96 + /* Home space -> access via kernel ASCE */ 87 97 return KERNEL_FAULT; 88 98 } 89 99 90 100 static unsigned long get_fault_address(struct pt_regs *regs) 91 101 { 92 - unsigned long trans_exc_code = regs->int_parm_long; 102 + union teid teid = { .val = regs->int_parm_long }; 93 103 94 - return trans_exc_code & __FAIL_ADDR_MASK; 104 + return teid.addr * PAGE_SIZE; 95 105 } 96 106 97 - static bool fault_is_write(struct pt_regs *regs) 107 + static __always_inline bool fault_is_write(struct pt_regs *regs) 98 108 { 99 - unsigned long trans_exc_code = regs->int_parm_long; 109 + union teid teid = { .val = regs->int_parm_long }; 100 110 101 - return (trans_exc_code & store_indication) == 0x400; 102 - } 103 - 104 - static int bad_address(void *p) 105 - { 106 - unsigned long dummy; 107 - 108 - return get_kernel_nofault(dummy, (unsigned long *)p); 111 + if (static_branch_likely(&have_store_indication)) 112 + return teid.fsi == TEID_FSI_STORE; 113 + return false; 109 114 } 110 115 111 116 static void dump_pagetable(unsigned long asce, unsigned long address) 112 117 { 113 - unsigned long *table = __va(asce & _ASCE_ORIGIN); 118 + unsigned long entry, *table = __va(asce & _ASCE_ORIGIN); 114 119 115 120 pr_alert("AS:%016lx ", asce); 116 121 switch (asce & _ASCE_TYPE_MASK) { 117 122 case _ASCE_TYPE_REGION1: 118 123 table += (address & _REGION1_INDEX) >> _REGION1_SHIFT; 119 - if (bad_address(table)) 124 + if (get_kernel_nofault(entry, table)) 120 125 goto bad; 121 - pr_cont("R1:%016lx ", *table); 122 - if (*table & _REGION_ENTRY_INVALID) 126 + pr_cont("R1:%016lx ", entry); 127 + if (entry & _REGION_ENTRY_INVALID) 123 128 goto out; 124 - table = __va(*table & _REGION_ENTRY_ORIGIN); 129 + table = __va(entry & _REGION_ENTRY_ORIGIN); 125 130 fallthrough; 126 131 case _ASCE_TYPE_REGION2: 127 132 table += (address & _REGION2_INDEX) >> _REGION2_SHIFT; 128 - if (bad_address(table)) 133 + if (get_kernel_nofault(entry, table)) 129 134 goto bad; 130 - pr_cont("R2:%016lx ", *table); 131 - if (*table & _REGION_ENTRY_INVALID) 135 + pr_cont("R2:%016lx ", entry); 136 + if (entry & _REGION_ENTRY_INVALID) 132 137 goto out; 133 - table = __va(*table & _REGION_ENTRY_ORIGIN); 138 + table = __va(entry & _REGION_ENTRY_ORIGIN); 134 139 fallthrough; 135 140 case _ASCE_TYPE_REGION3: 136 141 table += (address & _REGION3_INDEX) >> _REGION3_SHIFT; 137 - if (bad_address(table)) 142 + if (get_kernel_nofault(entry, table)) 138 143 goto bad; 139 - pr_cont("R3:%016lx ", *table); 140 - if (*table & (_REGION_ENTRY_INVALID | _REGION3_ENTRY_LARGE)) 144 + pr_cont("R3:%016lx ", entry); 145 + if (entry & (_REGION_ENTRY_INVALID | _REGION3_ENTRY_LARGE)) 141 146 goto out; 142 - table = __va(*table & _REGION_ENTRY_ORIGIN); 147 + table = __va(entry & _REGION_ENTRY_ORIGIN); 143 148 fallthrough; 144 149 case _ASCE_TYPE_SEGMENT: 145 150 table += (address & _SEGMENT_INDEX) >> _SEGMENT_SHIFT; 146 - if (bad_address(table)) 151 + if (get_kernel_nofault(entry, table)) 147 152 goto bad; 148 - pr_cont("S:%016lx ", *table); 149 - if (*table & (_SEGMENT_ENTRY_INVALID | _SEGMENT_ENTRY_LARGE)) 153 + pr_cont("S:%016lx ", entry); 154 + if (entry & (_SEGMENT_ENTRY_INVALID | _SEGMENT_ENTRY_LARGE)) 150 155 goto out; 151 - table = __va(*table & _SEGMENT_ENTRY_ORIGIN); 156 + table = __va(entry & _SEGMENT_ENTRY_ORIGIN); 152 157 } 153 158 table += (address & _PAGE_INDEX) >> _PAGE_SHIFT; 154 - if (bad_address(table)) 159 + if (get_kernel_nofault(entry, table)) 155 160 goto bad; 156 - pr_cont("P:%016lx ", *table); 161 + pr_cont("P:%016lx ", entry); 157 162 out: 158 163 pr_cont("\n"); 159 164 return; ··· 157 174 158 175 static void dump_fault_info(struct pt_regs *regs) 159 176 { 177 + union teid teid = { .val = regs->int_parm_long }; 160 178 unsigned long asce; 161 179 162 180 pr_alert("Failing address: %016lx TEID: %016lx\n", 163 - regs->int_parm_long & __FAIL_ADDR_MASK, regs->int_parm_long); 181 + get_fault_address(regs), teid.val); 164 182 pr_alert("Fault in "); 165 - switch (regs->int_parm_long & 3) { 166 - case 3: 183 + switch (teid.as) { 184 + case PSW_BITS_AS_HOME: 167 185 pr_cont("home space "); 168 186 break; 169 - case 2: 187 + case PSW_BITS_AS_SECONDARY: 170 188 pr_cont("secondary space "); 171 189 break; 172 - case 1: 190 + case PSW_BITS_AS_ACCREG: 173 191 pr_cont("access register "); 174 192 break; 175 - case 0: 193 + case PSW_BITS_AS_PRIMARY: 176 194 pr_cont("primary space "); 177 195 break; 178 196 } 179 197 pr_cont("mode while using "); 180 198 switch (get_fault_type(regs)) { 181 199 case USER_FAULT: 182 - asce = S390_lowcore.user_asce; 200 + asce = S390_lowcore.user_asce.val; 183 201 pr_cont("user "); 184 202 break; 185 203 case GMAP_FAULT: 186 - asce = ((struct gmap *) S390_lowcore.gmap)->asce; 204 + asce = ((struct gmap *)S390_lowcore.gmap)->asce; 187 205 pr_cont("gmap "); 188 206 break; 189 207 case KERNEL_FAULT: 190 - asce = S390_lowcore.kernel_asce; 208 + asce = S390_lowcore.kernel_asce.val; 191 209 pr_cont("kernel "); 192 210 break; 193 211 default: 194 212 unreachable(); 195 213 } 196 214 pr_cont("ASCE.\n"); 197 - dump_pagetable(asce, regs->int_parm_long & __FAIL_ADDR_MASK); 215 + dump_pagetable(asce, get_fault_address(regs)); 198 216 } 199 217 200 218 int show_unhandled_signals = 1; 201 219 202 220 void report_user_fault(struct pt_regs *regs, long signr, int is_mm_fault) 203 221 { 222 + static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL, DEFAULT_RATELIMIT_BURST); 223 + 204 224 if ((task_pid_nr(current) > 1) && !show_unhandled_signals) 205 225 return; 206 226 if (!unhandled_signal(current, signr)) 207 227 return; 208 - if (!printk_ratelimit()) 228 + if (!__ratelimit(&rs)) 209 229 return; 210 - printk(KERN_ALERT "User process fault: interruption code %04x ilc:%d ", 211 - regs->int_code & 0xffff, regs->int_code >> 17); 230 + pr_alert("User process fault: interruption code %04x ilc:%d ", 231 + regs->int_code & 0xffff, regs->int_code >> 17); 212 232 print_vma_addr(KERN_CONT "in ", regs->psw.addr); 213 - printk(KERN_CONT "\n"); 233 + pr_cont("\n"); 214 234 if (is_mm_fault) 215 235 dump_fault_info(regs); 216 236 show_regs(regs); 217 237 } 218 238 219 - /* 220 - * Send SIGSEGV to task. This is an external routine 221 - * to keep the stack usage of do_page_fault small. 222 - */ 223 - static noinline void do_sigsegv(struct pt_regs *regs, int si_code) 239 + static void do_sigsegv(struct pt_regs *regs, int si_code) 224 240 { 225 241 report_user_fault(regs, SIGSEGV, 1); 226 - force_sig_fault(SIGSEGV, si_code, 227 - (void __user *)(regs->int_parm_long & __FAIL_ADDR_MASK)); 242 + force_sig_fault(SIGSEGV, si_code, (void __user *)get_fault_address(regs)); 228 243 } 229 244 230 - static noinline void do_no_context(struct pt_regs *regs, vm_fault_t fault) 245 + static void handle_fault_error_nolock(struct pt_regs *regs, int si_code) 231 246 { 232 247 enum fault_type fault_type; 233 248 unsigned long address; 234 249 bool is_write; 235 250 251 + if (user_mode(regs)) { 252 + if (WARN_ON_ONCE(!si_code)) 253 + si_code = SEGV_MAPERR; 254 + return do_sigsegv(regs, si_code); 255 + } 236 256 if (fixup_exception(regs)) 237 257 return; 238 258 fault_type = get_fault_type(regs); 239 - if ((fault_type == KERNEL_FAULT) && (fault == VM_FAULT_BADCONTEXT)) { 259 + if (fault_type == KERNEL_FAULT) { 240 260 address = get_fault_address(regs); 241 261 is_write = fault_is_write(regs); 242 262 if (kfence_handle_page_fault(address, is_write, regs)) 243 263 return; 244 264 } 245 - /* 246 - * Oops. The kernel tried to access some bad page. We'll have to 247 - * terminate things with extreme prejudice. 248 - */ 249 265 if (fault_type == KERNEL_FAULT) 250 - printk(KERN_ALERT "Unable to handle kernel pointer dereference" 251 - " in virtual kernel address space\n"); 266 + pr_alert("Unable to handle kernel pointer dereference in virtual kernel address space\n"); 252 267 else 253 - printk(KERN_ALERT "Unable to handle kernel paging request" 254 - " in virtual user address space\n"); 268 + pr_alert("Unable to handle kernel paging request in virtual user address space\n"); 255 269 dump_fault_info(regs); 256 270 die(regs, "Oops"); 257 271 } 258 272 259 - static noinline void do_low_address(struct pt_regs *regs) 273 + static void handle_fault_error(struct pt_regs *regs, int si_code) 260 274 { 261 - /* Low-address protection hit in kernel mode means 262 - NULL pointer write access in kernel mode. */ 263 - if (regs->psw.mask & PSW_MASK_PSTATE) { 264 - /* Low-address protection hit in user mode 'cannot happen'. */ 265 - die (regs, "Low-address protection"); 266 - } 275 + struct mm_struct *mm = current->mm; 267 276 268 - do_no_context(regs, VM_FAULT_BADACCESS); 277 + mmap_read_unlock(mm); 278 + handle_fault_error_nolock(regs, si_code); 269 279 } 270 280 271 - static noinline void do_sigbus(struct pt_regs *regs) 281 + static void do_sigbus(struct pt_regs *regs) 272 282 { 273 - /* 274 - * Send a sigbus, regardless of whether we were in kernel 275 - * or user mode. 276 - */ 277 - force_sig_fault(SIGBUS, BUS_ADRERR, 278 - (void __user *)(regs->int_parm_long & __FAIL_ADDR_MASK)); 279 - } 280 - 281 - static noinline void do_fault_error(struct pt_regs *regs, vm_fault_t fault) 282 - { 283 - int si_code; 284 - 285 - switch (fault) { 286 - case VM_FAULT_BADACCESS: 287 - case VM_FAULT_BADMAP: 288 - /* Bad memory access. Check if it is kernel or user space. */ 289 - if (user_mode(regs)) { 290 - /* User mode accesses just cause a SIGSEGV */ 291 - si_code = (fault == VM_FAULT_BADMAP) ? 292 - SEGV_MAPERR : SEGV_ACCERR; 293 - do_sigsegv(regs, si_code); 294 - break; 295 - } 296 - fallthrough; 297 - case VM_FAULT_BADCONTEXT: 298 - case VM_FAULT_PFAULT: 299 - do_no_context(regs, fault); 300 - break; 301 - case VM_FAULT_SIGNAL: 302 - if (!user_mode(regs)) 303 - do_no_context(regs, fault); 304 - break; 305 - default: /* fault & VM_FAULT_ERROR */ 306 - if (fault & VM_FAULT_OOM) { 307 - if (!user_mode(regs)) 308 - do_no_context(regs, fault); 309 - else 310 - pagefault_out_of_memory(); 311 - } else if (fault & VM_FAULT_SIGSEGV) { 312 - /* Kernel mode? Handle exceptions or die */ 313 - if (!user_mode(regs)) 314 - do_no_context(regs, fault); 315 - else 316 - do_sigsegv(regs, SEGV_MAPERR); 317 - } else if (fault & VM_FAULT_SIGBUS) { 318 - /* Kernel mode? Handle exceptions or die */ 319 - if (!user_mode(regs)) 320 - do_no_context(regs, fault); 321 - else 322 - do_sigbus(regs); 323 - } else 324 - BUG(); 325 - break; 326 - } 283 + force_sig_fault(SIGBUS, BUS_ADRERR, (void __user *)get_fault_address(regs)); 327 284 } 328 285 329 286 /* ··· 272 349 * routines. 273 350 * 274 351 * interruption code (int_code): 275 - * 04 Protection -> Write-Protection (suppression) 276 - * 10 Segment translation -> Not present (nullification) 277 - * 11 Page translation -> Not present (nullification) 278 - * 3b Region third trans. -> Not present (nullification) 352 + * 04 Protection -> Write-Protection (suppression) 353 + * 10 Segment translation -> Not present (nullification) 354 + * 11 Page translation -> Not present (nullification) 355 + * 3b Region third trans. -> Not present (nullification) 279 356 */ 280 - static inline vm_fault_t do_exception(struct pt_regs *regs, int access) 357 + static void do_exception(struct pt_regs *regs, int access) 281 358 { 282 - struct gmap *gmap; 283 - struct task_struct *tsk; 284 - struct mm_struct *mm; 285 359 struct vm_area_struct *vma; 286 - enum fault_type type; 360 + struct task_struct *tsk; 287 361 unsigned long address; 362 + struct mm_struct *mm; 363 + enum fault_type type; 288 364 unsigned int flags; 365 + struct gmap *gmap; 289 366 vm_fault_t fault; 290 367 bool is_write; 291 368 ··· 295 372 * been nullified. Don't signal single step via SIGTRAP. 296 373 */ 297 374 clear_thread_flag(TIF_PER_TRAP); 298 - 299 375 if (kprobe_page_fault(regs, 14)) 300 - return 0; 301 - 376 + return; 302 377 mm = tsk->mm; 303 378 address = get_fault_address(regs); 304 379 is_write = fault_is_write(regs); 305 - 306 - /* 307 - * Verify that the fault happened in user space, that 308 - * we are not in an interrupt and that there is a 309 - * user context. 310 - */ 311 - fault = VM_FAULT_BADCONTEXT; 312 380 type = get_fault_type(regs); 313 381 switch (type) { 314 382 case KERNEL_FAULT: 315 - goto out; 383 + return handle_fault_error_nolock(regs, 0); 316 384 case USER_FAULT: 317 385 case GMAP_FAULT: 318 386 if (faulthandler_disabled() || !mm) 319 - goto out; 387 + return handle_fault_error_nolock(regs, 0); 320 388 break; 321 389 } 322 - 323 390 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address); 324 391 flags = FAULT_FLAG_DEFAULT; 325 392 if (user_mode(regs)) ··· 332 419 vma_end_read(vma); 333 420 if (!(fault & VM_FAULT_RETRY)) { 334 421 count_vm_vma_lock_event(VMA_LOCK_SUCCESS); 335 - if (likely(!(fault & VM_FAULT_ERROR))) 336 - fault = 0; 337 - goto out; 422 + if (unlikely(fault & VM_FAULT_ERROR)) 423 + goto error; 424 + return; 338 425 } 339 426 count_vm_vma_lock_event(VMA_LOCK_RETRY); 340 427 /* Quick path to respond to signals */ 341 428 if (fault_signal_pending(fault, regs)) { 342 - fault = VM_FAULT_SIGNAL; 343 - goto out; 429 + if (!user_mode(regs)) 430 + handle_fault_error_nolock(regs, 0); 431 + return; 344 432 } 345 433 lock_mmap: 346 434 mmap_read_lock(mm); 347 - 348 435 gmap = NULL; 349 436 if (IS_ENABLED(CONFIG_PGSTE) && type == GMAP_FAULT) { 350 - gmap = (struct gmap *) S390_lowcore.gmap; 437 + gmap = (struct gmap *)S390_lowcore.gmap; 351 438 current->thread.gmap_addr = address; 352 439 current->thread.gmap_write_flag = !!(flags & FAULT_FLAG_WRITE); 353 440 current->thread.gmap_int_code = regs->int_code & 0xffff; 354 441 address = __gmap_translate(gmap, address); 355 - if (address == -EFAULT) { 356 - fault = VM_FAULT_BADMAP; 357 - goto out_up; 358 - } 442 + if (address == -EFAULT) 443 + return handle_fault_error(regs, SEGV_MAPERR); 359 444 if (gmap->pfault_enabled) 360 445 flags |= FAULT_FLAG_RETRY_NOWAIT; 361 446 } 362 - 363 447 retry: 364 - fault = VM_FAULT_BADMAP; 365 448 vma = find_vma(mm, address); 366 449 if (!vma) 367 - goto out_up; 368 - 450 + return handle_fault_error(regs, SEGV_MAPERR); 369 451 if (unlikely(vma->vm_start > address)) { 370 452 if (!(vma->vm_flags & VM_GROWSDOWN)) 371 - goto out_up; 453 + return handle_fault_error(regs, SEGV_MAPERR); 372 454 vma = expand_stack(mm, address); 373 455 if (!vma) 374 - goto out; 456 + return handle_fault_error_nolock(regs, SEGV_MAPERR); 375 457 } 376 - 377 - /* 378 - * Ok, we have a good vm_area for this memory access, so 379 - * we can handle it.. 380 - */ 381 - fault = VM_FAULT_BADACCESS; 382 458 if (unlikely(!(vma->vm_flags & access))) 383 - goto out_up; 384 - 385 - /* 386 - * If for any reason at all we couldn't handle the fault, 387 - * make sure we exit gracefully rather than endlessly redo 388 - * the fault. 389 - */ 459 + return handle_fault_error(regs, SEGV_ACCERR); 390 460 fault = handle_mm_fault(vma, address, flags, regs); 391 461 if (fault_signal_pending(fault, regs)) { 392 - fault = VM_FAULT_SIGNAL; 393 462 if (flags & FAULT_FLAG_RETRY_NOWAIT) 394 - goto out_up; 395 - goto out; 463 + mmap_read_unlock(mm); 464 + if (!user_mode(regs)) 465 + handle_fault_error_nolock(regs, 0); 466 + return; 396 467 } 397 - 398 468 /* The fault is fully completed (including releasing mmap lock) */ 399 469 if (fault & VM_FAULT_COMPLETED) { 400 470 if (gmap) { 401 471 mmap_read_lock(mm); 402 - goto out_gmap; 472 + goto gmap; 403 473 } 404 - fault = 0; 405 - goto out; 474 + return; 406 475 } 407 - 408 - if (unlikely(fault & VM_FAULT_ERROR)) 409 - goto out_up; 410 - 476 + if (unlikely(fault & VM_FAULT_ERROR)) { 477 + mmap_read_unlock(mm); 478 + goto error; 479 + } 411 480 if (fault & VM_FAULT_RETRY) { 412 - if (IS_ENABLED(CONFIG_PGSTE) && gmap && 413 - (flags & FAULT_FLAG_RETRY_NOWAIT)) { 481 + if (IS_ENABLED(CONFIG_PGSTE) && gmap && (flags & FAULT_FLAG_RETRY_NOWAIT)) { 414 482 /* 415 - * FAULT_FLAG_RETRY_NOWAIT has been set, mmap_lock has 416 - * not been released 483 + * FAULT_FLAG_RETRY_NOWAIT has been set, 484 + * mmap_lock has not been released 417 485 */ 418 486 current->thread.gmap_pfault = 1; 419 - fault = VM_FAULT_PFAULT; 420 - goto out_up; 487 + return handle_fault_error(regs, 0); 421 488 } 422 489 flags &= ~FAULT_FLAG_RETRY_NOWAIT; 423 490 flags |= FAULT_FLAG_TRIED; 424 491 mmap_read_lock(mm); 425 492 goto retry; 426 493 } 427 - out_gmap: 494 + gmap: 428 495 if (IS_ENABLED(CONFIG_PGSTE) && gmap) { 429 496 address = __gmap_link(gmap, current->thread.gmap_addr, 430 497 address); 431 - if (address == -EFAULT) { 432 - fault = VM_FAULT_BADMAP; 433 - goto out_up; 434 - } 498 + if (address == -EFAULT) 499 + return handle_fault_error(regs, SEGV_MAPERR); 435 500 if (address == -ENOMEM) { 436 501 fault = VM_FAULT_OOM; 437 - goto out_up; 502 + mmap_read_unlock(mm); 503 + goto error; 438 504 } 439 505 } 440 - fault = 0; 441 - out_up: 442 506 mmap_read_unlock(mm); 443 - out: 444 - return fault; 507 + return; 508 + error: 509 + if (fault & VM_FAULT_OOM) { 510 + if (!user_mode(regs)) 511 + handle_fault_error_nolock(regs, 0); 512 + else 513 + pagefault_out_of_memory(); 514 + } else if (fault & VM_FAULT_SIGSEGV) { 515 + if (!user_mode(regs)) 516 + handle_fault_error_nolock(regs, 0); 517 + else 518 + do_sigsegv(regs, SEGV_MAPERR); 519 + } else if (fault & VM_FAULT_SIGBUS) { 520 + if (!user_mode(regs)) 521 + handle_fault_error_nolock(regs, 0); 522 + else 523 + do_sigbus(regs); 524 + } else { 525 + BUG(); 526 + } 445 527 } 446 528 447 529 void do_protection_exception(struct pt_regs *regs) 448 530 { 449 - unsigned long trans_exc_code; 450 - int access; 451 - vm_fault_t fault; 531 + union teid teid = { .val = regs->int_parm_long }; 452 532 453 - trans_exc_code = regs->int_parm_long; 454 533 /* 455 534 * Protection exceptions are suppressing, decrement psw address. 456 535 * The exception to this rule are aborted transactions, for these ··· 455 550 * as a special case because the translation exception code 456 551 * field is not guaranteed to contain valid data in this case. 457 552 */ 458 - if (unlikely(!(trans_exc_code & 4))) { 459 - do_low_address(regs); 460 - return; 553 + if (unlikely(!teid.b61)) { 554 + if (user_mode(regs)) { 555 + /* Low-address protection in user mode: cannot happen */ 556 + die(regs, "Low-address protection"); 557 + } 558 + /* 559 + * Low-address protection in kernel mode means 560 + * NULL pointer write access in kernel mode. 561 + */ 562 + return handle_fault_error_nolock(regs, 0); 461 563 } 462 - if (unlikely(MACHINE_HAS_NX && (trans_exc_code & 0x80))) { 463 - regs->int_parm_long = (trans_exc_code & ~PAGE_MASK) | 464 - (regs->psw.addr & PAGE_MASK); 465 - access = VM_EXEC; 466 - fault = VM_FAULT_BADACCESS; 467 - } else { 468 - access = VM_WRITE; 469 - fault = do_exception(regs, access); 564 + if (unlikely(MACHINE_HAS_NX && teid.b56)) { 565 + regs->int_parm_long = (teid.addr * PAGE_SIZE) | (regs->psw.addr & PAGE_MASK); 566 + return handle_fault_error_nolock(regs, SEGV_ACCERR); 470 567 } 471 - if (unlikely(fault)) 472 - do_fault_error(regs, fault); 568 + do_exception(regs, VM_WRITE); 473 569 } 474 570 NOKPROBE_SYMBOL(do_protection_exception); 475 571 476 572 void do_dat_exception(struct pt_regs *regs) 477 573 { 478 - int access; 479 - vm_fault_t fault; 480 - 481 - access = VM_ACCESS_FLAGS; 482 - fault = do_exception(regs, access); 483 - if (unlikely(fault)) 484 - do_fault_error(regs, fault); 574 + do_exception(regs, VM_ACCESS_FLAGS); 485 575 } 486 576 NOKPROBE_SYMBOL(do_dat_exception); 487 577 ··· 484 584 485 585 void do_secure_storage_access(struct pt_regs *regs) 486 586 { 487 - unsigned long addr = regs->int_parm_long & __FAIL_ADDR_MASK; 587 + union teid teid = { .val = regs->int_parm_long }; 588 + unsigned long addr = get_fault_address(regs); 488 589 struct vm_area_struct *vma; 489 590 struct mm_struct *mm; 490 591 struct page *page; ··· 493 592 int rc; 494 593 495 594 /* 496 - * bit 61 tells us if the address is valid, if it's not we 497 - * have a major problem and should stop the kernel or send a 498 - * SIGSEGV to the process. Unfortunately bit 61 is not 499 - * reliable without the misc UV feature so we need to check 500 - * for that as well. 595 + * Bit 61 indicates if the address is valid, if it is not the 596 + * kernel should be stopped or SIGSEGV should be sent to the 597 + * process. Bit 61 is not reliable without the misc UV feature, 598 + * therefore this needs to be checked too. 501 599 */ 502 - if (uv_has_feature(BIT_UV_FEAT_MISC) && 503 - !test_bit_inv(61, &regs->int_parm_long)) { 600 + if (uv_has_feature(BIT_UV_FEAT_MISC) && !teid.b61) { 504 601 /* 505 602 * When this happens, userspace did something that it 506 603 * was not supposed to do, e.g. branching into secure ··· 508 609 send_sig(SIGSEGV, current, 0); 509 610 return; 510 611 } 511 - 512 612 /* 513 - * The kernel should never run into this case and we 514 - * have no way out of this situation. 613 + * The kernel should never run into this case and 614 + * there is no way out of this situation. 515 615 */ 516 616 panic("Unexpected PGM 0x3d with TEID bit 61=0"); 517 617 } 518 - 519 618 switch (get_fault_type(regs)) { 520 619 case GMAP_FAULT: 521 620 mm = current->mm; ··· 521 624 mmap_read_lock(mm); 522 625 addr = __gmap_translate(gmap, addr); 523 626 mmap_read_unlock(mm); 524 - if (IS_ERR_VALUE(addr)) { 525 - do_fault_error(regs, VM_FAULT_BADMAP); 526 - break; 527 - } 627 + if (IS_ERR_VALUE(addr)) 628 + return handle_fault_error_nolock(regs, SEGV_MAPERR); 528 629 fallthrough; 529 630 case USER_FAULT: 530 631 mm = current->mm; 531 632 mmap_read_lock(mm); 532 633 vma = find_vma(mm, addr); 533 - if (!vma) { 534 - mmap_read_unlock(mm); 535 - do_fault_error(regs, VM_FAULT_BADMAP); 536 - break; 537 - } 634 + if (!vma) 635 + return handle_fault_error(regs, SEGV_MAPERR); 538 636 page = follow_page(vma, addr, FOLL_WRITE | FOLL_GET); 539 637 if (IS_ERR_OR_NULL(page)) { 540 638 mmap_read_unlock(mm); ··· 550 658 BUG(); 551 659 break; 552 660 default: 553 - do_fault_error(regs, VM_FAULT_BADMAP); 554 - WARN_ON_ONCE(1); 661 + unreachable(); 555 662 } 556 663 } 557 664 NOKPROBE_SYMBOL(do_secure_storage_access); 558 665 559 666 void do_non_secure_storage_access(struct pt_regs *regs) 560 667 { 561 - unsigned long gaddr = regs->int_parm_long & __FAIL_ADDR_MASK; 562 668 struct gmap *gmap = (struct gmap *)S390_lowcore.gmap; 669 + unsigned long gaddr = get_fault_address(regs); 563 670 564 - if (get_fault_type(regs) != GMAP_FAULT) { 565 - do_fault_error(regs, VM_FAULT_BADMAP); 566 - WARN_ON_ONCE(1); 567 - return; 568 - } 569 - 671 + if (WARN_ON_ONCE(get_fault_type(regs) != GMAP_FAULT)) 672 + return handle_fault_error_nolock(regs, SEGV_MAPERR); 570 673 if (gmap_convert_to_secure(gmap, gaddr) == -EINVAL) 571 674 send_sig(SIGSEGV, current, 0); 572 675 } ··· 569 682 570 683 void do_secure_storage_violation(struct pt_regs *regs) 571 684 { 572 - unsigned long gaddr = regs->int_parm_long & __FAIL_ADDR_MASK; 573 685 struct gmap *gmap = (struct gmap *)S390_lowcore.gmap; 686 + unsigned long gaddr = get_fault_address(regs); 574 687 575 688 /* 576 689 * If the VM has been rebooted, its address space might still contain ··· 586 699 * This exception is only triggered when a guest 2 is running 587 700 * and can therefore never occur in kernel context. 588 701 */ 589 - printk_ratelimited(KERN_WARNING 590 - "Secure storage violation in task: %s, pid %d\n", 591 - current->comm, current->pid); 702 + pr_warn_ratelimited("Secure storage violation in task: %s, pid %d\n", 703 + current->comm, current->pid); 592 704 send_sig(SIGSEGV, current, 0); 593 705 } 594 706

+18 -6

arch/s390/mm/gmap.c

··· 21 21 22 22 #include <asm/pgalloc.h> 23 23 #include <asm/gmap.h> 24 + #include <asm/page.h> 24 25 #include <asm/tlb.h> 25 26 26 27 #define GMAP_SHADOW_FAKE_TABLE 1ULL 28 + 29 + static struct page *gmap_alloc_crst(void) 30 + { 31 + struct page *page; 32 + 33 + page = alloc_pages(GFP_KERNEL_ACCOUNT, CRST_ALLOC_ORDER); 34 + if (!page) 35 + return NULL; 36 + arch_set_page_dat(page, CRST_ALLOC_ORDER); 37 + return page; 38 + } 27 39 28 40 /** 29 41 * gmap_alloc - allocate and initialize a guest address space ··· 79 67 spin_lock_init(&gmap->guest_table_lock); 80 68 spin_lock_init(&gmap->shadow_lock); 81 69 refcount_set(&gmap->ref_count, 1); 82 - page = alloc_pages(GFP_KERNEL_ACCOUNT, CRST_ALLOC_ORDER); 70 + page = gmap_alloc_crst(); 83 71 if (!page) 84 72 goto out_free; 85 73 page->index = 0; ··· 320 308 unsigned long *new; 321 309 322 310 /* since we dont free the gmap table until gmap_free we can unlock */ 323 - page = alloc_pages(GFP_KERNEL_ACCOUNT, CRST_ALLOC_ORDER); 311 + page = gmap_alloc_crst(); 324 312 if (!page) 325 313 return -ENOMEM; 326 314 new = page_to_virt(page); ··· 1771 1759 1772 1760 BUG_ON(!gmap_is_shadow(sg)); 1773 1761 /* Allocate a shadow region second table */ 1774 - page = alloc_pages(GFP_KERNEL_ACCOUNT, CRST_ALLOC_ORDER); 1762 + page = gmap_alloc_crst(); 1775 1763 if (!page) 1776 1764 return -ENOMEM; 1777 1765 page->index = r2t & _REGION_ENTRY_ORIGIN; ··· 1855 1843 1856 1844 BUG_ON(!gmap_is_shadow(sg)); 1857 1845 /* Allocate a shadow region second table */ 1858 - page = alloc_pages(GFP_KERNEL_ACCOUNT, CRST_ALLOC_ORDER); 1846 + page = gmap_alloc_crst(); 1859 1847 if (!page) 1860 1848 return -ENOMEM; 1861 1849 page->index = r3t & _REGION_ENTRY_ORIGIN; ··· 1939 1927 1940 1928 BUG_ON(!gmap_is_shadow(sg) || (sgt & _REGION3_ENTRY_LARGE)); 1941 1929 /* Allocate a shadow segment table */ 1942 - page = alloc_pages(GFP_KERNEL_ACCOUNT, CRST_ALLOC_ORDER); 1930 + page = gmap_alloc_crst(); 1943 1931 if (!page) 1944 1932 return -ENOMEM; 1945 1933 page->index = sgt & _REGION_ENTRY_ORIGIN; ··· 2867 2855 if ((gmap->asce & _ASCE_TYPE_MASK) == _ASCE_TYPE_SEGMENT) 2868 2856 return -EINVAL; 2869 2857 2870 - page = alloc_pages(GFP_KERNEL_ACCOUNT, CRST_ALLOC_ORDER); 2858 + page = gmap_alloc_crst(); 2871 2859 if (!page) 2872 2860 return -ENOMEM; 2873 2861 page->index = 0;

+2 -2

arch/s390/mm/init.c

··· 35 35 #include <asm/processor.h> 36 36 #include <linux/uaccess.h> 37 37 #include <asm/pgalloc.h> 38 + #include <asm/ctlreg.h> 38 39 #include <asm/kfence.h> 39 40 #include <asm/ptdump.h> 40 41 #include <asm/dma.h> ··· 43 42 #include <asm/tlb.h> 44 43 #include <asm/tlbflush.h> 45 44 #include <asm/sections.h> 46 - #include <asm/ctl_reg.h> 47 45 #include <asm/sclp.h> 48 46 #include <asm/set_memory.h> 49 47 #include <asm/kasan.h> ··· 54 54 pgd_t swapper_pg_dir[PTRS_PER_PGD] __section(".bss..swapper_pg_dir"); 55 55 pgd_t invalid_pg_dir[PTRS_PER_PGD] __section(".bss..invalid_pg_dir"); 56 56 57 - unsigned long __bootdata_preserved(s390_invalid_asce); 57 + struct ctlreg __bootdata_preserved(s390_invalid_asce); 58 58 59 59 unsigned long empty_zero_page, zero_page_mask; 60 60 EXPORT_SYMBOL(empty_zero_page);

+1 -1

arch/s390/mm/maccess.c

··· 15 15 #include <linux/uio.h> 16 16 #include <linux/io.h> 17 17 #include <asm/asm-extable.h> 18 - #include <asm/ctl_reg.h> 19 18 #include <asm/abs_lowcore.h> 20 19 #include <asm/stacktrace.h> 21 20 #include <asm/maccess.h> 21 + #include <asm/ctlreg.h> 22 22 23 23 unsigned long __bootdata_preserved(__memcpy_real_area); 24 24 pte_t *__bootdata_preserved(memcpy_real_ptep);

+19 -6

arch/s390/mm/page-states.c

··· 121 121 continue; 122 122 if (!pud_folded(*pud)) { 123 123 page = phys_to_page(pud_val(*pud)); 124 - for (i = 0; i < 3; i++) 124 + for (i = 0; i < 4; i++) 125 125 set_bit(PG_arch_1, &page[i].flags); 126 126 } 127 127 mark_kernel_pmd(pud, addr, next); ··· 142 142 continue; 143 143 if (!p4d_folded(*p4d)) { 144 144 page = phys_to_page(p4d_val(*p4d)); 145 - for (i = 0; i < 3; i++) 145 + for (i = 0; i < 4; i++) 146 146 set_bit(PG_arch_1, &page[i].flags); 147 147 } 148 148 mark_kernel_pud(p4d, addr, next); ··· 151 151 152 152 static void mark_kernel_pgd(void) 153 153 { 154 - unsigned long addr, next; 154 + unsigned long addr, next, max_addr; 155 155 struct page *page; 156 156 pgd_t *pgd; 157 157 int i; 158 158 159 159 addr = 0; 160 + /* 161 + * Figure out maximum virtual address accessible with the 162 + * kernel ASCE. This is required to keep the page table walker 163 + * from accessing non-existent entries. 164 + */ 165 + max_addr = (S390_lowcore.kernel_asce.val & _ASCE_TYPE_MASK) >> 2; 166 + max_addr = 1UL << (max_addr * 11 + 31); 160 167 pgd = pgd_offset_k(addr); 161 168 do { 162 - next = pgd_addr_end(addr, MODULES_END); 169 + next = pgd_addr_end(addr, max_addr); 163 170 if (pgd_none(*pgd)) 164 171 continue; 165 172 if (!pgd_folded(*pgd)) { 166 173 page = phys_to_page(pgd_val(*pgd)); 167 - for (i = 0; i < 3; i++) 174 + for (i = 0; i < 4; i++) 168 175 set_bit(PG_arch_1, &page[i].flags); 169 176 } 170 177 mark_kernel_p4d(pgd, addr, next); 171 - } while (pgd++, addr = next, addr != MODULES_END); 178 + } while (pgd++, addr = next, addr != max_addr); 172 179 } 173 180 174 181 void __init cmma_init_nodat(void) ··· 188 181 return; 189 182 /* Mark pages used in kernel page tables */ 190 183 mark_kernel_pgd(); 184 + page = virt_to_page(&swapper_pg_dir); 185 + for (i = 0; i < 4; i++) 186 + set_bit(PG_arch_1, &page[i].flags); 187 + page = virt_to_page(&invalid_pg_dir); 188 + for (i = 0; i < 4; i++) 189 + set_bit(PG_arch_1, &page[i].flags); 191 190 192 191 /* Set all kernel pages not used for page tables to stable/no-dat */ 193 192 for_each_mem_pfn_range(i, MAX_NUMNODES, &start, &end, NULL) {

+1 -1

arch/s390/mm/pageattr.c

··· 75 75 break; 76 76 } 77 77 table = (unsigned long *)((unsigned long)old & mask); 78 - crdte(*old, new, table, dtt, addr, S390_lowcore.kernel_asce); 78 + crdte(*old, new, table, dtt, addr, S390_lowcore.kernel_asce.val); 79 79 } else if (MACHINE_HAS_IDTE) { 80 80 cspg(old, *old, new); 81 81 } else {

+4 -4

arch/s390/mm/pgalloc.c

··· 61 61 62 62 /* change all active ASCEs to avoid the creation of new TLBs */ 63 63 if (current->active_mm == mm) { 64 - S390_lowcore.user_asce = mm->context.asce; 65 - __ctl_load(S390_lowcore.user_asce, 7, 7); 64 + S390_lowcore.user_asce.val = mm->context.asce; 65 + local_ctl_load(7, &S390_lowcore.user_asce); 66 66 } 67 67 __tlb_flush_local(); 68 68 } ··· 145 145 ptdesc = pagetable_alloc(GFP_KERNEL, 0); 146 146 if (ptdesc) { 147 147 table = (u64 *)ptdesc_to_virt(ptdesc); 148 + arch_set_page_dat(virt_to_page(table), 0); 148 149 memset64(table, _PAGE_INVALID, PTRS_PER_PTE); 149 150 memset64(table + PTRS_PER_PTE, 0, PTRS_PER_PTE); 150 151 } ··· 488 487 unsigned long *table; 489 488 struct ptdesc *ptdesc; 490 489 491 - ptdesc = pagetable_alloc(GFP_KERNEL & ~__GFP_HIGHMEM, CRST_ALLOC_ORDER); 490 + ptdesc = pagetable_alloc(GFP_KERNEL, CRST_ALLOC_ORDER); 492 491 if (!ptdesc) 493 492 return NULL; 494 493 table = ptdesc_address(ptdesc); 495 - 496 494 crst_table_init(table, val); 497 495 return table; 498 496 }

+12 -3

arch/s390/mm/vmem.c

··· 12 12 #include <linux/hugetlb.h> 13 13 #include <linux/slab.h> 14 14 #include <linux/sort.h> 15 + #include <asm/page-states.h> 15 16 #include <asm/cacheflush.h> 16 17 #include <asm/nospec-branch.h> 18 + #include <asm/ctlreg.h> 17 19 #include <asm/pgalloc.h> 18 20 #include <asm/setup.h> 19 21 #include <asm/tlbflush.h> ··· 47 45 unsigned long *table; 48 46 49 47 table = vmem_alloc_pages(CRST_ALLOC_ORDER); 50 - if (table) 51 - crst_table_init(table, val); 48 + if (!table) 49 + return NULL; 50 + crst_table_init(table, val); 51 + if (slab_is_available()) 52 + arch_set_page_dat(virt_to_page(table), CRST_ALLOC_ORDER); 52 53 return table; 53 54 } 54 55 ··· 503 498 return ret; 504 499 } 505 500 501 + #ifdef CONFIG_MEMORY_HOTPLUG 502 + 506 503 void vmemmap_free(unsigned long start, unsigned long end, 507 504 struct vmem_altmap *altmap) 508 505 { ··· 512 505 remove_pagetable(start, end, false); 513 506 mutex_unlock(&vmem_mutex); 514 507 } 508 + 509 + #endif 515 510 516 511 void vmem_remove_mapping(unsigned long start, unsigned long size) 517 512 { ··· 668 659 __set_memory_4k(__va(0), RELOC_HIDE(__va(0), ident_map_size)); 669 660 } 670 661 if (MACHINE_HAS_NX) 671 - ctl_set_bit(0, 20); 662 + system_ctl_set_bit(0, CR0_INSTRUCTION_EXEC_PROTECTION_BIT); 672 663 pr_info("Write protected kernel read-only data: %luk\n", 673 664 (unsigned long)(__end_rodata - _stext) >> 10); 674 665 }

+1 -1

arch/s390/pci/pci.c

··· 1094 1094 1095 1095 if (MACHINE_HAS_PCI_MIO) { 1096 1096 static_branch_enable(&have_mio); 1097 - ctl_set_bit(2, 5); 1097 + system_ctl_set_bit(2, CR2_MIO_ADDRESSING_BIT); 1098 1098 } 1099 1099 1100 1100 rc = zpci_debug_init();

+1 -1

drivers/s390/char/diag_ftp.c

··· 16 16 #include <linux/wait.h> 17 17 #include <linux/string.h> 18 18 #include <asm/asm-extable.h> 19 - #include <asm/ctl_reg.h> 19 + #include <asm/ctlreg.h> 20 20 #include <asm/diag.h> 21 21 22 22 #include "hmcdrv_ftp.h"

+7 -7

drivers/s390/char/sclp.c

··· 81 81 struct sclp_trace_entry e; 82 82 83 83 memset(&e, 0, sizeof(e)); 84 - strncpy(e.id, id, sizeof(e.id)); 84 + strtomem(e.id, id); 85 85 e.a = a; 86 86 e.b = b; 87 87 debug_event(&sclp_debug, prio, &e, sizeof(e)); ··· 706 706 sclp_sync_wait(void) 707 707 { 708 708 unsigned long long old_tick; 709 + struct ctlreg cr0, cr0_sync; 709 710 unsigned long flags; 710 - unsigned long cr0, cr0_sync; 711 711 static u64 sync_count; 712 712 u64 timeout; 713 713 int irq_context; ··· 732 732 /* Enable service-signal interruption, disable timer interrupts */ 733 733 old_tick = local_tick_disable(); 734 734 trace_hardirqs_on(); 735 - __ctl_store(cr0, 0, 0); 736 - cr0_sync = cr0 & ~CR0_IRQ_SUBCLASS_MASK; 737 - cr0_sync |= 1UL << (63 - 54); 738 - __ctl_load(cr0_sync, 0, 0); 735 + local_ctl_store(0, &cr0); 736 + cr0_sync.val = cr0.val & ~CR0_IRQ_SUBCLASS_MASK; 737 + cr0_sync.val |= 1UL << (63 - 54); 738 + local_ctl_load(0, &cr0_sync); 739 739 __arch_local_irq_stosm(0x01); 740 740 /* Loop until driver state indicates finished request */ 741 741 while (sclp_running_state != sclp_running_state_idle) { ··· 745 745 cpu_relax(); 746 746 } 747 747 local_irq_disable(); 748 - __ctl_load(cr0, 0, 0); 748 + local_ctl_load(0, &cr0); 749 749 if (!irq_context) 750 750 _local_bh_enable(); 751 751 local_tick_enable(old_tick);

+1 -2

drivers/s390/char/sclp_cmd.c

··· 19 19 #include <linux/mmzone.h> 20 20 #include <linux/memory.h> 21 21 #include <linux/module.h> 22 - #include <asm/ctl_reg.h> 22 + #include <asm/ctlreg.h> 23 23 #include <asm/chpid.h> 24 24 #include <asm/setup.h> 25 25 #include <asm/page.h> ··· 353 353 sclp_mem_change_state(start, size, 0); 354 354 break; 355 355 default: 356 - rc = -EINVAL; 357 356 break; 358 357 } 359 358 mutex_unlock(&sclp_mem_mutex);

+1 -1

drivers/s390/char/sclp_early.c

··· 10 10 11 11 #include <linux/errno.h> 12 12 #include <linux/memblock.h> 13 - #include <asm/ctl_reg.h> 13 + #include <asm/ctlreg.h> 14 14 #include <asm/sclp.h> 15 15 #include <asm/ipl.h> 16 16 #include <asm/setup.h>

+4 -3

drivers/s390/char/sclp_early_core.c

··· 7 7 #include <linux/kernel.h> 8 8 #include <asm/processor.h> 9 9 #include <asm/lowcore.h> 10 + #include <asm/ctlreg.h> 10 11 #include <asm/ebcdic.h> 11 12 #include <asm/irq.h> 12 13 #include <asm/sections.h> ··· 32 31 psw_t psw_ext_save, psw_wait; 33 32 union ctlreg0 cr0, cr0_new; 34 33 35 - __ctl_store(cr0.val, 0, 0); 34 + local_ctl_store(0, &cr0.reg); 36 35 cr0_new.val = cr0.val & ~CR0_IRQ_SUBCLASS_MASK; 37 36 cr0_new.lap = 0; 38 37 cr0_new.sssm = 1; 39 - __ctl_load(cr0_new.val, 0, 0); 38 + local_ctl_load(0, &cr0_new.reg); 40 39 41 40 psw_ext_save = S390_lowcore.external_new_psw; 42 41 psw_mask = __extract_psw(); ··· 59 58 } while (S390_lowcore.ext_int_code != EXT_IRQ_SERVICE_SIG); 60 59 61 60 S390_lowcore.external_new_psw = psw_ext_save; 62 - __ctl_load(cr0.val, 0, 0); 61 + local_ctl_load(0, &cr0.reg); 63 62 } 64 63 65 64 int sclp_early_cmd(sclp_cmdw_t cmd, void *sccb)

+1 -1

drivers/s390/cio/chp.c

··· 334 334 return 0; 335 335 if (chp->cmg == -1) /* channel measurements not available */ 336 336 return sprintf(buf, "unknown\n"); 337 - return sprintf(buf, "%x\n", chp->cmg); 337 + return sprintf(buf, "%d\n", chp->cmg); 338 338 } 339 339 340 340 static DEVICE_ATTR(cmg, 0444, chp_cmg_show, NULL);

+4 -4

drivers/s390/cio/chsc.c

··· 393 393 memset(id, 0, NODEID_LEN); 394 394 395 395 if (nd->validity != ND_VALIDITY_VALID) { 396 - strncpy(params, "n/a", PARAMS_LEN - 1); 397 - strncpy(id, "n/a", NODEID_LEN - 1); 396 + strscpy(params, "n/a", PARAMS_LEN); 397 + strscpy(id, "n/a", NODEID_LEN); 398 398 return; 399 399 } 400 400 ··· 881 881 secm_area->request.code = 0x0016; 882 882 883 883 secm_area->key = PAGE_DEFAULT_KEY >> 4; 884 - secm_area->cub_addr1 = (u64)(unsigned long)css->cub_addr1; 885 - secm_area->cub_addr2 = (u64)(unsigned long)css->cub_addr2; 884 + secm_area->cub_addr1 = virt_to_phys(css->cub_addr1); 885 + secm_area->cub_addr2 = virt_to_phys(css->cub_addr2); 886 886 887 887 secm_area->operation_code = enable ? 0 : 1; 888 888

+2 -2

drivers/s390/cio/crw.c

··· 12 12 #include <linux/kthread.h> 13 13 #include <linux/init.h> 14 14 #include <linux/wait.h> 15 + #include <asm/ctlreg.h> 15 16 #include <asm/crw.h> 16 - #include <asm/ctl_reg.h> 17 17 #include "ioasm.h" 18 18 19 19 static DEFINE_MUTEX(crw_handler_mutex); ··· 156 156 task = kthread_run(crw_collect_info, NULL, "kmcheck"); 157 157 if (IS_ERR(task)) 158 158 return PTR_ERR(task); 159 - ctl_set_bit(14, 28); /* enable channel report MCH */ 159 + system_ctl_set_bit(14, CR14_CHANNEL_REPORT_SUBMASK_BIT); 160 160 return 0; 161 161 } 162 162 device_initcall(crw_machine_check_init);

+2 -2

drivers/s390/cio/isc.c

··· 33 33 34 34 spin_lock(&isc_ref_lock); 35 35 if (isc_refs[isc] == 0) 36 - ctl_set_bit(6, 31 - isc); 36 + system_ctl_set_bit(6, 31 - isc); 37 37 isc_refs[isc]++; 38 38 spin_unlock(&isc_ref_lock); 39 39 } ··· 61 61 goto out_unlock; 62 62 } 63 63 if (isc_refs[isc] == 1) 64 - ctl_clear_bit(6, 31 - isc); 64 + system_ctl_clear_bit(6, 31 - isc); 65 65 isc_refs[isc]--; 66 66 out_unlock: 67 67 spin_unlock(&isc_ref_lock);

+10 -11

drivers/s390/crypto/ap_bus.c

··· 1865 1865 } 1866 1866 /* get it and thus adjust reference counter */ 1867 1867 get_device(dev); 1868 - if (decfg) 1868 + if (decfg) { 1869 1869 AP_DBF_INFO("%s(%d,%d) new (decfg) queue dev created\n", 1870 1870 __func__, ac->id, dom); 1871 - else if (chkstop) 1871 + } else if (chkstop) { 1872 1872 AP_DBF_INFO("%s(%d,%d) new (chkstop) queue dev created\n", 1873 1873 __func__, ac->id, dom); 1874 - else 1874 + } else { 1875 + /* nudge the queue's state machine */ 1876 + ap_queue_init_state(aq); 1875 1877 AP_DBF_INFO("%s(%d,%d) new queue dev created\n", 1876 1878 __func__, ac->id, dom); 1879 + } 1877 1880 goto put_dev_and_continue; 1878 1881 } 1879 1882 /* handle state changes on already existing queue device */ ··· 1898 1895 } else if (!chkstop && aq->chkstop) { 1899 1896 /* checkstop off */ 1900 1897 aq->chkstop = false; 1901 - if (aq->dev_state > AP_DEV_STATE_UNINITIATED) { 1902 - aq->dev_state = AP_DEV_STATE_OPERATING; 1903 - aq->sm_state = AP_SM_STATE_RESET_START; 1904 - } 1898 + if (aq->dev_state > AP_DEV_STATE_UNINITIATED) 1899 + _ap_queue_init_state(aq); 1905 1900 spin_unlock_bh(&aq->lock); 1906 1901 AP_DBF_DBG("%s(%d,%d) queue dev checkstop off\n", 1907 1902 __func__, ac->id, dom); ··· 1923 1922 } else if (!decfg && !aq->config) { 1924 1923 /* config on this queue device */ 1925 1924 aq->config = true; 1926 - if (aq->dev_state > AP_DEV_STATE_UNINITIATED) { 1927 - aq->dev_state = AP_DEV_STATE_OPERATING; 1928 - aq->sm_state = AP_SM_STATE_RESET_START; 1929 - } 1925 + if (aq->dev_state > AP_DEV_STATE_UNINITIATED) 1926 + _ap_queue_init_state(aq); 1930 1927 spin_unlock_bh(&aq->lock); 1931 1928 AP_DBF_DBG("%s(%d,%d) queue dev config on\n", 1932 1929 __func__, ac->id, dom);

+3

drivers/s390/crypto/ap_bus.h

··· 207 207 bool chkstop; /* checkstop state */ 208 208 ap_qid_t qid; /* AP queue id. */ 209 209 bool interrupt; /* indicate if interrupts are enabled */ 210 + bool se_bound; /* SE bound state */ 210 211 unsigned int assoc_idx; /* SE association index */ 211 212 int queue_count; /* # messages currently on AP queue. */ 212 213 int pendingq_count; /* # requests on pendingq list. */ ··· 272 271 int ap_queue_message(struct ap_queue *aq, struct ap_message *ap_msg); 273 272 void ap_cancel_message(struct ap_queue *aq, struct ap_message *ap_msg); 274 273 void ap_flush_queue(struct ap_queue *aq); 274 + bool ap_queue_usable(struct ap_queue *aq); 275 275 276 276 void *ap_airq_ptr(void); 277 277 int ap_sb_available(void); ··· 289 287 void ap_queue_prepare_remove(struct ap_queue *aq); 290 288 void ap_queue_remove(struct ap_queue *aq); 291 289 void ap_queue_init_state(struct ap_queue *aq); 290 + void _ap_queue_init_state(struct ap_queue *aq); 292 291 293 292 struct ap_card *ap_card_create(int id, int queue_depth, int raw_type, 294 293 int comp_type, unsigned int functions, int ml);

+55 -3

drivers/s390/crypto/ap_queue.c

··· 33 33 return ap_test_bit(&aq->card->functions, AP_FUNC_EP11); 34 34 } 35 35 36 + static inline bool ap_q_needs_bind(struct ap_queue *aq) 37 + { 38 + return ap_q_supports_bind(aq) && ap_sb_available(); 39 + } 40 + 36 41 /** 37 42 * ap_queue_enable_irq(): Enable interrupt support on this AP queue. 38 43 * @aq: The AP queue ··· 309 304 aq->sm_state = AP_SM_STATE_RESET_WAIT; 310 305 aq->interrupt = false; 311 306 aq->rapq_fbit = 0; 307 + aq->se_bound = false; 312 308 return AP_SM_WAIT_LOW_TIMEOUT; 313 309 default: 314 310 aq->dev_state = AP_DEV_STATE_ERROR; ··· 874 868 } 875 869 status = ap_bapq(aq->qid); 876 870 spin_unlock_bh(&aq->lock); 877 - if (status.response_code) { 871 + if (!status.response_code) { 872 + aq->se_bound = true; 873 + AP_DBF_INFO("%s bapq(0x%02x.%04x) success\n", __func__, 874 + AP_QID_CARD(aq->qid), 875 + AP_QID_QUEUE(aq->qid)); 876 + } else { 878 877 AP_DBF_WARN("%s RC 0x%02x on bapq(0x%02x.%04x)\n", 879 878 __func__, status.response_code, 880 879 AP_QID_CARD(aq->qid), ··· 1085 1074 EXPORT_SYMBOL(ap_queue_message); 1086 1075 1087 1076 /** 1077 + * ap_queue_usable(): Check if queue is usable just now. 1078 + * @aq: The AP queue device to test for usability. 1079 + * This function is intended for the scheduler to query if it makes 1080 + * sense to enqueue a message into this AP queue device by calling 1081 + * ap_queue_message(). The perspective is very short-term as the 1082 + * state machine and device state(s) may change at any time. 1083 + */ 1084 + bool ap_queue_usable(struct ap_queue *aq) 1085 + { 1086 + bool rc = true; 1087 + 1088 + spin_lock_bh(&aq->lock); 1089 + 1090 + /* check for not configured or checkstopped */ 1091 + if (!aq->config || aq->chkstop) { 1092 + rc = false; 1093 + goto unlock_and_out; 1094 + } 1095 + 1096 + /* device state needs to be ok */ 1097 + if (aq->dev_state != AP_DEV_STATE_OPERATING) { 1098 + rc = false; 1099 + goto unlock_and_out; 1100 + } 1101 + 1102 + /* SE guest's queues additionally need to be bound */ 1103 + if (ap_q_needs_bind(aq) && !aq->se_bound) 1104 + rc = false; 1105 + 1106 + unlock_and_out: 1107 + spin_unlock_bh(&aq->lock); 1108 + return rc; 1109 + } 1110 + EXPORT_SYMBOL(ap_queue_usable); 1111 + 1112 + /** 1088 1113 * ap_cancel_message(): Cancel a crypto request. 1089 1114 * @aq: The AP device that has the message queued 1090 1115 * @ap_msg: The message that is to be removed ··· 1207 1160 spin_unlock_bh(&aq->lock); 1208 1161 } 1209 1162 1210 - void ap_queue_init_state(struct ap_queue *aq) 1163 + void _ap_queue_init_state(struct ap_queue *aq) 1211 1164 { 1212 - spin_lock_bh(&aq->lock); 1213 1165 aq->dev_state = AP_DEV_STATE_OPERATING; 1214 1166 aq->sm_state = AP_SM_STATE_RESET_START; 1215 1167 aq->last_err_rc = 0; 1216 1168 aq->assoc_idx = ASSOC_IDX_INVALID; 1217 1169 ap_wait(ap_sm_event(aq, AP_SM_EVENT_POLL)); 1170 + } 1171 + 1172 + void ap_queue_init_state(struct ap_queue *aq) 1173 + { 1174 + spin_lock_bh(&aq->lock); 1175 + _ap_queue_init_state(aq); 1218 1176 spin_unlock_bh(&aq->lock); 1219 1177 } 1220 1178 EXPORT_SYMBOL(ap_queue_init_state);

+5 -5

drivers/s390/crypto/zcrypt_api.c

··· 693 693 for_each_zcrypt_queue(zq, zc) { 694 694 /* check if device is usable and eligible */ 695 695 if (!zq->online || !zq->ops->rsa_modexpo || 696 - !zq->queue->config || zq->queue->chkstop) 696 + !ap_queue_usable(zq->queue)) 697 697 continue; 698 698 /* check if device node has admission for this queue */ 699 699 if (!zcrypt_check_queue(perms, ··· 798 798 for_each_zcrypt_queue(zq, zc) { 799 799 /* check if device is usable and eligible */ 800 800 if (!zq->online || !zq->ops->rsa_modexpo_crt || 801 - !zq->queue->config || zq->queue->chkstop) 801 + !ap_queue_usable(zq->queue)) 802 802 continue; 803 803 /* check if device node has admission for this queue */ 804 804 if (!zcrypt_check_queue(perms, ··· 916 916 for_each_zcrypt_queue(zq, zc) { 917 917 /* check for device usable and eligible */ 918 918 if (!zq->online || !zq->ops->send_cprb || 919 - !zq->queue->config || zq->queue->chkstop || 919 + !ap_queue_usable(zq->queue) || 920 920 (tdom != AUTOSEL_DOM && 921 921 tdom != AP_QID_QUEUE(zq->queue->qid))) 922 922 continue; ··· 1087 1087 for_each_zcrypt_queue(zq, zc) { 1088 1088 /* check if device is usable and eligible */ 1089 1089 if (!zq->online || !zq->ops->send_ep11_cprb || 1090 - !zq->queue->config || zq->queue->chkstop || 1090 + !ap_queue_usable(zq->queue) || 1091 1091 (targets && 1092 1092 !is_desired_ep11_queue(zq->queue->qid, 1093 1093 target_num, targets))) ··· 1186 1186 for_each_zcrypt_queue(zq, zc) { 1187 1187 /* check if device is usable and eligible */ 1188 1188 if (!zq->online || !zq->ops->rng || 1189 - !zq->queue->config || zq->queue->chkstop) 1189 + !ap_queue_usable(zq->queue)) 1190 1190 continue; 1191 1191 if (!zcrypt_queue_compare(zq, pref_zq, wgt, pref_wgt)) 1192 1192 continue;

+4

drivers/s390/crypto/zcrypt_cex4.c

··· 279 279 { 1, "BSI2009" }, 280 280 { 2, "FIPS2011" }, 281 281 { 3, "BSI2011" }, 282 + { 4, "SIGG-IMPORT" }, 283 + { 5, "SIGG" }, 282 284 { 6, "BSICC2017" }, 285 + { 7, "FIPS2021" }, 286 + { 8, "FIPS2024" }, 283 287 { 0, NULL } 284 288 }; 285 289

+16 -2

drivers/s390/crypto/zcrypt_error.h

··· 98 98 case REP88_ERROR_MESSAGE_MALFORMD: /* 0x22 */ 99 99 case REP88_ERROR_KEY_TYPE: /* 0x34 */ 100 100 /* RY indicates malformed request */ 101 - ZCRYPT_DBF_WARN("%s dev=%02x.%04x RY=0x%02x => rc=EINVAL\n", 102 - __func__, card, queue, ehdr->reply_code); 101 + if (ehdr->reply_code == REP82_ERROR_FILTERED_BY_HYPERVISOR && 102 + ehdr->type == TYPE86_RSP_CODE) { 103 + struct { 104 + struct type86_hdr hdr; 105 + struct type86_fmt2_ext fmt2; 106 + } __packed * head = reply->msg; 107 + unsigned int apfs = *((u32 *)head->fmt2.apfs); 108 + 109 + ZCRYPT_DBF_WARN("%s dev=%02x.%04x RY=0x%02x apfs=0x%x => rc=EINVAL\n", 110 + __func__, card, queue, 111 + ehdr->reply_code, apfs); 112 + } else { 113 + ZCRYPT_DBF_WARN("%s dev=%02x.%04x RY=0x%02x => rc=EINVAL\n", 114 + __func__, card, queue, 115 + ehdr->reply_code); 116 + } 103 117 return -EINVAL; 104 118 case REP82_ERROR_MACHINE_FAILURE: /* 0x10 */ 105 119 case REP82_ERROR_MESSAGE_TYPE: /* 0x20 */

+2 -2

net/iucv/iucv.c

··· 1823 1823 rc = -EPROTONOSUPPORT; 1824 1824 goto out; 1825 1825 } 1826 - ctl_set_bit(0, 1); 1826 + system_ctl_set_bit(0, CR0_IUCV_BIT); 1827 1827 rc = iucv_query_maxconn(); 1828 1828 if (rc) 1829 1829 goto out_ctl; ··· 1871 1871 out_int: 1872 1872 unregister_external_irq(EXT_IRQ_IUCV, iucv_external_interrupt); 1873 1873 out_ctl: 1874 - ctl_clear_bit(0, 1); 1874 + system_ctl_clear_bit(0, 1); 1875 1875 out: 1876 1876 return rc; 1877 1877 }

Configure Feed

Configure Feed