Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge tag 'arm64-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux
Pull arm64 fixes from Will Deacon:
"I think the main one is fixing the dynamic SCS patching when full LTO
is enabled (clang was silently getting this horribly wrong), but it's
all good stuff.
Rob just pointed out that the fix to the workaround for erratum
#2966298 might not be necessary, but in the worst case it's harmless
and since the official description leaves a little to be desired here,
I've left it in.
Summary:
- Fix shadow call stack patching with LTO=full
- Fix voluntary preemption of the FPSIMD registers from assembly code
- Fix workaround for A520 CPU erratum #2966298 and extend to A510
- Fix SME issues that resulted in corruption of the register state
- Minor fixes (missing includes, formatting)"
* tag 'arm64-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux:
arm64: Fix silcon-errata.rst formatting
arm64/sme: Always exit sme_alloc() early with existing storage
arm64/fpsimd: Remove spurious check for SVE support
arm64/ptrace: Don't flush ZA/ZT storage when writing ZA via ptrace
arm64: entry: simplify kernel_exit logic
arm64: entry: fix ARM64_WORKAROUND_SPECULATIVE_UNPRIV_LOAD
arm64: errata: Add Cortex-A510 speculative unprivileged load workaround
arm64: Rename ARM64_WORKAROUND_2966298
arm64: fpsimd: Bring cond_yield asm macro in line with new rules
arm64: scs: Work around full LTO issue with dynamic SCS
arm64: irq: include <linux/cpumask.h>
+83
-49
+2
-2
Documentation/arch/arm64/silicon-errata.rst
+2
-2
Documentation/arch/arm64/silicon-errata.rst
···
71
71
+----------------+-----------------+-----------------+-----------------------------+
72
72
| ARM | Cortex-A510 | #2658417 | ARM64_ERRATUM_2658417 |
73
73
+----------------+-----------------+-----------------+-----------------------------+
74
+
| ARM | Cortex-A510 | #3117295 | ARM64_ERRATUM_3117295 |
75
+
+----------------+-----------------+-----------------+-----------------------------+
74
76
| ARM | Cortex-A520 | #2966298 | ARM64_ERRATUM_2966298 |
75
77
+----------------+-----------------+-----------------+-----------------------------+
76
78
| ARM | Cortex-A53 | #826319 | ARM64_ERRATUM_826319 |
···
237
235
+----------------+-----------------+-----------------+-----------------------------+
238
236
| Rockchip | RK3588 | #3588001 | ROCKCHIP_ERRATUM_3588001 |
239
237
+----------------+-----------------+-----------------+-----------------------------+
240
-
241
238
+----------------+-----------------+-----------------+-----------------------------+
242
239
| Fujitsu | A64FX | E#010001 | FUJITSU_ERRATUM_010001 |
243
240
+----------------+-----------------+-----------------+-----------------------------+
244
-
245
241
+----------------+-----------------+-----------------+-----------------------------+
246
242
| ASR | ASR8601 | #8601001 | N/A |
247
243
+----------------+-----------------+-----------------+-----------------------------+
+18
arch/arm64/Kconfig
+18
arch/arm64/Kconfig
···
1039
1039
1040
1040
If unsure, say Y.
1041
1041
1042
+
config ARM64_WORKAROUND_SPECULATIVE_UNPRIV_LOAD
1043
+
bool
1044
+
1042
1045
config ARM64_ERRATUM_2966298
1043
1046
bool "Cortex-A520: 2966298: workaround for speculatively executed unprivileged load"
1047
+
select ARM64_WORKAROUND_SPECULATIVE_UNPRIV_LOAD
1044
1048
default y
1045
1049
help
1046
1050
This option adds the workaround for ARM Cortex-A520 erratum 2966298.
1047
1051
1048
1052
On an affected Cortex-A520 core, a speculatively executed unprivileged
1053
+
load might leak data from a privileged level via a cache side channel.
1054
+
1055
+
Work around this problem by executing a TLBI before returning to EL0.
1056
+
1057
+
If unsure, say Y.
1058
+
1059
+
config ARM64_ERRATUM_3117295
1060
+
bool "Cortex-A510: 3117295: workaround for speculatively executed unprivileged load"
1061
+
select ARM64_WORKAROUND_SPECULATIVE_UNPRIV_LOAD
1062
+
default y
1063
+
help
1064
+
This option adds the workaround for ARM Cortex-A510 erratum 3117295.
1065
+
1066
+
On an affected Cortex-A510 core, a speculatively executed unprivileged
1049
1067
load might leak data from a privileged level via a cache side channel.
1050
1068
1051
1069
Work around this problem by executing a TLBI before returning to EL0.
+9
-16
arch/arm64/include/asm/assembler.h
+9
-16
arch/arm64/include/asm/assembler.h
···
760
760
.endm
761
761
762
762
/*
763
-
* Check whether preempt/bh-disabled asm code should yield as soon as
764
-
* it is able. This is the case if we are currently running in task
765
-
* context, and either a softirq is pending, or the TIF_NEED_RESCHED
766
-
* flag is set and re-enabling preemption a single time would result in
767
-
* a preempt count of zero. (Note that the TIF_NEED_RESCHED flag is
768
-
* stored negated in the top word of the thread_info::preempt_count
763
+
* Check whether asm code should yield as soon as it is able. This is
764
+
* the case if we are currently running in task context, and the
765
+
* TIF_NEED_RESCHED flag is set. (Note that the TIF_NEED_RESCHED flag
766
+
* is stored negated in the top word of the thread_info::preempt_count
769
767
* field)
770
768
*/
771
-
.macro cond_yield, lbl:req, tmp:req, tmp2:req
769
+
.macro cond_yield, lbl:req, tmp:req, tmp2
770
+
#ifdef CONFIG_PREEMPT_VOLUNTARY
772
771
get_current_task \tmp
773
772
ldr \tmp, [\tmp, #TSK_TI_PREEMPT]
774
773
/*
775
774
* If we are serving a softirq, there is no point in yielding: the
776
775
* softirq will not be preempted no matter what we do, so we should
777
-
* run to completion as quickly as we can.
776
+
* run to completion as quickly as we can. The preempt_count field will
777
+
* have BIT(SOFTIRQ_SHIFT) set in this case, so the zero check will
778
+
* catch this case too.
778
779
*/
779
-
tbnz \tmp, #SOFTIRQ_SHIFT, .Lnoyield_\@
780
-
#ifdef CONFIG_PREEMPTION
781
-
sub \tmp, \tmp, #PREEMPT_DISABLE_OFFSET
782
780
cbz \tmp, \lbl
783
781
#endif
784
-
adr_l \tmp, irq_stat + IRQ_CPUSTAT_SOFTIRQ_PENDING
785
-
get_this_cpu_offset \tmp2
786
-
ldr w\tmp, [\tmp, \tmp2]
787
-
cbnz w\tmp, \lbl // yield on pending softirq in task context
788
-
.Lnoyield_\@:
789
782
.endm
790
783
791
784
/*
+2
arch/arm64/include/asm/irq.h
+2
arch/arm64/include/asm/irq.h
+7
-1
arch/arm64/kernel/Makefile
+7
-1
arch/arm64/kernel/Makefile
···
73
73
obj-y += vdso-wrap.o
74
74
obj-$(CONFIG_COMPAT_VDSO) += vdso32-wrap.o
75
75
obj-$(CONFIG_UNWIND_PATCH_PAC_INTO_SCS) += patch-scs.o
76
-
CFLAGS_patch-scs.o += -mbranch-protection=none
76
+
77
+
# We need to prevent the SCS patching code from patching itself. Using
78
+
# -mbranch-protection=none here to avoid the patchable PAC opcodes from being
79
+
# generated triggers an issue with full LTO on Clang, which stops emitting PAC
80
+
# instructions altogether. So instead, omit the unwind tables used by the
81
+
# patching code, so it will not be able to locate its own PAC instructions.
82
+
CFLAGS_patch-scs.o += -fno-asynchronous-unwind-tables -fno-unwind-tables
77
83
78
84
# Force dependency (vdso*-wrap.S includes vdso.so through incbin)
79
85
$(obj)/vdso-wrap.o: $(obj)/vdso/vdso.so
-2
arch/arm64/kernel/asm-offsets.c
-2
arch/arm64/kernel/asm-offsets.c
···
117
117
DEFINE(DMA_FROM_DEVICE, DMA_FROM_DEVICE);
118
118
BLANK();
119
119
DEFINE(PREEMPT_DISABLE_OFFSET, PREEMPT_DISABLE_OFFSET);
120
-
DEFINE(SOFTIRQ_SHIFT, SOFTIRQ_SHIFT);
121
-
DEFINE(IRQ_CPUSTAT_SOFTIRQ_PENDING, offsetof(irq_cpustat_t, __softirq_pending));
122
120
BLANK();
123
121
DEFINE(CPU_BOOT_TASK, offsetof(struct secondary_data, task));
124
122
BLANK();
+17
-4
arch/arm64/kernel/cpu_errata.c
+17
-4
arch/arm64/kernel/cpu_errata.c
···
416
416
};
417
417
#endif /* CONFIG_ARM64_WORKAROUND_TRBE_WRITE_OUT_OF_RANGE */
418
418
419
+
#ifdef CONFIG_ARM64_WORKAROUND_SPECULATIVE_UNPRIV_LOAD
420
+
static const struct midr_range erratum_spec_unpriv_load_list[] = {
421
+
#ifdef CONFIG_ARM64_ERRATUM_3117295
422
+
MIDR_ALL_VERSIONS(MIDR_CORTEX_A510),
423
+
#endif
424
+
#ifdef CONFIG_ARM64_ERRATUM_2966298
425
+
/* Cortex-A520 r0p0 to r0p1 */
426
+
MIDR_REV_RANGE(MIDR_CORTEX_A520, 0, 0, 1),
427
+
#endif
428
+
{},
429
+
};
430
+
#endif
431
+
419
432
const struct arm64_cpu_capabilities arm64_errata[] = {
420
433
#ifdef CONFIG_ARM64_WORKAROUND_CLEAN_CACHE
421
434
{
···
726
713
MIDR_FIXED(MIDR_CPU_VAR_REV(1,1), BIT(25)),
727
714
},
728
715
#endif
729
-
#ifdef CONFIG_ARM64_ERRATUM_2966298
716
+
#ifdef CONFIG_ARM64_WORKAROUND_SPECULATIVE_UNPRIV_LOAD
730
717
{
731
-
.desc = "ARM erratum 2966298",
732
-
.capability = ARM64_WORKAROUND_2966298,
718
+
.desc = "ARM errata 2966298, 3117295",
719
+
.capability = ARM64_WORKAROUND_SPECULATIVE_UNPRIV_LOAD,
733
720
/* Cortex-A520 r0p0 - r0p1 */
734
-
ERRATA_MIDR_REV_RANGE(MIDR_CORTEX_A520, 0, 0, 1),
721
+
ERRATA_MIDR_RANGE_LIST(erratum_spec_unpriv_load_list),
735
722
},
736
723
#endif
737
724
#ifdef CONFIG_AMPERE_ERRATUM_AC03_CPU_38
+14
-11
arch/arm64/kernel/entry.S
+14
-11
arch/arm64/kernel/entry.S
···
428
428
ldp x28, x29, [sp, #16 * 14]
429
429
430
430
.if \el == 0
431
-
alternative_if ARM64_WORKAROUND_2966298
432
-
tlbi vale1, xzr
433
-
dsb nsh
434
-
alternative_else_nop_endif
435
-
alternative_if_not ARM64_UNMAP_KERNEL_AT_EL0
436
-
ldr lr, [sp, #S_LR]
437
-
add sp, sp, #PT_REGS_SIZE // restore sp
438
-
eret
439
-
alternative_else_nop_endif
440
431
#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
432
+
alternative_insn "b .L_skip_tramp_exit_\@", nop, ARM64_UNMAP_KERNEL_AT_EL0
433
+
441
434
msr far_el1, x29
442
435
443
436
ldr_this_cpu x30, this_cpu_vector, x29
···
439
446
ldr lr, [sp, #S_LR] // restore x30
440
447
add sp, sp, #PT_REGS_SIZE // restore sp
441
448
br x29
449
+
450
+
.L_skip_tramp_exit_\@:
442
451
#endif
443
-
.else
452
+
.endif
453
+
444
454
ldr lr, [sp, #S_LR]
445
455
add sp, sp, #PT_REGS_SIZE // restore sp
446
456
457
+
.if \el == 0
458
+
/* This must be after the last explicit memory access */
459
+
alternative_if ARM64_WORKAROUND_SPECULATIVE_UNPRIV_LOAD
460
+
tlbi vale1, xzr
461
+
dsb nsh
462
+
alternative_else_nop_endif
463
+
.else
447
464
/* Ensure any device/NC reads complete */
448
465
alternative_insn nop, "dmb sy", ARM64_WORKAROUND_1508412
466
+
.endif
449
467
450
468
eret
451
-
.endif
452
469
sb
453
470
.endm
454
471
+6
-6
arch/arm64/kernel/fpsimd.c
+6
-6
arch/arm64/kernel/fpsimd.c
···
898
898
* allocate SVE now in case it is needed for use in streaming
899
899
* mode.
900
900
*/
901
-
if (system_supports_sve()) {
902
-
sve_free(task);
903
-
sve_alloc(task, true);
904
-
}
901
+
sve_free(task);
902
+
sve_alloc(task, true);
905
903
906
904
if (free_sme)
907
905
sme_free(task);
···
1217
1219
*/
1218
1220
void sme_alloc(struct task_struct *task, bool flush)
1219
1221
{
1220
-
if (task->thread.sme_state && flush) {
1221
-
memset(task->thread.sme_state, 0, sme_state_size(task));
1222
+
if (task->thread.sme_state) {
1223
+
if (flush)
1224
+
memset(task->thread.sme_state, 0,
1225
+
sme_state_size(task));
1222
1226
return;
1223
1227
}
1224
1228
+7
-6
arch/arm64/kernel/ptrace.c
+7
-6
arch/arm64/kernel/ptrace.c
···
1108
1108
}
1109
1109
}
1110
1110
1111
-
/* Allocate/reinit ZA storage */
1112
-
sme_alloc(target, true);
1113
-
if (!target->thread.sme_state) {
1114
-
ret = -ENOMEM;
1115
-
goto out;
1116
-
}
1111
+
/*
1112
+
* Only flush the storage if PSTATE.ZA was not already set,
1113
+
* otherwise preserve any existing data.
1114
+
*/
1115
+
sme_alloc(target, !thread_za_enabled(&target->thread));
1116
+
if (!target->thread.sme_state)
1117
+
return -ENOMEM;
1117
1118
1118
1119
/* If there is no data then disable ZA */
1119
1120
if (!count) {
+1
-1
arch/arm64/tools/cpucaps
+1
-1
arch/arm64/tools/cpucaps
···
84
84
WORKAROUND_2457168
85
85
WORKAROUND_2645198
86
86
WORKAROUND_2658417
87
-
WORKAROUND_2966298
88
87
WORKAROUND_AMPERE_AC03_CPU_38
89
88
WORKAROUND_TRBE_OVERWRITE_FILL_MODE
90
89
WORKAROUND_TSB_FLUSH_FAILURE
···
99
100
WORKAROUND_QCOM_FALKOR_E1003
100
101
WORKAROUND_REPEAT_TLBI
101
102
WORKAROUND_SPECULATIVE_AT
103
+
WORKAROUND_SPECULATIVE_UNPRIV_LOAD