Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
arm64: errata: Work around early CME DVMSync acknowledgement
C1-Pro acknowledges DVMSync messages before completing the SME/CME
memory accesses. Work around this by issuing an IPI to the affected CPUs
if they are running in EL0 with SME enabled.
Note that we avoid the local DSB in the IPI handler as the kernel runs
with SCTLR_EL1.IESB=1. This is sufficient to complete SME memory
accesses at EL0 on taking an exception to EL1. On the return to user
path, no barrier is necessary either. See the comment in
sme_set_active() and the more detailed explanation in the link below.
To avoid a potential IPI flood from malicious applications (e.g.
madvise(MADV_PAGEOUT) in a tight loop), track where a process is active
via mm_cpumask() and only interrupt those CPUs.
Link: https://lore.kernel.org/r/ablEXwhfKyJW1i7l@J2N7QTR9R3
Cc: Will Deacon <will@kernel.org>
Cc: Mark Rutland <mark.rutland@arm.com>
Cc: James Morse <james.morse@arm.com>
Cc: Mark Brown <broonie@kernel.org>
Reviewed-by: Will Deacon <will@kernel.org>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
+264
-4
+2
Documentation/arch/arm64/silicon-errata.rst
+2
Documentation/arch/arm64/silicon-errata.rst
···
202
202
+----------------+-----------------+-----------------+-----------------------------+
203
203
| ARM | Neoverse-V3AE | #3312417 | ARM64_ERRATUM_3194386 |
204
204
+----------------+-----------------+-----------------+-----------------------------+
205
+
| ARM | C1-Pro | #4193714 | ARM64_ERRATUM_4193714 |
206
+
+----------------+-----------------+-----------------+-----------------------------+
205
207
| ARM | MMU-500 | #841119,826419 | ARM_SMMU_MMU_500_CPRE_ERRATA|
206
208
| | | #562869,1047329 | |
207
209
+----------------+-----------------+-----------------+-----------------------------+
+12
arch/arm64/Kconfig
+12
arch/arm64/Kconfig
···
1175
1175
1176
1176
If unsure, say Y.
1177
1177
1178
+
config ARM64_ERRATUM_4193714
1179
+
bool "C1-Pro: 4193714: SME DVMSync early acknowledgement"
1180
+
depends on ARM64_SME
1181
+
default y
1182
+
help
1183
+
Enable workaround for C1-Pro acknowledging the DVMSync before
1184
+
the SME memory accesses are complete. This will cause TLB
1185
+
maintenance for processes using SME to also issue an IPI to
1186
+
the affected CPUs.
1187
+
1188
+
If unsure, say Y.
1189
+
1178
1190
config CAVIUM_ERRATUM_22375
1179
1191
bool "Cavium erratum 22375, 24313"
1180
1192
default y
+2
arch/arm64/include/asm/cpucaps.h
+2
arch/arm64/include/asm/cpucaps.h
···
64
64
return IS_ENABLED(CONFIG_ARM64_WORKAROUND_REPEAT_TLBI);
65
65
case ARM64_WORKAROUND_SPECULATIVE_SSBS:
66
66
return IS_ENABLED(CONFIG_ARM64_ERRATUM_3194386);
67
+
case ARM64_WORKAROUND_4193714:
68
+
return IS_ENABLED(CONFIG_ARM64_ERRATUM_4193714);
67
69
case ARM64_MPAM:
68
70
/*
69
71
* KVM MPAM support doesn't rely on the host kernel supporting MPAM.
+21
arch/arm64/include/asm/fpsimd.h
+21
arch/arm64/include/asm/fpsimd.h
···
428
428
return __sme_state_size(task_get_sme_vl(task));
429
429
}
430
430
431
+
void sme_enable_dvmsync(void);
432
+
void sme_set_active(void);
433
+
void sme_clear_active(void);
434
+
435
+
static inline void sme_enter_from_user_mode(void)
436
+
{
437
+
if (alternative_has_cap_unlikely(ARM64_WORKAROUND_4193714) &&
438
+
test_thread_flag(TIF_SME))
439
+
sme_clear_active();
440
+
}
441
+
442
+
static inline void sme_exit_to_user_mode(void)
443
+
{
444
+
if (alternative_has_cap_unlikely(ARM64_WORKAROUND_4193714) &&
445
+
test_thread_flag(TIF_SME))
446
+
sme_set_active();
447
+
}
448
+
431
449
#else
432
450
433
451
static inline void sme_user_disable(void) { BUILD_BUG(); }
···
473
455
{
474
456
return 0;
475
457
}
458
+
459
+
static inline void sme_enter_from_user_mode(void) { }
460
+
static inline void sme_exit_to_user_mode(void) { }
476
461
477
462
#endif /* ! CONFIG_ARM64_SME */
478
463
+8
-2
arch/arm64/include/asm/tlbbatch.h
+8
-2
arch/arm64/include/asm/tlbbatch.h
···
2
2
#ifndef _ARCH_ARM64_TLBBATCH_H
3
3
#define _ARCH_ARM64_TLBBATCH_H
4
4
5
+
#include <linux/cpumask.h>
6
+
5
7
struct arch_tlbflush_unmap_batch {
8
+
#ifdef CONFIG_ARM64_ERRATUM_4193714
6
9
/*
7
-
* For arm64, HW can do tlb shootdown, so we don't
8
-
* need to record cpumask for sending IPI
10
+
* Track CPUs that need SME DVMSync on completion of this batch.
11
+
* Otherwise, the arm64 HW can do tlb shootdown, so we don't need to
12
+
* record cpumask for sending IPI
9
13
*/
14
+
cpumask_var_t cpumask;
15
+
#endif
10
16
};
11
17
12
18
#endif /* _ARCH_ARM64_TLBBATCH_H */
+70
-2
arch/arm64/include/asm/tlbflush.h
+70
-2
arch/arm64/include/asm/tlbflush.h
···
80
80
}
81
81
}
82
82
83
+
#ifdef CONFIG_ARM64_ERRATUM_4193714
84
+
85
+
void sme_do_dvmsync(const struct cpumask *mask);
86
+
87
+
static inline void sme_dvmsync(struct mm_struct *mm)
88
+
{
89
+
if (!alternative_has_cap_unlikely(ARM64_WORKAROUND_4193714))
90
+
return;
91
+
92
+
sme_do_dvmsync(mm_cpumask(mm));
93
+
}
94
+
95
+
static inline void sme_dvmsync_add_pending(struct arch_tlbflush_unmap_batch *batch,
96
+
struct mm_struct *mm)
97
+
{
98
+
if (!alternative_has_cap_unlikely(ARM64_WORKAROUND_4193714))
99
+
return;
100
+
101
+
/*
102
+
* Order the mm_cpumask() read after the hardware DVMSync.
103
+
*/
104
+
dsb(ish);
105
+
if (cpumask_empty(mm_cpumask(mm)))
106
+
return;
107
+
108
+
/*
109
+
* Allocate the batch cpumask on first use. Fall back to an immediate
110
+
* IPI for this mm in case of failure.
111
+
*/
112
+
if (!cpumask_available(batch->cpumask) &&
113
+
!zalloc_cpumask_var(&batch->cpumask, GFP_ATOMIC)) {
114
+
sme_do_dvmsync(mm_cpumask(mm));
115
+
return;
116
+
}
117
+
118
+
cpumask_or(batch->cpumask, batch->cpumask, mm_cpumask(mm));
119
+
}
120
+
121
+
static inline void sme_dvmsync_batch(struct arch_tlbflush_unmap_batch *batch)
122
+
{
123
+
if (!alternative_has_cap_unlikely(ARM64_WORKAROUND_4193714))
124
+
return;
125
+
126
+
if (!cpumask_available(batch->cpumask))
127
+
return;
128
+
129
+
sme_do_dvmsync(batch->cpumask);
130
+
cpumask_clear(batch->cpumask);
131
+
}
132
+
133
+
#else
134
+
135
+
static inline void sme_dvmsync(struct mm_struct *mm)
136
+
{
137
+
}
138
+
static inline void sme_dvmsync_add_pending(struct arch_tlbflush_unmap_batch *batch,
139
+
struct mm_struct *mm)
140
+
{
141
+
}
142
+
static inline void sme_dvmsync_batch(struct arch_tlbflush_unmap_batch *batch)
143
+
{
144
+
}
145
+
146
+
#endif /* CONFIG_ARM64_ERRATUM_4193714 */
147
+
83
148
/*
84
149
* Level-based TLBI operations.
85
150
*
···
254
189
{
255
190
dsb(ish);
256
191
__repeat_tlbi_sync(vale1is, 0);
192
+
sme_dvmsync(mm);
257
193
}
258
194
259
-
static inline void __tlbi_sync_s1ish_batch(void)
195
+
static inline void __tlbi_sync_s1ish_batch(struct arch_tlbflush_unmap_batch *batch)
260
196
{
261
197
dsb(ish);
262
198
__repeat_tlbi_sync(vale1is, 0);
199
+
sme_dvmsync_batch(batch);
263
200
}
264
201
265
202
static inline void __tlbi_sync_s1ish_kernel(void)
···
464
397
*/
465
398
static inline void arch_tlbbatch_flush(struct arch_tlbflush_unmap_batch *batch)
466
399
{
467
-
__tlbi_sync_s1ish_batch();
400
+
__tlbi_sync_s1ish_batch(batch);
468
401
}
469
402
470
403
/*
···
669
602
struct mm_struct *mm, unsigned long start, unsigned long end)
670
603
{
671
604
__flush_tlb_range_nosync(mm, start, end, PAGE_SIZE, true, 3);
605
+
sme_dvmsync_add_pending(batch, mm);
672
606
}
673
607
674
608
static inline bool __pte_flags_need_flush(ptdesc_t oldval, ptdesc_t newval)
+30
arch/arm64/kernel/cpu_errata.c
+30
arch/arm64/kernel/cpu_errata.c
···
11
11
#include <asm/cpu.h>
12
12
#include <asm/cputype.h>
13
13
#include <asm/cpufeature.h>
14
+
#include <asm/fpsimd.h>
14
15
#include <asm/kvm_asm.h>
15
16
#include <asm/smp_plat.h>
16
17
···
576
575
};
577
576
#endif
578
577
578
+
#ifdef CONFIG_ARM64_ERRATUM_4193714
579
+
static bool has_sme_dvmsync_erratum(const struct arm64_cpu_capabilities *entry,
580
+
int scope)
581
+
{
582
+
if (!id_aa64pfr1_sme(read_sanitised_ftr_reg(SYS_ID_AA64PFR1_EL1)))
583
+
return false;
584
+
585
+
return is_affected_midr_range(entry, scope);
586
+
}
587
+
588
+
static void cpu_enable_sme_dvmsync(const struct arm64_cpu_capabilities *__unused)
589
+
{
590
+
if (this_cpu_has_cap(ARM64_WORKAROUND_4193714))
591
+
sme_enable_dvmsync();
592
+
}
593
+
#endif
594
+
579
595
#ifdef CONFIG_AMPERE_ERRATUM_AC03_CPU_38
580
596
static const struct midr_range erratum_ac03_cpu_38_list[] = {
581
597
MIDR_ALL_VERSIONS(MIDR_AMPERE1),
···
917
899
.capability = ARM64_WORKAROUND_4311569,
918
900
.type = ARM64_CPUCAP_SYSTEM_FEATURE,
919
901
.matches = need_arm_si_l1_workaround_4311569,
902
+
},
903
+
#endif
904
+
#ifdef CONFIG_ARM64_ERRATUM_4193714
905
+
{
906
+
.desc = "C1-Pro SME DVMSync early acknowledgement",
907
+
.capability = ARM64_WORKAROUND_4193714,
908
+
.type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM,
909
+
.matches = has_sme_dvmsync_erratum,
910
+
.cpu_enable = cpu_enable_sme_dvmsync,
911
+
/* C1-Pro r0p0 - r1p2 (the latter only when REVIDR_EL1[0]==0) */
912
+
.midr_range = MIDR_RANGE(MIDR_C1_PRO, 0, 0, 1, 2),
913
+
MIDR_FIXED(MIDR_CPU_VAR_REV(1, 2), BIT(0)),
920
914
},
921
915
#endif
922
916
#ifdef CONFIG_ARM64_WORKAROUND_SPECULATIVE_UNPRIV_LOAD
+3
arch/arm64/kernel/entry-common.c
+3
arch/arm64/kernel/entry-common.c
···
21
21
#include <asm/daifflags.h>
22
22
#include <asm/esr.h>
23
23
#include <asm/exception.h>
24
+
#include <asm/fpsimd.h>
24
25
#include <asm/irq_regs.h>
25
26
#include <asm/kprobes.h>
26
27
#include <asm/mmu.h>
···
68
67
{
69
68
enter_from_user_mode(regs);
70
69
mte_disable_tco_entry(current);
70
+
sme_enter_from_user_mode();
71
71
}
72
72
73
73
/*
···
82
80
local_irq_disable();
83
81
exit_to_user_mode_prepare_legacy(regs);
84
82
local_daif_mask();
83
+
sme_exit_to_user_mode();
85
84
mte_check_tfsr_exit();
86
85
exit_to_user_mode();
87
86
}
+79
arch/arm64/kernel/fpsimd.c
+79
arch/arm64/kernel/fpsimd.c
···
15
15
#include <linux/compiler.h>
16
16
#include <linux/cpu.h>
17
17
#include <linux/cpu_pm.h>
18
+
#include <linux/cpumask.h>
18
19
#include <linux/ctype.h>
19
20
#include <linux/kernel.h>
20
21
#include <linux/linkage.h>
···
29
28
#include <linux/sched/task_stack.h>
30
29
#include <linux/signal.h>
31
30
#include <linux/slab.h>
31
+
#include <linux/smp.h>
32
32
#include <linux/stddef.h>
33
33
#include <linux/sysctl.h>
34
34
#include <linux/swab.h>
···
1359
1357
1360
1358
put_cpu_fpsimd_context();
1361
1359
}
1360
+
1361
+
#ifdef CONFIG_ARM64_ERRATUM_4193714
1362
+
1363
+
/*
1364
+
* SME/CME erratum handling.
1365
+
*/
1366
+
static cpumask_t sme_dvmsync_cpus;
1367
+
1368
+
/*
1369
+
* These helpers are only called from non-preemptible contexts, so
1370
+
* smp_processor_id() is safe here.
1371
+
*/
1372
+
void sme_set_active(void)
1373
+
{
1374
+
unsigned int cpu = smp_processor_id();
1375
+
1376
+
if (!cpumask_test_cpu(cpu, &sme_dvmsync_cpus))
1377
+
return;
1378
+
1379
+
cpumask_set_cpu(cpu, mm_cpumask(current->mm));
1380
+
1381
+
/*
1382
+
* A subsequent (post ERET) SME access may use a stale address
1383
+
* translation. On C1-Pro, a TLBI+DSB on a different CPU will wait for
1384
+
* the completion of cpumask_set_cpu() above as it appears in program
1385
+
* order before the SME access. The post-TLBI+DSB read of mm_cpumask()
1386
+
* will lead to the IPI being issued.
1387
+
*
1388
+
* https://lore.kernel.org/r/ablEXwhfKyJW1i7l@J2N7QTR9R3
1389
+
*/
1390
+
}
1391
+
1392
+
void sme_clear_active(void)
1393
+
{
1394
+
unsigned int cpu = smp_processor_id();
1395
+
1396
+
if (!cpumask_test_cpu(cpu, &sme_dvmsync_cpus))
1397
+
return;
1398
+
1399
+
/*
1400
+
* With SCTLR_EL1.IESB enabled, the SME memory transactions are
1401
+
* completed on entering EL1.
1402
+
*/
1403
+
cpumask_clear_cpu(cpu, mm_cpumask(current->mm));
1404
+
}
1405
+
1406
+
static void sme_dvmsync_ipi(void *unused)
1407
+
{
1408
+
/*
1409
+
* With SCTLR_EL1.IESB on, taking an exception is sufficient to ensure
1410
+
* the completion of the SME memory accesses, so no need for an
1411
+
* explicit DSB.
1412
+
*/
1413
+
}
1414
+
1415
+
void sme_do_dvmsync(const struct cpumask *mask)
1416
+
{
1417
+
/*
1418
+
* This is called from the TLB maintenance functions after the DSB ISH
1419
+
* to send the hardware DVMSync message. If this CPU sees the mask as
1420
+
* empty, the remote CPU executing sme_set_active() would have seen
1421
+
* the DVMSync and no IPI required.
1422
+
*/
1423
+
if (cpumask_empty(mask))
1424
+
return;
1425
+
1426
+
preempt_disable();
1427
+
smp_call_function_many(mask, sme_dvmsync_ipi, NULL, true);
1428
+
preempt_enable();
1429
+
}
1430
+
1431
+
void sme_enable_dvmsync(void)
1432
+
{
1433
+
cpumask_set_cpu(smp_processor_id(), &sme_dvmsync_cpus);
1434
+
}
1435
+
1436
+
#endif /* CONFIG_ARM64_ERRATUM_4193714 */
1362
1437
1363
1438
/*
1364
1439
* Trapped SME access
+36
arch/arm64/kernel/process.c
+36
arch/arm64/kernel/process.c
···
26
26
#include <linux/reboot.h>
27
27
#include <linux/interrupt.h>
28
28
#include <linux/init.h>
29
+
#include <linux/cpumask.h>
29
30
#include <linux/cpu.h>
30
31
#include <linux/elfcore.h>
31
32
#include <linux/pm.h>
···
340
339
flush_gcs();
341
340
}
342
341
342
+
#ifdef CONFIG_ARM64_ERRATUM_4193714
343
+
344
+
static void arch_dup_tlbbatch_mask(struct task_struct *dst)
345
+
{
346
+
/*
347
+
* Clear the inherited cpumask with memset() to cover both cases where
348
+
* cpumask_var_t is a pointer or an array. It will be allocated lazily
349
+
* in sme_dvmsync_add_pending() if CPUMASK_OFFSTACK=y.
350
+
*/
351
+
if (alternative_has_cap_unlikely(ARM64_WORKAROUND_4193714))
352
+
memset(&dst->tlb_ubc.arch.cpumask, 0,
353
+
sizeof(dst->tlb_ubc.arch.cpumask));
354
+
}
355
+
356
+
static void arch_release_tlbbatch_mask(struct task_struct *tsk)
357
+
{
358
+
if (alternative_has_cap_unlikely(ARM64_WORKAROUND_4193714))
359
+
free_cpumask_var(tsk->tlb_ubc.arch.cpumask);
360
+
}
361
+
362
+
#else
363
+
364
+
static void arch_dup_tlbbatch_mask(struct task_struct *dst)
365
+
{
366
+
}
367
+
368
+
static void arch_release_tlbbatch_mask(struct task_struct *tsk)
369
+
{
370
+
}
371
+
372
+
#endif /* CONFIG_ARM64_ERRATUM_4193714 */
373
+
343
374
void arch_release_task_struct(struct task_struct *tsk)
344
375
{
376
+
arch_release_tlbbatch_mask(tsk);
345
377
fpsimd_release_task(tsk);
346
378
}
347
379
···
389
355
fpsimd_sync_from_effective_state(src);
390
356
391
357
*dst = *src;
358
+
359
+
arch_dup_tlbbatch_mask(dst);
392
360
393
361
/*
394
362
* Drop stale reference to src's sve_state and convert dst to