Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge tag 'powerpc-4.16-6' of git://git.kernel.org/pub/scm/linux/kernel/git/powerpc/linux
Pull powerpc fixes from Michael Ellerman:
"Some more powerpc fixes for 4.16. Apologies if this is a bit big at
rc7, but they're all reasonably important fixes. None are actually for
new code, so they aren't indicative of 4.16 being in bad shape from
our point of view.
- Fix missing AT_BASE_PLATFORM (in auxv) when we're using a new
firmware interface for describing CPU features.
- Fix lost pending interrupts due to a race in our interrupt
soft-masking code.
- A workaround for a nest MMU bug with TLB invalidations on Power9.
- A workaround for broadcast TLB invalidations on Power9.
- Fix a bug in our instruction SLB miss handler, when handling bad
addresses (eg. >= TASK_SIZE), which could corrupt non-volatile user
GPRs.
Thanks to: Aneesh Kumar K.V, Balbir Singh, Benjamin Herrenschmidt,
Nicholas Piggin"
* tag 'powerpc-4.16-6' of git://git.kernel.org/pub/scm/linux/kernel/git/powerpc/linux:
powerpc/64s: Fix i-side SLB miss bad address handler saving nonvolatile GPRs
powerpc/mm: Fixup tlbie vs store ordering issue on POWER9
powerpc/mm/radix: Move the functions that does the actual tlbie closer
powerpc/mm/radix: Remove unused code
powerpc/mm: Workaround Nest MMU bug with TLB invalidations
powerpc/mm: Add tracking of the number of coprocessors using a context
powerpc/64s: Fix lost pending interrupt due to race causing lost update to irq_happened
powerpc/64s: Fix NULL AT_BASE_PLATFORM when using DT CPU features
+154
-90
+3
arch/powerpc/include/asm/book3s/64/mmu.h
+3
arch/powerpc/include/asm/book3s/64/mmu.h
-3
arch/powerpc/include/asm/book3s/64/tlbflush-radix.h
-3
arch/powerpc/include/asm/book3s/64/tlbflush-radix.h
···
47
47
#endif
48
48
extern void radix__flush_tlb_pwc(struct mmu_gather *tlb, unsigned long addr);
49
49
extern void radix__flush_tlb_collapsed_pmd(struct mm_struct *mm, unsigned long addr);
50
-
extern void radix__flush_tlb_lpid_va(unsigned long lpid, unsigned long gpa,
51
-
unsigned long page_size);
52
-
extern void radix__flush_tlb_lpid(unsigned long lpid);
53
50
extern void radix__flush_tlb_all(void);
54
51
extern void radix__flush_tlb_pte_p9_dd1(unsigned long old_pte, struct mm_struct *mm,
55
52
unsigned long address);
+2
-1
arch/powerpc/include/asm/cputable.h
+2
-1
arch/powerpc/include/asm/cputable.h
···
203
203
#define CPU_FTR_DAWR LONG_ASM_CONST(0x0400000000000000)
204
204
#define CPU_FTR_DABRX LONG_ASM_CONST(0x0800000000000000)
205
205
#define CPU_FTR_PMAO_BUG LONG_ASM_CONST(0x1000000000000000)
206
+
#define CPU_FTR_P9_TLBIE_BUG LONG_ASM_CONST(0x2000000000000000)
206
207
#define CPU_FTR_POWER9_DD1 LONG_ASM_CONST(0x4000000000000000)
207
208
#define CPU_FTR_POWER9_DD2_1 LONG_ASM_CONST(0x8000000000000000)
208
209
···
466
465
CPU_FTR_CFAR | CPU_FTR_HVMODE | CPU_FTR_VMX_COPY | \
467
466
CPU_FTR_DBELL | CPU_FTR_HAS_PPR | CPU_FTR_DAWR | \
468
467
CPU_FTR_ARCH_207S | CPU_FTR_TM_COMP | CPU_FTR_ARCH_300 | \
469
-
CPU_FTR_PKEY)
468
+
CPU_FTR_PKEY | CPU_FTR_P9_TLBIE_BUG)
470
469
#define CPU_FTRS_POWER9_DD1 ((CPU_FTRS_POWER9 | CPU_FTR_POWER9_DD1) & \
471
470
(~CPU_FTR_SAO))
472
471
#define CPU_FTRS_POWER9_DD2_0 CPU_FTRS_POWER9
+13
-5
arch/powerpc/include/asm/mmu_context.h
+13
-5
arch/powerpc/include/asm/mmu_context.h
···
92
92
static inline void mm_context_add_copro(struct mm_struct *mm)
93
93
{
94
94
/*
95
-
* On hash, should only be called once over the lifetime of
96
-
* the context, as we can't decrement the active cpus count
97
-
* and flush properly for the time being.
95
+
* If any copro is in use, increment the active CPU count
96
+
* in order to force TLB invalidations to be global as to
97
+
* propagate to the Nest MMU.
98
98
*/
99
-
inc_mm_active_cpus(mm);
99
+
if (atomic_inc_return(&mm->context.copros) == 1)
100
+
inc_mm_active_cpus(mm);
100
101
}
101
102
102
103
static inline void mm_context_remove_copro(struct mm_struct *mm)
103
104
{
105
+
int c;
106
+
107
+
c = atomic_dec_if_positive(&mm->context.copros);
108
+
109
+
/* Detect imbalance between add and remove */
110
+
WARN_ON(c < 0);
111
+
104
112
/*
105
113
* Need to broadcast a global flush of the full mm before
106
114
* decrementing active_cpus count, as the next TLBI may be
···
119
111
* for the time being. Invalidations will remain global if
120
112
* used on hash.
121
113
*/
122
-
if (radix_enabled()) {
114
+
if (c == 0 && radix_enabled()) {
123
115
flush_all_mm(mm);
124
116
dec_mm_active_cpus(mm);
125
117
}
+6
arch/powerpc/kernel/dt_cpu_ftrs.c
+6
arch/powerpc/kernel/dt_cpu_ftrs.c
···
709
709
cur_cpu_spec->cpu_features |= CPU_FTR_POWER9_DD1;
710
710
else if ((version & 0xffffefff) == 0x004e0201)
711
711
cur_cpu_spec->cpu_features |= CPU_FTR_POWER9_DD2_1;
712
+
713
+
if ((version & 0xffff0000) == 0x004e0000)
714
+
cur_cpu_spec->cpu_features |= CPU_FTR_P9_TLBIE_BUG;
712
715
}
713
716
714
717
static void __init cpufeatures_setup_finished(void)
···
722
719
pr_err("hypervisor not present in device tree but HV mode is enabled in the CPU. Enabling.\n");
723
720
cur_cpu_spec->cpu_features |= CPU_FTR_HVMODE;
724
721
}
722
+
723
+
/* Make sure powerpc_base_platform is non-NULL */
724
+
powerpc_base_platform = cur_cpu_spec->platform;
725
725
726
726
system_registers.lpcr = mfspr(SPRN_LPCR);
727
727
system_registers.hfscr = mfspr(SPRN_HFSCR);
+1
-1
arch/powerpc/kernel/exceptions-64s.S
+1
-1
arch/powerpc/kernel/exceptions-64s.S
+8
arch/powerpc/kernel/irq.c
+8
arch/powerpc/kernel/irq.c
···
476
476
*/
477
477
WARN_ON(!arch_irqs_disabled());
478
478
479
+
/*
480
+
* Interrupts must always be hard disabled before irq_happened is
481
+
* modified (to prevent lost update in case of interrupt between
482
+
* load and store).
483
+
*/
484
+
__hard_irq_disable();
485
+
local_paca->irq_happened |= PACA_IRQ_HARD_DIS;
486
+
479
487
/* Indicate in the PACA that we have an interrupt to replay */
480
488
local_paca->irq_happened |= PACA_IRQ_EE;
481
489
}
+3
arch/powerpc/kvm/book3s_64_mmu_radix.c
+3
arch/powerpc/kvm/book3s_64_mmu_radix.c
···
157
157
asm volatile("ptesync": : :"memory");
158
158
asm volatile(PPC_TLBIE_5(%0, %1, 0, 0, 1)
159
159
: : "r" (addr), "r" (kvm->arch.lpid) : "memory");
160
+
if (cpu_has_feature(CPU_FTR_P9_TLBIE_BUG))
161
+
asm volatile(PPC_TLBIE_5(%0, %1, 0, 0, 1)
162
+
: : "r" (addr), "r" (kvm->arch.lpid) : "memory");
160
163
asm volatile("ptesync": : :"memory");
161
164
}
162
165
+11
arch/powerpc/kvm/book3s_hv_rm_mmu.c
+11
arch/powerpc/kvm/book3s_hv_rm_mmu.c
···
473
473
trace_tlbie(kvm->arch.lpid, 0, rbvalues[i],
474
474
kvm->arch.lpid, 0, 0, 0);
475
475
}
476
+
477
+
if (cpu_has_feature(CPU_FTR_P9_TLBIE_BUG)) {
478
+
/*
479
+
* Need the extra ptesync to make sure we don't
480
+
* re-order the tlbie
481
+
*/
482
+
asm volatile("ptesync": : :"memory");
483
+
asm volatile(PPC_TLBIE_5(%0,%1,0,0,0) : :
484
+
"r" (rbvalues[0]), "r" (kvm->arch.lpid));
485
+
}
486
+
476
487
asm volatile("eieio; tlbsync; ptesync" : : : "memory");
477
488
kvm->arch.tlbie_lock = 0;
478
489
} else {
+15
-1
arch/powerpc/mm/hash_native_64.c
+15
-1
arch/powerpc/mm/hash_native_64.c
···
201
201
return va;
202
202
}
203
203
204
+
static inline void fixup_tlbie(unsigned long vpn, int psize, int apsize, int ssize)
205
+
{
206
+
if (cpu_has_feature(CPU_FTR_P9_TLBIE_BUG)) {
207
+
/* Need the extra ptesync to ensure we don't reorder tlbie*/
208
+
asm volatile("ptesync": : :"memory");
209
+
___tlbie(vpn, psize, apsize, ssize);
210
+
}
211
+
}
212
+
204
213
static inline void __tlbie(unsigned long vpn, int psize, int apsize, int ssize)
205
214
{
206
215
unsigned long rb;
···
287
278
asm volatile("ptesync": : :"memory");
288
279
} else {
289
280
__tlbie(vpn, psize, apsize, ssize);
281
+
fixup_tlbie(vpn, psize, apsize, ssize);
290
282
asm volatile("eieio; tlbsync; ptesync": : :"memory");
291
283
}
292
284
if (lock_tlbie && !use_local)
···
781
771
*/
782
772
static void native_flush_hash_range(unsigned long number, int local)
783
773
{
784
-
unsigned long vpn;
774
+
unsigned long vpn = 0;
785
775
unsigned long hash, index, hidx, shift, slot;
786
776
struct hash_pte *hptep;
787
777
unsigned long hpte_v;
···
853
843
__tlbie(vpn, psize, psize, ssize);
854
844
} pte_iterate_hashed_end();
855
845
}
846
+
/*
847
+
* Just do one more with the last used values.
848
+
*/
849
+
fixup_tlbie(vpn, psize, psize, ssize);
856
850
asm volatile("eieio; tlbsync; ptesync":::"memory");
857
851
858
852
if (lock_tlbie)
+1
arch/powerpc/mm/mmu_context_book3s64.c
+1
arch/powerpc/mm/mmu_context_book3s64.c
+1
arch/powerpc/mm/pgtable_64.c
+1
arch/powerpc/mm/pgtable_64.c
+90
-79
arch/powerpc/mm/tlb-radix.c
+90
-79
arch/powerpc/mm/tlb-radix.c
···
119
119
trace_tlbie(0, 0, rb, rs, ric, prs, r);
120
120
}
121
121
122
+
static inline void __tlbiel_va(unsigned long va, unsigned long pid,
123
+
unsigned long ap, unsigned long ric)
124
+
{
125
+
unsigned long rb,rs,prs,r;
126
+
127
+
rb = va & ~(PPC_BITMASK(52, 63));
128
+
rb |= ap << PPC_BITLSHIFT(58);
129
+
rs = pid << PPC_BITLSHIFT(31);
130
+
prs = 1; /* process scoped */
131
+
r = 1; /* raidx format */
132
+
133
+
asm volatile(PPC_TLBIEL(%0, %4, %3, %2, %1)
134
+
: : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");
135
+
trace_tlbie(0, 1, rb, rs, ric, prs, r);
136
+
}
137
+
138
+
static inline void __tlbie_va(unsigned long va, unsigned long pid,
139
+
unsigned long ap, unsigned long ric)
140
+
{
141
+
unsigned long rb,rs,prs,r;
142
+
143
+
rb = va & ~(PPC_BITMASK(52, 63));
144
+
rb |= ap << PPC_BITLSHIFT(58);
145
+
rs = pid << PPC_BITLSHIFT(31);
146
+
prs = 1; /* process scoped */
147
+
r = 1; /* raidx format */
148
+
149
+
asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1)
150
+
: : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");
151
+
trace_tlbie(0, 0, rb, rs, ric, prs, r);
152
+
}
153
+
154
+
static inline void fixup_tlbie(void)
155
+
{
156
+
unsigned long pid = 0;
157
+
unsigned long va = ((1UL << 52) - 1);
158
+
159
+
if (cpu_has_feature(CPU_FTR_P9_TLBIE_BUG)) {
160
+
asm volatile("ptesync": : :"memory");
161
+
__tlbie_va(va, pid, mmu_get_ap(MMU_PAGE_64K), RIC_FLUSH_TLB);
162
+
}
163
+
}
164
+
122
165
/*
123
166
* We use 128 set in radix mode and 256 set in hpt mode.
124
167
*/
···
194
151
static inline void _tlbie_pid(unsigned long pid, unsigned long ric)
195
152
{
196
153
asm volatile("ptesync": : :"memory");
197
-
__tlbie_pid(pid, ric);
154
+
155
+
/*
156
+
* Workaround the fact that the "ric" argument to __tlbie_pid
157
+
* must be a compile-time contraint to match the "i" constraint
158
+
* in the asm statement.
159
+
*/
160
+
switch (ric) {
161
+
case RIC_FLUSH_TLB:
162
+
__tlbie_pid(pid, RIC_FLUSH_TLB);
163
+
break;
164
+
case RIC_FLUSH_PWC:
165
+
__tlbie_pid(pid, RIC_FLUSH_PWC);
166
+
break;
167
+
case RIC_FLUSH_ALL:
168
+
default:
169
+
__tlbie_pid(pid, RIC_FLUSH_ALL);
170
+
}
171
+
fixup_tlbie();
198
172
asm volatile("eieio; tlbsync; ptesync": : :"memory");
199
-
}
200
-
201
-
static inline void __tlbiel_va(unsigned long va, unsigned long pid,
202
-
unsigned long ap, unsigned long ric)
203
-
{
204
-
unsigned long rb,rs,prs,r;
205
-
206
-
rb = va & ~(PPC_BITMASK(52, 63));
207
-
rb |= ap << PPC_BITLSHIFT(58);
208
-
rs = pid << PPC_BITLSHIFT(31);
209
-
prs = 1; /* process scoped */
210
-
r = 1; /* raidx format */
211
-
212
-
asm volatile(PPC_TLBIEL(%0, %4, %3, %2, %1)
213
-
: : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");
214
-
trace_tlbie(0, 1, rb, rs, ric, prs, r);
215
173
}
216
174
217
175
static inline void __tlbiel_va_range(unsigned long start, unsigned long end,
···
247
203
asm volatile("ptesync": : :"memory");
248
204
}
249
205
250
-
static inline void __tlbie_va(unsigned long va, unsigned long pid,
251
-
unsigned long ap, unsigned long ric)
252
-
{
253
-
unsigned long rb,rs,prs,r;
254
-
255
-
rb = va & ~(PPC_BITMASK(52, 63));
256
-
rb |= ap << PPC_BITLSHIFT(58);
257
-
rs = pid << PPC_BITLSHIFT(31);
258
-
prs = 1; /* process scoped */
259
-
r = 1; /* raidx format */
260
-
261
-
asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1)
262
-
: : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");
263
-
trace_tlbie(0, 0, rb, rs, ric, prs, r);
264
-
}
265
-
266
206
static inline void __tlbie_va_range(unsigned long start, unsigned long end,
267
207
unsigned long pid, unsigned long page_size,
268
208
unsigned long psize)
···
265
237
266
238
asm volatile("ptesync": : :"memory");
267
239
__tlbie_va(va, pid, ap, ric);
240
+
fixup_tlbie();
268
241
asm volatile("eieio; tlbsync; ptesync": : :"memory");
269
242
}
270
243
···
277
248
if (also_pwc)
278
249
__tlbie_pid(pid, RIC_FLUSH_PWC);
279
250
__tlbie_va_range(start, end, pid, page_size, psize);
251
+
fixup_tlbie();
280
252
asm volatile("eieio; tlbsync; ptesync": : :"memory");
281
253
}
282
254
···
341
311
}
342
312
EXPORT_SYMBOL(radix__local_flush_tlb_page);
343
313
314
+
static bool mm_needs_flush_escalation(struct mm_struct *mm)
315
+
{
316
+
/*
317
+
* P9 nest MMU has issues with the page walk cache
318
+
* caching PTEs and not flushing them properly when
319
+
* RIC = 0 for a PID/LPID invalidate
320
+
*/
321
+
return atomic_read(&mm->context.copros) != 0;
322
+
}
323
+
344
324
#ifdef CONFIG_SMP
345
325
void radix__flush_tlb_mm(struct mm_struct *mm)
346
326
{
···
361
321
return;
362
322
363
323
preempt_disable();
364
-
if (!mm_is_thread_local(mm))
365
-
_tlbie_pid(pid, RIC_FLUSH_TLB);
366
-
else
324
+
if (!mm_is_thread_local(mm)) {
325
+
if (mm_needs_flush_escalation(mm))
326
+
_tlbie_pid(pid, RIC_FLUSH_ALL);
327
+
else
328
+
_tlbie_pid(pid, RIC_FLUSH_TLB);
329
+
} else
367
330
_tlbiel_pid(pid, RIC_FLUSH_TLB);
368
331
preempt_enable();
369
332
}
···
478
435
}
479
436
480
437
if (full) {
481
-
if (local)
438
+
if (local) {
482
439
_tlbiel_pid(pid, RIC_FLUSH_TLB);
483
-
else
484
-
_tlbie_pid(pid, RIC_FLUSH_TLB);
440
+
} else {
441
+
if (mm_needs_flush_escalation(mm))
442
+
_tlbie_pid(pid, RIC_FLUSH_ALL);
443
+
else
444
+
_tlbie_pid(pid, RIC_FLUSH_TLB);
445
+
}
485
446
} else {
486
447
bool hflush = false;
487
448
unsigned long hstart, hend;
···
512
465
if (hflush)
513
466
__tlbie_va_range(hstart, hend, pid,
514
467
HPAGE_PMD_SIZE, MMU_PAGE_2M);
468
+
fixup_tlbie();
515
469
asm volatile("eieio; tlbsync; ptesync": : :"memory");
516
470
}
517
471
}
···
596
548
}
597
549
598
550
if (full) {
551
+
if (!local && mm_needs_flush_escalation(mm))
552
+
also_pwc = true;
553
+
599
554
if (local)
600
555
_tlbiel_pid(pid, also_pwc ? RIC_FLUSH_ALL : RIC_FLUSH_TLB);
601
556
else
···
653
602
preempt_enable();
654
603
}
655
604
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
656
-
657
-
void radix__flush_tlb_lpid_va(unsigned long lpid, unsigned long gpa,
658
-
unsigned long page_size)
659
-
{
660
-
unsigned long rb,rs,prs,r;
661
-
unsigned long ap;
662
-
unsigned long ric = RIC_FLUSH_TLB;
663
-
664
-
ap = mmu_get_ap(radix_get_mmu_psize(page_size));
665
-
rb = gpa & ~(PPC_BITMASK(52, 63));
666
-
rb |= ap << PPC_BITLSHIFT(58);
667
-
rs = lpid & ((1UL << 32) - 1);
668
-
prs = 0; /* process scoped */
669
-
r = 1; /* raidx format */
670
-
671
-
asm volatile("ptesync": : :"memory");
672
-
asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1)
673
-
: : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");
674
-
asm volatile("eieio; tlbsync; ptesync": : :"memory");
675
-
trace_tlbie(lpid, 0, rb, rs, ric, prs, r);
676
-
}
677
-
EXPORT_SYMBOL(radix__flush_tlb_lpid_va);
678
-
679
-
void radix__flush_tlb_lpid(unsigned long lpid)
680
-
{
681
-
unsigned long rb,rs,prs,r;
682
-
unsigned long ric = RIC_FLUSH_ALL;
683
-
684
-
rb = 0x2 << PPC_BITLSHIFT(53); /* IS = 2 */
685
-
rs = lpid & ((1UL << 32) - 1);
686
-
prs = 0; /* partition scoped */
687
-
r = 1; /* raidx format */
688
-
689
-
asm volatile("ptesync": : :"memory");
690
-
asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1)
691
-
: : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");
692
-
asm volatile("eieio; tlbsync; ptesync": : :"memory");
693
-
trace_tlbie(lpid, 0, rb, rs, ric, prs, r);
694
-
}
695
-
EXPORT_SYMBOL(radix__flush_tlb_lpid);
696
605
697
606
void radix__flush_pmd_tlb_range(struct vm_area_struct *vma,
698
607
unsigned long start, unsigned long end)