Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge tag 'powerpc-4.7-4' of git://git.kernel.org/pub/scm/linux/kernel/git/powerpc/linux
Pull powerpc fixes from Michael Ellerman:
"mm/radix (Aneesh Kumar K.V):
- Update to tlb functions ric argument
- Flush page walk cache when freeing page table
- Update Radix tree size as per ISA 3.0
mm/hash (Aneesh Kumar K.V):
- Use the correct PPP mask when updating HPTE
- Don't add memory coherence if cache inhibited is set
eeh (Gavin Shan):
- Fix invalid cached PE primary bus
bpf/jit (Naveen N. Rao):
- Disable classic BPF JIT on ppc64le
.. and fix faults caused by radix patching of SLB miss handler
(Michael Ellerman)"
* tag 'powerpc-4.7-4' of git://git.kernel.org/pub/scm/linux/kernel/git/powerpc/linux:
powerpc/bpf/jit: Disable classic BPF JIT on ppc64le
powerpc: Fix faults caused by radix patching of SLB miss handler
powerpc/eeh: Fix invalid cached PE primary bus
powerpc/mm/radix: Update Radix tree size as per ISA 3.0
powerpc/mm/hash: Don't add memory coherence if cache inhibited is set
powerpc/mm/hash: Use the correct PPP mask when updating HPTE
powerpc/mm/radix: Flush page walk cache when freeing page table
powerpc/mm/radix: Update to tlb functions ric argument
+137
-51
+1
-1
arch/powerpc/Kconfig
+1
-1
arch/powerpc/Kconfig
···
128
128
select IRQ_FORCED_THREADING
129
129
select HAVE_RCU_TABLE_FREE if SMP
130
130
select HAVE_SYSCALL_TRACEPOINTS
131
-
select HAVE_CBPF_JIT
131
+
select HAVE_CBPF_JIT if CPU_BIG_ENDIAN
132
132
select HAVE_ARCH_JUMP_LABEL
133
133
select ARCH_HAVE_NMI_SAFE_CMPXCHG
134
134
select ARCH_HAS_GCOV_PROFILE_ALL
-1
arch/powerpc/include/asm/book3s/32/pgalloc.h
-1
arch/powerpc/include/asm/book3s/32/pgalloc.h
+1
arch/powerpc/include/asm/book3s/64/mmu-hash.h
+1
arch/powerpc/include/asm/book3s/64/mmu-hash.h
···
88
88
#define HPTE_R_RPN_SHIFT 12
89
89
#define HPTE_R_RPN ASM_CONST(0x0ffffffffffff000)
90
90
#define HPTE_R_PP ASM_CONST(0x0000000000000003)
91
+
#define HPTE_R_PPP ASM_CONST(0x8000000000000003)
91
92
#define HPTE_R_N ASM_CONST(0x0000000000000004)
92
93
#define HPTE_R_G ASM_CONST(0x0000000000000008)
93
94
#define HPTE_R_M ASM_CONST(0x0000000000000010)
+15
-1
arch/powerpc/include/asm/book3s/64/pgalloc.h
+15
-1
arch/powerpc/include/asm/book3s/64/pgalloc.h
···
109
109
static inline void __pud_free_tlb(struct mmu_gather *tlb, pud_t *pud,
110
110
unsigned long address)
111
111
{
112
+
/*
113
+
* By now all the pud entries should be none entries. So go
114
+
* ahead and flush the page walk cache
115
+
*/
116
+
flush_tlb_pgtable(tlb, address);
112
117
pgtable_free_tlb(tlb, pud, PUD_INDEX_SIZE);
113
118
}
114
119
···
130
125
static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd,
131
126
unsigned long address)
132
127
{
128
+
/*
129
+
* By now all the pud entries should be none entries. So go
130
+
* ahead and flush the page walk cache
131
+
*/
132
+
flush_tlb_pgtable(tlb, address);
133
133
return pgtable_free_tlb(tlb, pmd, PMD_CACHE_INDEX);
134
134
}
135
135
···
206
196
static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t table,
207
197
unsigned long address)
208
198
{
209
-
tlb_flush_pgtable(tlb, address);
199
+
/*
200
+
* By now all the pud entries should be none entries. So go
201
+
* ahead and flush the page walk cache
202
+
*/
203
+
flush_tlb_pgtable(tlb, address);
210
204
pgtable_free_tlb(tlb, table, 0);
211
205
}
212
206
+15
arch/powerpc/include/asm/book3s/64/radix.h
+15
arch/powerpc/include/asm/book3s/64/radix.h
···
228
228
229
229
extern int radix__map_kernel_page(unsigned long ea, unsigned long pa,
230
230
pgprot_t flags, unsigned int psz);
231
+
232
+
static inline unsigned long radix__get_tree_size(void)
233
+
{
234
+
unsigned long rts_field;
235
+
/*
236
+
* we support 52 bits, hence 52-31 = 21, 0b10101
237
+
* RTS encoding details
238
+
* bits 0 - 3 of rts -> bits 6 - 8 unsigned long
239
+
* bits 4 - 5 of rts -> bits 62 - 63 of unsigned long
240
+
*/
241
+
rts_field = (0x5UL << 5); /* 6 - 8 bits */
242
+
rts_field |= (0x2UL << 61);
243
+
244
+
return rts_field;
245
+
}
231
246
#endif /* __ASSEMBLY__ */
232
247
#endif
+3
arch/powerpc/include/asm/book3s/64/tlbflush-radix.h
+3
arch/powerpc/include/asm/book3s/64/tlbflush-radix.h
···
18
18
extern void radix__local_flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr);
19
19
extern void radix___local_flush_tlb_page(struct mm_struct *mm, unsigned long vmaddr,
20
20
unsigned long ap, int nid);
21
+
extern void radix__local_flush_tlb_pwc(struct mmu_gather *tlb, unsigned long addr);
21
22
extern void radix__tlb_flush(struct mmu_gather *tlb);
22
23
#ifdef CONFIG_SMP
23
24
extern void radix__flush_tlb_mm(struct mm_struct *mm);
24
25
extern void radix__flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr);
25
26
extern void radix___flush_tlb_page(struct mm_struct *mm, unsigned long vmaddr,
26
27
unsigned long ap, int nid);
28
+
extern void radix__flush_tlb_pwc(struct mmu_gather *tlb, unsigned long addr);
27
29
#else
28
30
#define radix__flush_tlb_mm(mm) radix__local_flush_tlb_mm(mm)
29
31
#define radix__flush_tlb_page(vma,addr) radix__local_flush_tlb_page(vma,addr)
30
32
#define radix___flush_tlb_page(mm,addr,p,i) radix___local_flush_tlb_page(mm,addr,p,i)
33
+
#define radix__flush_tlb_pwc(tlb, addr) radix__local_flush_tlb_pwc(tlb, addr)
31
34
#endif
32
35
33
36
#endif
+14
arch/powerpc/include/asm/book3s/64/tlbflush.h
+14
arch/powerpc/include/asm/book3s/64/tlbflush.h
···
72
72
#define flush_tlb_mm(mm) local_flush_tlb_mm(mm)
73
73
#define flush_tlb_page(vma, addr) local_flush_tlb_page(vma, addr)
74
74
#endif /* CONFIG_SMP */
75
+
/*
76
+
* flush the page walk cache for the address
77
+
*/
78
+
static inline void flush_tlb_pgtable(struct mmu_gather *tlb, unsigned long address)
79
+
{
80
+
/*
81
+
* Flush the page table walk cache on freeing a page table. We already
82
+
* have marked the upper/higher level page table entry none by now.
83
+
* So it is safe to flush PWC here.
84
+
*/
85
+
if (!radix_enabled())
86
+
return;
75
87
88
+
radix__flush_tlb_pwc(tlb, address);
89
+
}
76
90
#endif /* _ASM_POWERPC_BOOK3S_64_TLBFLUSH_H */
-5
arch/powerpc/include/asm/book3s/pgalloc.h
-5
arch/powerpc/include/asm/book3s/pgalloc.h
+4
-3
arch/powerpc/kernel/eeh_driver.c
+4
-3
arch/powerpc/kernel/eeh_driver.c
···
642
642
if (pe->type & EEH_PE_VF) {
643
643
eeh_pe_dev_traverse(pe, eeh_rmv_device, NULL);
644
644
} else {
645
-
eeh_pe_state_clear(pe, EEH_PE_PRI_BUS);
646
645
pci_lock_rescan_remove();
647
646
pci_hp_remove_devices(bus);
648
647
pci_unlock_rescan_remove();
···
691
692
*/
692
693
edev = list_first_entry(&pe->edevs, struct eeh_dev, list);
693
694
eeh_pe_traverse(pe, eeh_pe_detach_dev, NULL);
694
-
if (pe->type & EEH_PE_VF)
695
+
if (pe->type & EEH_PE_VF) {
695
696
eeh_add_virt_device(edev, NULL);
696
-
else
697
+
} else {
698
+
eeh_pe_state_clear(pe, EEH_PE_PRI_BUS);
697
699
pci_hp_add_devices(bus);
700
+
}
698
701
} else if (frozen_bus && rmv_data->removed) {
699
702
pr_info("EEH: Sleep 5s ahead of partial hotplug\n");
700
703
ssleep(5);
+4
-3
arch/powerpc/kernel/exceptions-64s.S
+4
-3
arch/powerpc/kernel/exceptions-64s.S
···
1399
1399
lwz r9,PACA_EXSLB+EX_CCR(r13) /* get saved CR */
1400
1400
1401
1401
mtlr r10
1402
-
BEGIN_MMU_FTR_SECTION
1403
-
b 2f
1404
-
END_MMU_FTR_SECTION_IFSET(MMU_FTR_RADIX)
1405
1402
andi. r10,r12,MSR_RI /* check for unrecoverable exception */
1403
+
BEGIN_MMU_FTR_SECTION
1406
1404
beq- 2f
1405
+
FTR_SECTION_ELSE
1406
+
b 2f
1407
+
ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_RADIX)
1407
1408
1408
1409
.machine push
1409
1410
.machine "power4"
+4
-4
arch/powerpc/mm/hash_native_64.c
+4
-4
arch/powerpc/mm/hash_native_64.c
···
316
316
DBG_LOW(" -> hit\n");
317
317
/* Update the HPTE */
318
318
hptep->r = cpu_to_be64((be64_to_cpu(hptep->r) &
319
-
~(HPTE_R_PP | HPTE_R_N)) |
320
-
(newpp & (HPTE_R_PP | HPTE_R_N |
319
+
~(HPTE_R_PPP | HPTE_R_N)) |
320
+
(newpp & (HPTE_R_PPP | HPTE_R_N |
321
321
HPTE_R_C)));
322
322
}
323
323
native_unlock_hpte(hptep);
···
385
385
386
386
/* Update the HPTE */
387
387
hptep->r = cpu_to_be64((be64_to_cpu(hptep->r) &
388
-
~(HPTE_R_PP | HPTE_R_N)) |
389
-
(newpp & (HPTE_R_PP | HPTE_R_N)));
388
+
~(HPTE_R_PPP | HPTE_R_N)) |
389
+
(newpp & (HPTE_R_PPP | HPTE_R_N)));
390
390
/*
391
391
* Ensure it is out of the tlb too. Bolted entries base and
392
392
* actual page size will be same.
+9
-5
arch/powerpc/mm/hash_utils_64.c
+9
-5
arch/powerpc/mm/hash_utils_64.c
···
201
201
/*
202
202
* We can't allow hardware to update hpte bits. Hence always
203
203
* set 'R' bit and set 'C' if it is a write fault
204
-
* Memory coherence is always enabled
205
204
*/
206
-
rflags |= HPTE_R_R | HPTE_R_M;
205
+
rflags |= HPTE_R_R;
207
206
208
207
if (pteflags & _PAGE_DIRTY)
209
208
rflags |= HPTE_R_C;
···
212
213
213
214
if ((pteflags & _PAGE_CACHE_CTL) == _PAGE_TOLERANT)
214
215
rflags |= HPTE_R_I;
215
-
if ((pteflags & _PAGE_CACHE_CTL ) == _PAGE_NON_IDEMPOTENT)
216
+
else if ((pteflags & _PAGE_CACHE_CTL) == _PAGE_NON_IDEMPOTENT)
216
217
rflags |= (HPTE_R_I | HPTE_R_G);
217
-
if ((pteflags & _PAGE_CACHE_CTL) == _PAGE_SAO)
218
-
rflags |= (HPTE_R_I | HPTE_R_W);
218
+
else if ((pteflags & _PAGE_CACHE_CTL) == _PAGE_SAO)
219
+
rflags |= (HPTE_R_W | HPTE_R_I | HPTE_R_M);
220
+
else
221
+
/*
222
+
* Add memory coherence if cache inhibited is not set
223
+
*/
224
+
rflags |= HPTE_R_M;
219
225
220
226
return rflags;
221
227
}
+1
-1
arch/powerpc/mm/mmu_context_book3s64.c
+1
-1
arch/powerpc/mm/mmu_context_book3s64.c
+3
-6
arch/powerpc/mm/pgtable-radix.c
+3
-6
arch/powerpc/mm/pgtable-radix.c
···
160
160
process_tb = early_alloc_pgtable(1UL << PRTB_SIZE_SHIFT);
161
161
/*
162
162
* Fill in the process table.
163
-
* we support 52 bits, hence 52-28 = 24, 11000
164
163
*/
165
-
rts_field = 3ull << PPC_BITLSHIFT(2);
164
+
rts_field = radix__get_tree_size();
166
165
process_tb->prtb0 = cpu_to_be64(rts_field | __pa(init_mm.pgd) | RADIX_PGD_INDEX_SIZE);
167
166
/*
168
167
* Fill in the partition table. We are suppose to use effective address
···
175
176
static void __init radix_init_partition_table(void)
176
177
{
177
178
unsigned long rts_field;
178
-
/*
179
-
* we support 52 bits, hence 52-28 = 24, 11000
180
-
*/
181
-
rts_field = 3ull << PPC_BITLSHIFT(2);
179
+
180
+
rts_field = radix__get_tree_size();
182
181
183
182
BUILD_BUG_ON_MSG((PATB_SIZE_SHIFT > 24), "Partition table size too large.");
184
183
partition_tb = early_alloc_pgtable(1UL << PATB_SIZE_SHIFT);
+63
-21
arch/powerpc/mm/tlb-radix.c
+63
-21
arch/powerpc/mm/tlb-radix.c
···
18
18
19
19
static DEFINE_RAW_SPINLOCK(native_tlbie_lock);
20
20
21
-
static inline void __tlbiel_pid(unsigned long pid, int set)
21
+
#define RIC_FLUSH_TLB 0
22
+
#define RIC_FLUSH_PWC 1
23
+
#define RIC_FLUSH_ALL 2
24
+
25
+
static inline void __tlbiel_pid(unsigned long pid, int set,
26
+
unsigned long ric)
22
27
{
23
-
unsigned long rb,rs,ric,prs,r;
28
+
unsigned long rb,rs,prs,r;
24
29
25
30
rb = PPC_BIT(53); /* IS = 1 */
26
31
rb |= set << PPC_BITLSHIFT(51);
27
32
rs = ((unsigned long)pid) << PPC_BITLSHIFT(31);
28
33
prs = 1; /* process scoped */
29
34
r = 1; /* raidx format */
30
-
ric = 2; /* invalidate all the caches */
31
35
32
36
asm volatile("ptesync": : :"memory");
33
37
asm volatile(".long 0x7c000224 | (%0 << 11) | (%1 << 16) |"
···
43
39
/*
44
40
* We use 128 set in radix mode and 256 set in hpt mode.
45
41
*/
46
-
static inline void _tlbiel_pid(unsigned long pid)
42
+
static inline void _tlbiel_pid(unsigned long pid, unsigned long ric)
47
43
{
48
44
int set;
49
45
50
46
for (set = 0; set < POWER9_TLB_SETS_RADIX ; set++) {
51
-
__tlbiel_pid(pid, set);
47
+
__tlbiel_pid(pid, set, ric);
52
48
}
53
49
return;
54
50
}
55
51
56
-
static inline void _tlbie_pid(unsigned long pid)
52
+
static inline void _tlbie_pid(unsigned long pid, unsigned long ric)
57
53
{
58
-
unsigned long rb,rs,ric,prs,r;
54
+
unsigned long rb,rs,prs,r;
59
55
60
56
rb = PPC_BIT(53); /* IS = 1 */
61
57
rs = pid << PPC_BITLSHIFT(31);
62
58
prs = 1; /* process scoped */
63
59
r = 1; /* raidx format */
64
-
ric = 2; /* invalidate all the caches */
65
60
66
61
asm volatile("ptesync": : :"memory");
67
62
asm volatile(".long 0x7c000264 | (%0 << 11) | (%1 << 16) |"
···
70
67
}
71
68
72
69
static inline void _tlbiel_va(unsigned long va, unsigned long pid,
73
-
unsigned long ap)
70
+
unsigned long ap, unsigned long ric)
74
71
{
75
-
unsigned long rb,rs,ric,prs,r;
72
+
unsigned long rb,rs,prs,r;
76
73
77
74
rb = va & ~(PPC_BITMASK(52, 63));
78
75
rb |= ap << PPC_BITLSHIFT(58);
79
76
rs = pid << PPC_BITLSHIFT(31);
80
77
prs = 1; /* process scoped */
81
78
r = 1; /* raidx format */
82
-
ric = 0; /* no cluster flush yet */
83
79
84
80
asm volatile("ptesync": : :"memory");
85
81
asm volatile(".long 0x7c000224 | (%0 << 11) | (%1 << 16) |"
···
88
86
}
89
87
90
88
static inline void _tlbie_va(unsigned long va, unsigned long pid,
91
-
unsigned long ap)
89
+
unsigned long ap, unsigned long ric)
92
90
{
93
-
unsigned long rb,rs,ric,prs,r;
91
+
unsigned long rb,rs,prs,r;
94
92
95
93
rb = va & ~(PPC_BITMASK(52, 63));
96
94
rb |= ap << PPC_BITLSHIFT(58);
97
95
rs = pid << PPC_BITLSHIFT(31);
98
96
prs = 1; /* process scoped */
99
97
r = 1; /* raidx format */
100
-
ric = 0; /* no cluster flush yet */
101
98
102
99
asm volatile("ptesync": : :"memory");
103
100
asm volatile(".long 0x7c000264 | (%0 << 11) | (%1 << 16) |"
···
123
122
preempt_disable();
124
123
pid = mm->context.id;
125
124
if (pid != MMU_NO_CONTEXT)
126
-
_tlbiel_pid(pid);
125
+
_tlbiel_pid(pid, RIC_FLUSH_ALL);
127
126
preempt_enable();
128
127
}
129
128
EXPORT_SYMBOL(radix__local_flush_tlb_mm);
129
+
130
+
void radix__local_flush_tlb_pwc(struct mmu_gather *tlb, unsigned long addr)
131
+
{
132
+
unsigned long pid;
133
+
struct mm_struct *mm = tlb->mm;
134
+
135
+
preempt_disable();
136
+
137
+
pid = mm->context.id;
138
+
if (pid != MMU_NO_CONTEXT)
139
+
_tlbiel_pid(pid, RIC_FLUSH_PWC);
140
+
141
+
preempt_enable();
142
+
}
143
+
EXPORT_SYMBOL(radix__local_flush_tlb_pwc);
130
144
131
145
void radix___local_flush_tlb_page(struct mm_struct *mm, unsigned long vmaddr,
132
146
unsigned long ap, int nid)
···
151
135
preempt_disable();
152
136
pid = mm ? mm->context.id : 0;
153
137
if (pid != MMU_NO_CONTEXT)
154
-
_tlbiel_va(vmaddr, pid, ap);
138
+
_tlbiel_va(vmaddr, pid, ap, RIC_FLUSH_TLB);
155
139
preempt_enable();
156
140
}
157
141
···
188
172
189
173
if (lock_tlbie)
190
174
raw_spin_lock(&native_tlbie_lock);
191
-
_tlbie_pid(pid);
175
+
_tlbie_pid(pid, RIC_FLUSH_ALL);
192
176
if (lock_tlbie)
193
177
raw_spin_unlock(&native_tlbie_lock);
194
178
} else
195
-
_tlbiel_pid(pid);
179
+
_tlbiel_pid(pid, RIC_FLUSH_ALL);
196
180
no_context:
197
181
preempt_enable();
198
182
}
199
183
EXPORT_SYMBOL(radix__flush_tlb_mm);
184
+
185
+
void radix__flush_tlb_pwc(struct mmu_gather *tlb, unsigned long addr)
186
+
{
187
+
unsigned long pid;
188
+
struct mm_struct *mm = tlb->mm;
189
+
190
+
preempt_disable();
191
+
192
+
pid = mm->context.id;
193
+
if (unlikely(pid == MMU_NO_CONTEXT))
194
+
goto no_context;
195
+
196
+
if (!mm_is_core_local(mm)) {
197
+
int lock_tlbie = !mmu_has_feature(MMU_FTR_LOCKLESS_TLBIE);
198
+
199
+
if (lock_tlbie)
200
+
raw_spin_lock(&native_tlbie_lock);
201
+
_tlbie_pid(pid, RIC_FLUSH_PWC);
202
+
if (lock_tlbie)
203
+
raw_spin_unlock(&native_tlbie_lock);
204
+
} else
205
+
_tlbiel_pid(pid, RIC_FLUSH_PWC);
206
+
no_context:
207
+
preempt_enable();
208
+
}
209
+
EXPORT_SYMBOL(radix__flush_tlb_pwc);
200
210
201
211
void radix___flush_tlb_page(struct mm_struct *mm, unsigned long vmaddr,
202
212
unsigned long ap, int nid)
···
238
196
239
197
if (lock_tlbie)
240
198
raw_spin_lock(&native_tlbie_lock);
241
-
_tlbie_va(vmaddr, pid, ap);
199
+
_tlbie_va(vmaddr, pid, ap, RIC_FLUSH_TLB);
242
200
if (lock_tlbie)
243
201
raw_spin_unlock(&native_tlbie_lock);
244
202
} else
245
-
_tlbiel_va(vmaddr, pid, ap);
203
+
_tlbiel_va(vmaddr, pid, ap, RIC_FLUSH_TLB);
246
204
bail:
247
205
preempt_enable();
248
206
}
···
266
224
267
225
if (lock_tlbie)
268
226
raw_spin_lock(&native_tlbie_lock);
269
-
_tlbie_pid(0);
227
+
_tlbie_pid(0, RIC_FLUSH_ALL);
270
228
if (lock_tlbie)
271
229
raw_spin_unlock(&native_tlbie_lock);
272
230
}