Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
xtensa: implement the new page table range API
Add PFN_PTE_SHIFT, update_mmu_cache_range(), flush_dcache_folio() and
flush_icache_pages().
Link: https://lkml.kernel.org/r/20230802151406.3735276-30-willy@infradead.org
Signed-off-by: Matthew Wilcox (Oracle) <willy@infradead.org>
Acked-by: Mike Rapoport (IBM) <rppt@kernel.org>
Cc: Max Filippov <jcmvbkbc@gmail.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
authored by
Matthew Wilcox (Oracle)
and committed by
Andrew Morton
4fbb7e7f
a3e1c937
+64
-48
+8
-1
arch/xtensa/include/asm/cacheflush.h
+8
-1
arch/xtensa/include/asm/cacheflush.h
···
119
119
#define flush_cache_vmap(start,end) flush_cache_all()
120
120
#define flush_cache_vunmap(start,end) flush_cache_all()
121
121
122
+
void flush_dcache_folio(struct folio *folio);
123
+
#define flush_dcache_folio flush_dcache_folio
124
+
122
125
#define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE 1
123
-
void flush_dcache_page(struct page *);
126
+
static inline void flush_dcache_page(struct page *page)
127
+
{
128
+
flush_dcache_folio(page_folio(page));
129
+
}
124
130
125
131
void local_flush_cache_range(struct vm_area_struct *vma,
126
132
unsigned long start, unsigned long end);
···
162
156
163
157
/* This is not required, see Documentation/core-api/cachetlb.rst */
164
158
#define flush_icache_page(vma,page) do { } while (0)
159
+
#define flush_icache_pages(vma, page, nr) do { } while (0)
165
160
166
161
#define flush_dcache_mmap_lock(mapping) do { } while (0)
167
162
#define flush_dcache_mmap_unlock(mapping) do { } while (0)
+8
-10
arch/xtensa/include/asm/pgtable.h
+8
-10
arch/xtensa/include/asm/pgtable.h
···
274
274
* and a page entry and page directory to the page they refer to.
275
275
*/
276
276
277
+
#define PFN_PTE_SHIFT PAGE_SHIFT
277
278
#define pte_pfn(pte) (pte_val(pte) >> PAGE_SHIFT)
278
279
#define pte_same(a,b) (pte_val(a) == pte_val(b))
279
280
#define pte_page(x) pfn_to_page(pte_pfn(x))
···
302
301
303
302
struct mm_struct;
304
303
305
-
static inline void
306
-
set_pte_at(struct mm_struct *mm, unsigned long addr, pte_t *ptep, pte_t pteval)
304
+
static inline void set_pte(pte_t *ptep, pte_t pte)
307
305
{
308
-
update_pte(ptep, pteval);
309
-
}
310
-
311
-
static inline void set_pte(pte_t *ptep, pte_t pteval)
312
-
{
313
-
update_pte(ptep, pteval);
306
+
update_pte(ptep, pte);
314
307
}
315
308
316
309
static inline void
···
402
407
403
408
#else
404
409
405
-
extern void update_mmu_cache(struct vm_area_struct * vma,
406
-
unsigned long address, pte_t *ptep);
410
+
struct vm_fault;
411
+
void update_mmu_cache_range(struct vm_fault *vmf, struct vm_area_struct *vma,
412
+
unsigned long address, pte_t *ptep, unsigned int nr);
413
+
#define update_mmu_cache(vma, address, ptep) \
414
+
update_mmu_cache_range(NULL, vma, address, ptep, 1)
407
415
408
416
typedef pte_t *pte_addr_t;
409
417
+48
-37
arch/xtensa/mm/cache.c
+48
-37
arch/xtensa/mm/cache.c
···
121
121
*
122
122
*/
123
123
124
-
void flush_dcache_page(struct page *page)
124
+
void flush_dcache_folio(struct folio *folio)
125
125
{
126
-
struct address_space *mapping = page_mapping_file(page);
126
+
struct address_space *mapping = folio_flush_mapping(folio);
127
127
128
128
/*
129
129
* If we have a mapping but the page is not mapped to user-space
···
132
132
*/
133
133
134
134
if (mapping && !mapping_mapped(mapping)) {
135
-
if (!test_bit(PG_arch_1, &page->flags))
136
-
set_bit(PG_arch_1, &page->flags);
135
+
if (!test_bit(PG_arch_1, &folio->flags))
136
+
set_bit(PG_arch_1, &folio->flags);
137
137
return;
138
138
139
139
} else {
140
-
141
-
unsigned long phys = page_to_phys(page);
142
-
unsigned long temp = page->index << PAGE_SHIFT;
140
+
unsigned long phys = folio_pfn(folio) * PAGE_SIZE;
141
+
unsigned long temp = folio_pos(folio);
142
+
unsigned int i, nr = folio_nr_pages(folio);
143
143
unsigned long alias = !(DCACHE_ALIAS_EQ(temp, phys));
144
144
unsigned long virt;
145
145
···
154
154
return;
155
155
156
156
preempt_disable();
157
-
virt = TLBTEMP_BASE_1 + (phys & DCACHE_ALIAS_MASK);
158
-
__flush_invalidate_dcache_page_alias(virt, phys);
159
-
160
-
virt = TLBTEMP_BASE_1 + (temp & DCACHE_ALIAS_MASK);
161
-
162
-
if (alias)
157
+
for (i = 0; i < nr; i++) {
158
+
virt = TLBTEMP_BASE_1 + (phys & DCACHE_ALIAS_MASK);
163
159
__flush_invalidate_dcache_page_alias(virt, phys);
164
160
165
-
if (mapping)
166
-
__invalidate_icache_page_alias(virt, phys);
161
+
virt = TLBTEMP_BASE_1 + (temp & DCACHE_ALIAS_MASK);
162
+
163
+
if (alias)
164
+
__flush_invalidate_dcache_page_alias(virt, phys);
165
+
166
+
if (mapping)
167
+
__invalidate_icache_page_alias(virt, phys);
168
+
phys += PAGE_SIZE;
169
+
temp += PAGE_SIZE;
170
+
}
167
171
preempt_enable();
168
172
}
169
173
170
174
/* There shouldn't be an entry in the cache for this page anymore. */
171
175
}
172
-
EXPORT_SYMBOL(flush_dcache_page);
176
+
EXPORT_SYMBOL(flush_dcache_folio);
173
177
174
178
/*
175
179
* For now, flush the whole cache. FIXME??
···
211
207
212
208
#endif /* DCACHE_WAY_SIZE > PAGE_SIZE */
213
209
214
-
void
215
-
update_mmu_cache(struct vm_area_struct * vma, unsigned long addr, pte_t *ptep)
210
+
void update_mmu_cache_range(struct vm_fault *vmf, struct vm_area_struct *vma,
211
+
unsigned long addr, pte_t *ptep, unsigned int nr)
216
212
{
217
213
unsigned long pfn = pte_pfn(*ptep);
218
-
struct page *page;
214
+
struct folio *folio;
215
+
unsigned int i;
219
216
220
217
if (!pfn_valid(pfn))
221
218
return;
222
219
223
-
page = pfn_to_page(pfn);
220
+
folio = page_folio(pfn_to_page(pfn));
224
221
225
-
/* Invalidate old entry in TLBs */
226
-
227
-
flush_tlb_page(vma, addr);
222
+
/* Invalidate old entries in TLBs */
223
+
for (i = 0; i < nr; i++)
224
+
flush_tlb_page(vma, addr + i * PAGE_SIZE);
225
+
nr = folio_nr_pages(folio);
228
226
229
227
#if (DCACHE_WAY_SIZE > PAGE_SIZE)
230
228
231
-
if (!PageReserved(page) && test_bit(PG_arch_1, &page->flags)) {
232
-
unsigned long phys = page_to_phys(page);
229
+
if (!folio_test_reserved(folio) && test_bit(PG_arch_1, &folio->flags)) {
230
+
unsigned long phys = folio_pfn(folio) * PAGE_SIZE;
233
231
unsigned long tmp;
234
232
235
233
preempt_disable();
236
-
tmp = TLBTEMP_BASE_1 + (phys & DCACHE_ALIAS_MASK);
237
-
__flush_invalidate_dcache_page_alias(tmp, phys);
238
-
tmp = TLBTEMP_BASE_1 + (addr & DCACHE_ALIAS_MASK);
239
-
__flush_invalidate_dcache_page_alias(tmp, phys);
240
-
__invalidate_icache_page_alias(tmp, phys);
234
+
for (i = 0; i < nr; i++) {
235
+
tmp = TLBTEMP_BASE_1 + (phys & DCACHE_ALIAS_MASK);
236
+
__flush_invalidate_dcache_page_alias(tmp, phys);
237
+
tmp = TLBTEMP_BASE_1 + (addr & DCACHE_ALIAS_MASK);
238
+
__flush_invalidate_dcache_page_alias(tmp, phys);
239
+
__invalidate_icache_page_alias(tmp, phys);
240
+
phys += PAGE_SIZE;
241
+
}
241
242
preempt_enable();
242
243
243
-
clear_bit(PG_arch_1, &page->flags);
244
+
clear_bit(PG_arch_1, &folio->flags);
244
245
}
245
246
#else
246
-
if (!PageReserved(page) && !test_bit(PG_arch_1, &page->flags)
247
+
if (!folio_test_reserved(folio) && !test_bit(PG_arch_1, &folio->flags)
247
248
&& (vma->vm_flags & VM_EXEC) != 0) {
248
-
unsigned long paddr = (unsigned long)kmap_atomic(page);
249
-
__flush_dcache_page(paddr);
250
-
__invalidate_icache_page(paddr);
251
-
set_bit(PG_arch_1, &page->flags);
252
-
kunmap_atomic((void *)paddr);
249
+
for (i = 0; i < nr; i++) {
250
+
void *paddr = kmap_local_folio(folio, i * PAGE_SIZE);
251
+
__flush_dcache_page((unsigned long)paddr);
252
+
__invalidate_icache_page((unsigned long)paddr);
253
+
kunmap_local(paddr);
254
+
}
255
+
set_bit(PG_arch_1, &folio->flags);
253
256
}
254
257
#endif
255
258
}