Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
arm64: mm: Wrap flush_tlb_page() around __do_flush_tlb_range()
Flushing a page from the tlb is just a special case of flushing a range.
So let's rework flush_tlb_page() so that it simply wraps
__do_flush_tlb_range(). While at it, let's also update the API to take
the same flags that we use when flushing a range. This allows us to
delete all the ugly "_nosync", "_local" and "_nonotify" variants.
Thanks to constant folding, all of the complex looping and tlbi-by-range
options get eliminated so that the generated code for flush_tlb_page()
looks very similar to the previous version.
Reviewed-by: Linu Cherian <linu.cherian@arm.com>
Signed-off-by: Ryan Roberts <ryan.roberts@arm.com>
Reviewed-by: Jonathan Cameron <jonathan.cameron@huawei.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
authored by
Ryan Roberts
and committed by
Catalin Marinas
15397e3c
0477fc56
+29
-60
+3
-3
arch/arm64/include/asm/pgtable.h
+3
-3
arch/arm64/include/asm/pgtable.h
···
101
101
* entries exist.
102
102
*/
103
103
#define flush_tlb_fix_spurious_fault(vma, address, ptep) \
104
-
local_flush_tlb_page_nonotify(vma, address)
104
+
__flush_tlb_page(vma, address, TLBF_NOBROADCAST | TLBF_NONOTIFY)
105
105
106
106
#define flush_tlb_fix_spurious_fault_pmd(vma, address, pmdp) \
107
-
local_flush_tlb_page_nonotify(vma, address)
107
+
__flush_tlb_page(vma, address, TLBF_NOBROADCAST | TLBF_NONOTIFY)
108
108
109
109
/*
110
110
* ZERO_PAGE is a global shared page that is always zero: used
···
1320
1320
* context-switch, which provides a DSB to complete the TLB
1321
1321
* invalidation.
1322
1322
*/
1323
-
flush_tlb_page_nosync(vma, address);
1323
+
__flush_tlb_page(vma, address, TLBF_NOSYNC);
1324
1324
}
1325
1325
1326
1326
return young;
+25
-56
arch/arm64/include/asm/tlbflush.h
+25
-56
arch/arm64/include/asm/tlbflush.h
···
269
269
* unmapping pages from vmalloc/io space.
270
270
*
271
271
* flush_tlb_page(vma, addr)
272
-
* Invalidate a single user mapping for address 'addr' in the
273
-
* address space corresponding to 'vma->mm'. Note that this
274
-
* operation only invalidates a single, last-level page-table
275
-
* entry and therefore does not affect any walk-caches.
272
+
* Equivalent to __flush_tlb_page(..., flags=TLBF_NONE)
276
273
*
277
274
*
278
275
* Next, we have some undocumented invalidation routines that you probably
···
297
300
* TLBF_NOSYNC (don't issue trailing dsb) and TLBF_NOBROADCAST
298
301
* (only perform the invalidation for the local cpu).
299
302
*
300
-
* local_flush_tlb_page(vma, addr)
301
-
* Local variant of flush_tlb_page(). Stale TLB entries may
302
-
* remain in remote CPUs.
303
-
*
304
-
* local_flush_tlb_page_nonotify(vma, addr)
305
-
* Same as local_flush_tlb_page() except MMU notifier will not be
306
-
* called.
303
+
* __flush_tlb_page(vma, addr, flags)
304
+
* Invalidate a single user mapping for address 'addr' in the
305
+
* address space corresponding to 'vma->mm'. Note that this
306
+
* operation only invalidates a single, last-level page-table entry
307
+
* and therefore does not affect any walk-caches. flags may contain
308
+
* any combination of TLBF_NONOTIFY (don't call mmu notifiers),
309
+
* TLBF_NOSYNC (don't issue trailing dsb) and TLBF_NOBROADCAST
310
+
* (only perform the invalidation for the local cpu).
307
311
*
308
312
* Finally, take a look at asm/tlb.h to see how tlb_flush() is implemented
309
313
* on top of these routines, since that is our interface to the mmu_gather
···
336
338
__tlbi_user(aside1is, asid);
337
339
__tlbi_sync_s1ish();
338
340
mmu_notifier_arch_invalidate_secondary_tlbs(mm, 0, -1UL);
339
-
}
340
-
341
-
static inline void __local_flush_tlb_page_nonotify_nosync(struct mm_struct *mm,
342
-
unsigned long uaddr)
343
-
{
344
-
dsb(nshst);
345
-
__tlbi_level_asid(vale1, uaddr, TLBI_TTL_UNKNOWN, ASID(mm));
346
-
}
347
-
348
-
static inline void local_flush_tlb_page_nonotify(struct vm_area_struct *vma,
349
-
unsigned long uaddr)
350
-
{
351
-
__local_flush_tlb_page_nonotify_nosync(vma->vm_mm, uaddr);
352
-
dsb(nsh);
353
-
}
354
-
355
-
static inline void local_flush_tlb_page(struct vm_area_struct *vma,
356
-
unsigned long uaddr)
357
-
{
358
-
__local_flush_tlb_page_nonotify_nosync(vma->vm_mm, uaddr);
359
-
mmu_notifier_arch_invalidate_secondary_tlbs(vma->vm_mm, uaddr & PAGE_MASK,
360
-
(uaddr & PAGE_MASK) + PAGE_SIZE);
361
-
dsb(nsh);
362
-
}
363
-
364
-
static inline void __flush_tlb_page_nosync(struct mm_struct *mm,
365
-
unsigned long uaddr)
366
-
{
367
-
dsb(ishst);
368
-
__tlbi_level_asid(vale1is, uaddr, TLBI_TTL_UNKNOWN, ASID(mm));
369
-
mmu_notifier_arch_invalidate_secondary_tlbs(mm, uaddr & PAGE_MASK,
370
-
(uaddr & PAGE_MASK) + PAGE_SIZE);
371
-
}
372
-
373
-
static inline void flush_tlb_page_nosync(struct vm_area_struct *vma,
374
-
unsigned long uaddr)
375
-
{
376
-
return __flush_tlb_page_nosync(vma->vm_mm, uaddr);
377
-
}
378
-
379
-
static inline void flush_tlb_page(struct vm_area_struct *vma,
380
-
unsigned long uaddr)
381
-
{
382
-
flush_tlb_page_nosync(vma, uaddr);
383
-
__tlbi_sync_s1ish();
384
341
}
385
342
386
343
static inline bool arch_tlbbatch_should_defer(struct mm_struct *mm)
···
583
630
* information here.
584
631
*/
585
632
__flush_tlb_range(vma, start, end, PAGE_SIZE, TLBI_TTL_UNKNOWN, TLBF_NONE);
633
+
}
634
+
635
+
static inline void __flush_tlb_page(struct vm_area_struct *vma,
636
+
unsigned long uaddr, tlbf_t flags)
637
+
{
638
+
unsigned long start = round_down(uaddr, PAGE_SIZE);
639
+
unsigned long end = start + PAGE_SIZE;
640
+
641
+
__do_flush_tlb_range(vma, start, end, PAGE_SIZE, TLBI_TTL_UNKNOWN,
642
+
TLBF_NOWALKCACHE | flags);
643
+
}
644
+
645
+
static inline void flush_tlb_page(struct vm_area_struct *vma,
646
+
unsigned long uaddr)
647
+
{
648
+
__flush_tlb_page(vma, uaddr, TLBF_NONE);
586
649
}
587
650
588
651
static inline void flush_tlb_kernel_range(unsigned long start, unsigned long end)