Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
arm64: mm: Refactor __flush_tlb_range() to take flags
We have function variants with "_nosync", "_local", "_nonotify" as well
as the "last_level" parameter. Let's generalize and simplify by using a
flags parameter to encode all these variants.
As a first step, convert the "last_level" boolean parameter to a flags
parameter and create the first flag, TLBF_NOWALKCACHE. When present,
walk cache entries are not evicted, which is the same as the old
last_level=true.
Reviewed-by: Linu Cherian <linu.cherian@arm.com>
Reviewed-by: Jonathan Cameron <jonathan.cameron@huawei.com>
Signed-off-by: Ryan Roberts <ryan.roberts@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
authored by
Ryan Roberts
and committed by
Catalin Marinas
11f6dd8d
64212d68
+37
-28
+6
-6
arch/arm64/include/asm/hugetlb.h
+6
-6
arch/arm64/include/asm/hugetlb.h
···
71
71
unsigned long start,
72
72
unsigned long end,
73
73
unsigned long stride,
74
-
bool last_level)
74
+
tlbf_t flags)
75
75
{
76
76
switch (stride) {
77
77
#ifndef __PAGETABLE_PMD_FOLDED
78
78
case PUD_SIZE:
79
-
__flush_tlb_range(vma, start, end, PUD_SIZE, last_level, 1);
79
+
__flush_tlb_range(vma, start, end, PUD_SIZE, 1, flags);
80
80
break;
81
81
#endif
82
82
case CONT_PMD_SIZE:
83
83
case PMD_SIZE:
84
-
__flush_tlb_range(vma, start, end, PMD_SIZE, last_level, 2);
84
+
__flush_tlb_range(vma, start, end, PMD_SIZE, 2, flags);
85
85
break;
86
86
case CONT_PTE_SIZE:
87
-
__flush_tlb_range(vma, start, end, PAGE_SIZE, last_level, 3);
87
+
__flush_tlb_range(vma, start, end, PAGE_SIZE, 3, flags);
88
88
break;
89
89
default:
90
-
__flush_tlb_range(vma, start, end, PAGE_SIZE, last_level, TLBI_TTL_UNKNOWN);
90
+
__flush_tlb_range(vma, start, end, PAGE_SIZE, TLBI_TTL_UNKNOWN, flags);
91
91
}
92
92
}
93
93
···
98
98
{
99
99
unsigned long stride = huge_page_size(hstate_vma(vma));
100
100
101
-
__flush_hugetlb_tlb_range(vma, start, end, stride, false);
101
+
__flush_hugetlb_tlb_range(vma, start, end, stride, TLBF_NONE);
102
102
}
103
103
104
104
#endif /* __ASM_HUGETLB_H */
+2
-2
arch/arm64/include/asm/pgtable.h
+2
-2
arch/arm64/include/asm/pgtable.h
···
89
89
90
90
/* Set stride and tlb_level in flush_*_tlb_range */
91
91
#define flush_pmd_tlb_range(vma, addr, end) \
92
-
__flush_tlb_range(vma, addr, end, PMD_SIZE, false, 2)
92
+
__flush_tlb_range(vma, addr, end, PMD_SIZE, 2, TLBF_NONE)
93
93
#define flush_pud_tlb_range(vma, addr, end) \
94
-
__flush_tlb_range(vma, addr, end, PUD_SIZE, false, 1)
94
+
__flush_tlb_range(vma, addr, end, PUD_SIZE, 1, TLBF_NONE)
95
95
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
96
96
97
97
/*
+3
-3
arch/arm64/include/asm/tlb.h
+3
-3
arch/arm64/include/asm/tlb.h
···
53
53
static inline void tlb_flush(struct mmu_gather *tlb)
54
54
{
55
55
struct vm_area_struct vma = TLB_FLUSH_VMA(tlb->mm, 0);
56
-
bool last_level = !tlb->freed_tables;
56
+
tlbf_t flags = tlb->freed_tables ? TLBF_NONE : TLBF_NOWALKCACHE;
57
57
unsigned long stride = tlb_get_unmap_size(tlb);
58
58
int tlb_level = tlb_get_level(tlb);
59
59
···
63
63
* reallocate our ASID without invalidating the entire TLB.
64
64
*/
65
65
if (tlb->fullmm) {
66
-
if (!last_level)
66
+
if (tlb->freed_tables)
67
67
flush_tlb_mm(tlb->mm);
68
68
return;
69
69
}
70
70
71
71
__flush_tlb_range(&vma, tlb->start, tlb->end, stride,
72
-
last_level, tlb_level);
72
+
tlb_level, flags);
73
73
}
74
74
75
75
static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte,
+20
-12
arch/arm64/include/asm/tlbflush.h
+20
-12
arch/arm64/include/asm/tlbflush.h
···
286
286
* CPUs, ensuring that any walk-cache entries associated with the
287
287
* translation are also invalidated.
288
288
*
289
-
* __flush_tlb_range(vma, start, end, stride, last_level, tlb_level)
289
+
* __flush_tlb_range(vma, start, end, stride, tlb_level, flags)
290
290
* Invalidate the virtual-address range '[start, end)' on all
291
291
* CPUs for the user address space corresponding to 'vma->mm'.
292
292
* The invalidation operations are issued at a granularity
293
-
* determined by 'stride' and only affect any walk-cache entries
294
-
* if 'last_level' is equal to false. tlb_level is the level at
293
+
* determined by 'stride'. tlb_level is the level at
295
294
* which the invalidation must take place. If the level is wrong,
296
295
* no invalidation may take place. In the case where the level
297
296
* cannot be easily determined, the value TLBI_TTL_UNKNOWN will
298
-
* perform a non-hinted invalidation.
297
+
* perform a non-hinted invalidation. flags may be TLBF_NONE (0) or
298
+
* TLBF_NOWALKCACHE (elide eviction of walk cache entries).
299
299
*
300
300
* local_flush_tlb_page(vma, addr)
301
301
* Local variant of flush_tlb_page(). Stale TLB entries may
···
544
544
return pages >= (MAX_DVM_OPS * stride) >> PAGE_SHIFT;
545
545
}
546
546
547
+
typedef unsigned __bitwise tlbf_t;
548
+
549
+
/* No special behaviour. */
550
+
#define TLBF_NONE ((__force tlbf_t)0)
551
+
552
+
/* Invalidate tlb entries only, leaving the page table walk cache intact. */
553
+
#define TLBF_NOWALKCACHE ((__force tlbf_t)BIT(0))
554
+
547
555
static inline void __flush_tlb_range_nosync(struct mm_struct *mm,
548
556
unsigned long start, unsigned long end,
549
-
unsigned long stride, bool last_level,
550
-
int tlb_level)
557
+
unsigned long stride, int tlb_level,
558
+
tlbf_t flags)
551
559
{
552
560
unsigned long asid, pages;
553
561
···
571
563
dsb(ishst);
572
564
asid = ASID(mm);
573
565
574
-
if (last_level)
566
+
if (flags & TLBF_NOWALKCACHE)
575
567
__flush_s1_tlb_range_op(vale1is, start, pages, stride,
576
568
asid, tlb_level);
577
569
else
···
583
575
584
576
static inline void __flush_tlb_range(struct vm_area_struct *vma,
585
577
unsigned long start, unsigned long end,
586
-
unsigned long stride, bool last_level,
587
-
int tlb_level)
578
+
unsigned long stride, int tlb_level,
579
+
tlbf_t flags)
588
580
{
589
581
__flush_tlb_range_nosync(vma->vm_mm, start, end, stride,
590
-
last_level, tlb_level);
582
+
tlb_level, flags);
591
583
__tlbi_sync_s1ish();
592
584
}
593
585
···
615
607
* Set the tlb_level to TLBI_TTL_UNKNOWN because we can not get enough
616
608
* information here.
617
609
*/
618
-
__flush_tlb_range(vma, start, end, PAGE_SIZE, false, TLBI_TTL_UNKNOWN);
610
+
__flush_tlb_range(vma, start, end, PAGE_SIZE, TLBI_TTL_UNKNOWN, TLBF_NONE);
619
611
}
620
612
621
613
static inline void flush_tlb_kernel_range(unsigned long start, unsigned long end)
···
656
648
static inline void arch_tlbbatch_add_pending(struct arch_tlbflush_unmap_batch *batch,
657
649
struct mm_struct *mm, unsigned long start, unsigned long end)
658
650
{
659
-
__flush_tlb_range_nosync(mm, start, end, PAGE_SIZE, true, 3);
651
+
__flush_tlb_range_nosync(mm, start, end, PAGE_SIZE, 3, TLBF_NOWALKCACHE);
660
652
}
661
653
662
654
static inline bool __pte_flags_need_flush(ptdesc_t oldval, ptdesc_t newval)
+3
-2
arch/arm64/mm/contpte.c
+3
-2
arch/arm64/mm/contpte.c
···
225
225
*/
226
226
227
227
if (!system_supports_bbml2_noabort())
228
-
__flush_tlb_range(&vma, start_addr, addr, PAGE_SIZE, true, 3);
228
+
__flush_tlb_range(&vma, start_addr, addr, PAGE_SIZE, 3,
229
+
TLBF_NOWALKCACHE);
229
230
230
231
__set_ptes(mm, start_addr, start_ptep, pte, CONT_PTES);
231
232
}
···
553
552
* eliding the trailing DSB applies here.
554
553
*/
555
554
__flush_tlb_range_nosync(vma->vm_mm, addr, end,
556
-
PAGE_SIZE, true, 3);
555
+
PAGE_SIZE, 3, TLBF_NOWALKCACHE);
557
556
}
558
557
559
558
return young;
+2
-2
arch/arm64/mm/hugetlbpage.c
+2
-2
arch/arm64/mm/hugetlbpage.c
···
181
181
struct vm_area_struct vma = TLB_FLUSH_VMA(mm, 0);
182
182
unsigned long end = addr + (pgsize * ncontig);
183
183
184
-
__flush_hugetlb_tlb_range(&vma, addr, end, pgsize, true);
184
+
__flush_hugetlb_tlb_range(&vma, addr, end, pgsize, TLBF_NOWALKCACHE);
185
185
return orig_pte;
186
186
}
187
187
···
209
209
if (mm == &init_mm)
210
210
flush_tlb_kernel_range(saddr, addr);
211
211
else
212
-
__flush_hugetlb_tlb_range(&vma, saddr, addr, pgsize, true);
212
+
__flush_hugetlb_tlb_range(&vma, saddr, addr, pgsize, TLBF_NOWALKCACHE);
213
213
}
214
214
215
215
void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,