Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
arm64: mm: Re-implement the __flush_tlb_range_op macro in C
The __flush_tlb_range_op() macro is horrible and has been a previous
source of bugs thanks to multiple expansions of its arguments (see
commit f7edb07ad7c6 ("Fix mmu notifiers for range-based invalidates")).
Rewrite the thing in C.
Suggested-by: Linus Torvalds <torvalds@linux-foundation.org>
Co-developed-by: Will Deacon <will@kernel.org>
Signed-off-by: Will Deacon <will@kernel.org>
Reviewed-by: Jonathan Cameron <jonathan.cameron@huawei.com>
Signed-off-by: Ryan Roberts <ryan.roberts@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
authored by
Ryan Roberts
and committed by
Catalin Marinas
5e63b73f
d4b048ca
+46
-38
+46
-38
arch/arm64/include/asm/tlbflush.h
+46
-38
arch/arm64/include/asm/tlbflush.h
···
429
429
/*
430
430
* __flush_tlb_range_op - Perform TLBI operation upon a range
431
431
*
432
-
* @op: TLBI instruction that operates on a range (has 'r' prefix)
432
+
* @lop: TLBI level operation to perform
433
+
* @rop: TLBI range operation to perform
433
434
* @start: The start address of the range
434
435
* @pages: Range as the number of pages from 'start'
435
436
* @stride: Flush granularity
436
437
* @asid: The ASID of the task (0 for IPA instructions)
437
-
* @tlb_level: Translation Table level hint, if known
438
+
* @level: Translation Table level hint, if known
438
439
* @lpa2: If 'true', the lpa2 scheme is used as set out below
439
440
*
440
441
* When the CPU does not support TLB range operations, flush the TLB
···
502
501
op(arg);
503
502
}
504
503
505
-
#define __flush_tlb_range_op(op, start, pages, stride, \
506
-
asid, tlb_level, lpa2) \
507
-
do { \
508
-
typeof(start) __flush_start = start; \
509
-
typeof(pages) __flush_pages = pages; \
510
-
int num = 0; \
511
-
int scale = 3; \
512
-
\
513
-
while (__flush_pages > 0) { \
514
-
if (!system_supports_tlb_range() || \
515
-
__flush_pages == 1 || \
516
-
(lpa2 && __flush_start != ALIGN(__flush_start, SZ_64K))) { \
517
-
__tlbi_level_asid(op, __flush_start, tlb_level, asid); \
518
-
__flush_start += stride; \
519
-
__flush_pages -= stride >> PAGE_SHIFT; \
520
-
continue; \
521
-
} \
522
-
\
523
-
num = __TLBI_RANGE_NUM(__flush_pages, scale); \
524
-
if (num >= 0) { \
525
-
__tlbi_range(r##op, __flush_start, asid, scale, num, tlb_level, lpa2); \
526
-
__flush_start += __TLBI_RANGE_PAGES(num, scale) << PAGE_SHIFT; \
527
-
__flush_pages -= __TLBI_RANGE_PAGES(num, scale);\
528
-
} \
529
-
scale--; \
530
-
} \
531
-
} while (0)
504
+
static __always_inline void __flush_tlb_range_op(tlbi_op lop, tlbi_op rop,
505
+
u64 start, size_t pages,
506
+
u64 stride, u16 asid,
507
+
u32 level, bool lpa2)
508
+
{
509
+
u64 addr = start, end = start + pages * PAGE_SIZE;
510
+
int scale = 3;
511
+
512
+
while (addr != end) {
513
+
int num;
514
+
515
+
pages = (end - addr) >> PAGE_SHIFT;
516
+
517
+
if (!system_supports_tlb_range() || pages == 1)
518
+
goto invalidate_one;
519
+
520
+
if (lpa2 && !IS_ALIGNED(addr, SZ_64K))
521
+
goto invalidate_one;
522
+
523
+
num = __TLBI_RANGE_NUM(pages, scale);
524
+
if (num >= 0) {
525
+
__tlbi_range(rop, addr, asid, scale, num, level, lpa2);
526
+
addr += __TLBI_RANGE_PAGES(num, scale) << PAGE_SHIFT;
527
+
}
528
+
529
+
scale--;
530
+
continue;
531
+
invalidate_one:
532
+
__tlbi_level_asid(lop, addr, level, asid);
533
+
addr += stride;
534
+
}
535
+
}
536
+
537
+
#define __flush_s1_tlb_range_op(op, start, pages, stride, asid, tlb_level) \
538
+
__flush_tlb_range_op(op, r##op, start, pages, stride, asid, tlb_level, lpa2_is_enabled())
532
539
533
540
#define __flush_s2_tlb_range_op(op, start, pages, stride, tlb_level) \
534
-
__flush_tlb_range_op(op, start, pages, stride, 0, tlb_level, kvm_lpa2_is_enabled());
541
+
__flush_tlb_range_op(op, r##op, start, pages, stride, 0, tlb_level, kvm_lpa2_is_enabled())
535
542
536
543
static inline bool __flush_tlb_range_limit_excess(unsigned long start,
537
544
unsigned long end, unsigned long pages, unsigned long stride)
···
578
569
asid = ASID(mm);
579
570
580
571
if (last_level)
581
-
__flush_tlb_range_op(vale1is, start, pages, stride, asid,
582
-
tlb_level, lpa2_is_enabled());
572
+
__flush_s1_tlb_range_op(vale1is, start, pages, stride,
573
+
asid, tlb_level);
583
574
else
584
-
__flush_tlb_range_op(vae1is, start, pages, stride, asid,
585
-
tlb_level, lpa2_is_enabled());
575
+
__flush_s1_tlb_range_op(vae1is, start, pages, stride,
576
+
asid, tlb_level);
586
577
587
578
mmu_notifier_arch_invalidate_secondary_tlbs(mm, start, end);
588
579
}
···
606
597
607
598
dsb(nshst);
608
599
asid = ASID(vma->vm_mm);
609
-
__flush_tlb_range_op(vale1, addr, CONT_PTES, PAGE_SIZE, asid,
610
-
3, lpa2_is_enabled());
600
+
__flush_s1_tlb_range_op(vale1, addr, CONT_PTES, PAGE_SIZE, asid, 3);
611
601
mmu_notifier_arch_invalidate_secondary_tlbs(vma->vm_mm, addr,
612
602
addr + CONT_PTE_SIZE);
613
603
dsb(nsh);
···
639
631
}
640
632
641
633
dsb(ishst);
642
-
__flush_tlb_range_op(vaale1is, start, pages, stride, 0,
643
-
TLBI_TTL_UNKNOWN, lpa2_is_enabled());
634
+
__flush_s1_tlb_range_op(vaale1is, start, pages, stride, 0,
635
+
TLBI_TTL_UNKNOWN);
644
636
__tlbi_sync_s1ish();
645
637
isb();
646
638
}