tjh.dev / kernel
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

mm: percpu: add generic pcpu_populate_pte() function

With NEED_PER_CPU_PAGE_FIRST_CHUNK enabled, we need a function to
populate pte, this patch adds a generic pcpu populate pte function,
pcpu_populate_pte(), which is marked __weak and used on most
architectures, but it is overridden on x86, which has its own
implementation.

Link: https://lkml.kernel.org/r/20211216112359.103822-5-wangkefeng.wang@huawei.com
Signed-off-by: Kefeng Wang <wangkefeng.wang@huawei.com>
Cc: Michael Ellerman <mpe@ellerman.id.au>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: "David S. Miller" <davem@davemloft.net>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Dave Hansen <dave.hansen@linux.intel.com>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Cc: "Rafael J. Wysocki" <rafael@kernel.org>
Cc: Dennis Zhou <dennis@kernel.org>
Cc: Tejun Heo <tj@kernel.org>
Cc: Christoph Lameter <cl@linux.com>
Cc: Albert Ou <aou@eecs.berkeley.edu>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Palmer Dabbelt <palmer@dabbelt.com>
Cc: Paul Walmsley <paul.walmsley@sifive.com>
Cc: Thomas Bogendoerfer <tsbogend@alpha.franken.de>
Cc: Will Deacon <will@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

authored by

Kefeng Wang and committed by
Linus Torvalds 20c03576 23f91716

+78 -162
+1 -46
arch/powerpc/kernel/setup_64.c
··· 787 787 unsigned long __per_cpu_offset[NR_CPUS] __read_mostly; 788 788 EXPORT_SYMBOL(__per_cpu_offset); 789 789 790 - static void __init pcpu_populate_pte(unsigned long addr) 791 - { 792 - pgd_t *pgd = pgd_offset_k(addr); 793 - p4d_t *p4d; 794 - pud_t *pud; 795 - pmd_t *pmd; 796 - 797 - p4d = p4d_offset(pgd, addr); 798 - if (p4d_none(*p4d)) { 799 - pud_t *new; 800 - 801 - new = memblock_alloc(PUD_TABLE_SIZE, PUD_TABLE_SIZE); 802 - if (!new) 803 - goto err_alloc; 804 - p4d_populate(&init_mm, p4d, new); 805 - } 806 - 807 - pud = pud_offset(p4d, addr); 808 - if (pud_none(*pud)) { 809 - pmd_t *new; 810 - 811 - new = memblock_alloc(PMD_TABLE_SIZE, PMD_TABLE_SIZE); 812 - if (!new) 813 - goto err_alloc; 814 - pud_populate(&init_mm, pud, new); 815 - } 816 - 817 - pmd = pmd_offset(pud, addr); 818 - if (!pmd_present(*pmd)) { 819 - pte_t *new; 820 - 821 - new = memblock_alloc(PTE_TABLE_SIZE, PTE_TABLE_SIZE); 822 - if (!new) 823 - goto err_alloc; 824 - pmd_populate_kernel(&init_mm, pmd, new); 825 - } 826 - 827 - return; 828 - 829 - err_alloc: 830 - panic("%s: Failed to allocate %lu bytes align=%lx from=%lx\n", 831 - __func__, PAGE_SIZE, PAGE_SIZE, PAGE_SIZE); 832 - } 833 - 834 - 835 790 void __init setup_per_cpu_areas(void) 836 791 { 837 792 const size_t dyn_size = PERCPU_MODULE_RESERVE + PERCPU_DYNAMIC_RESERVE; ··· 815 860 } 816 861 817 862 if (rc < 0) 818 - rc = pcpu_page_first_chunk(0, pcpu_cpu_to_node, pcpu_populate_pte); 863 + rc = pcpu_page_first_chunk(0, pcpu_cpu_to_node); 819 864 if (rc < 0) 820 865 panic("cannot initialize percpu area (err=%d)", rc); 821 866
+1 -55
arch/sparc/kernel/smp_64.c
··· 1539 1539 return cpu_to_node(cpu); 1540 1540 } 1541 1541 1542 - static void __init pcpu_populate_pte(unsigned long addr) 1543 - { 1544 - pgd_t *pgd = pgd_offset_k(addr); 1545 - p4d_t *p4d; 1546 - pud_t *pud; 1547 - pmd_t *pmd; 1548 - 1549 - if (pgd_none(*pgd)) { 1550 - pud_t *new; 1551 - 1552 - new = memblock_alloc_from(PAGE_SIZE, PAGE_SIZE, PAGE_SIZE); 1553 - if (!new) 1554 - goto err_alloc; 1555 - pgd_populate(&init_mm, pgd, new); 1556 - } 1557 - 1558 - p4d = p4d_offset(pgd, addr); 1559 - if (p4d_none(*p4d)) { 1560 - pud_t *new; 1561 - 1562 - new = memblock_alloc_from(PAGE_SIZE, PAGE_SIZE, PAGE_SIZE); 1563 - if (!new) 1564 - goto err_alloc; 1565 - p4d_populate(&init_mm, p4d, new); 1566 - } 1567 - 1568 - pud = pud_offset(p4d, addr); 1569 - if (pud_none(*pud)) { 1570 - pmd_t *new; 1571 - 1572 - new = memblock_alloc_from(PAGE_SIZE, PAGE_SIZE, PAGE_SIZE); 1573 - if (!new) 1574 - goto err_alloc; 1575 - pud_populate(&init_mm, pud, new); 1576 - } 1577 - 1578 - pmd = pmd_offset(pud, addr); 1579 - if (!pmd_present(*pmd)) { 1580 - pte_t *new; 1581 - 1582 - new = memblock_alloc_from(PAGE_SIZE, PAGE_SIZE, PAGE_SIZE); 1583 - if (!new) 1584 - goto err_alloc; 1585 - pmd_populate_kernel(&init_mm, pmd, new); 1586 - } 1587 - 1588 - return; 1589 - 1590 - err_alloc: 1591 - panic("%s: Failed to allocate %lu bytes align=%lx from=%lx\n", 1592 - __func__, PAGE_SIZE, PAGE_SIZE, PAGE_SIZE); 1593 - } 1594 - 1595 1542 void __init setup_per_cpu_areas(void) 1596 1543 { 1597 1544 unsigned long delta; ··· 1557 1610 } 1558 1611 if (rc < 0) 1559 1612 rc = pcpu_page_first_chunk(PERCPU_MODULE_RESERVE, 1560 - pcpu_cpu_to_node, 1561 - pcpu_populate_pte); 1613 + pcpu_cpu_to_node); 1562 1614 if (rc < 0) 1563 1615 panic("cannot initialize percpu area (err=%d)", rc); 1564 1616
+2 -3
arch/x86/kernel/setup_percpu.c
··· 101 101 return early_cpu_to_node(cpu); 102 102 } 103 103 104 - static void __init pcpup_populate_pte(unsigned long addr) 104 + void __init pcpu_populate_pte(unsigned long addr) 105 105 { 106 106 populate_extra_pte(addr); 107 107 } ··· 163 163 } 164 164 if (rc < 0) 165 165 rc = pcpu_page_first_chunk(PERCPU_FIRST_CHUNK_RESERVE, 166 - pcpu_cpu_to_node, 167 - pcpup_populate_pte); 166 + pcpu_cpu_to_node); 168 167 if (rc < 0) 169 168 panic("cannot initialize percpu area (err=%d)", rc); 170 169
+1 -50
drivers/base/arch_numa.c
··· 14 14 #include <linux/of.h> 15 15 16 16 #include <asm/sections.h> 17 - #include <asm/pgalloc.h> 18 17 19 18 struct pglist_data *node_data[MAX_NUMNODES] __read_mostly; 20 19 EXPORT_SYMBOL(node_data); ··· 154 155 return node_distance(early_cpu_to_node(from), early_cpu_to_node(to)); 155 156 } 156 157 157 - #ifdef CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK 158 - static void __init pcpu_populate_pte(unsigned long addr) 159 - { 160 - pgd_t *pgd = pgd_offset_k(addr); 161 - p4d_t *p4d; 162 - pud_t *pud; 163 - pmd_t *pmd; 164 - 165 - p4d = p4d_offset(pgd, addr); 166 - if (p4d_none(*p4d)) { 167 - pud_t *new; 168 - 169 - new = memblock_alloc(PAGE_SIZE, PAGE_SIZE); 170 - if (!new) 171 - goto err_alloc; 172 - p4d_populate(&init_mm, p4d, new); 173 - } 174 - 175 - pud = pud_offset(p4d, addr); 176 - if (pud_none(*pud)) { 177 - pmd_t *new; 178 - 179 - new = memblock_alloc(PAGE_SIZE, PAGE_SIZE); 180 - if (!new) 181 - goto err_alloc; 182 - pud_populate(&init_mm, pud, new); 183 - } 184 - 185 - pmd = pmd_offset(pud, addr); 186 - if (!pmd_present(*pmd)) { 187 - pte_t *new; 188 - 189 - new = memblock_alloc(PAGE_SIZE, PAGE_SIZE); 190 - if (!new) 191 - goto err_alloc; 192 - pmd_populate_kernel(&init_mm, pmd, new); 193 - } 194 - 195 - return; 196 - 197 - err_alloc: 198 - panic("%s: Failed to allocate %lu bytes align=%lx from=%lx\n", 199 - __func__, PAGE_SIZE, PAGE_SIZE, PAGE_SIZE); 200 - } 201 - #endif 202 - 203 158 void __init setup_per_cpu_areas(void) 204 159 { 205 160 unsigned long delta; ··· 178 225 179 226 #ifdef CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK 180 227 if (rc < 0) 181 - rc = pcpu_page_first_chunk(PERCPU_MODULE_RESERVE, 182 - early_cpu_to_node, 183 - pcpu_populate_pte); 228 + rc = pcpu_page_first_chunk(PERCPU_MODULE_RESERVE, early_cpu_to_node); 184 229 #endif 185 230 if (rc < 0) 186 231 panic("Failed to initialize percpu areas (err=%d).", rc);
+2 -3
include/linux/percpu.h
··· 95 95 extern enum pcpu_fc pcpu_chosen_fc; 96 96 97 97 typedef int (pcpu_fc_cpu_to_node_fn_t)(int cpu); 98 - typedef void (*pcpu_fc_populate_pte_fn_t)(unsigned long addr); 99 98 typedef int (pcpu_fc_cpu_distance_fn_t)(unsigned int from, unsigned int to); 100 99 101 100 extern struct pcpu_alloc_info * __init pcpu_alloc_alloc_info(int nr_groups, ··· 112 113 #endif 113 114 114 115 #ifdef CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK 116 + void __init pcpu_populate_pte(unsigned long addr); 115 117 extern int __init pcpu_page_first_chunk(size_t reserved_size, 116 - pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn, 117 - pcpu_fc_populate_pte_fn_t populate_pte_fn); 118 + pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn); 118 119 #endif 119 120 120 121 extern void __percpu *__alloc_reserved_percpu(size_t size, size_t align) __alloc_size(1);
+71 -5
mm/percpu.c
··· 3174 3174 #endif /* BUILD_EMBED_FIRST_CHUNK */ 3175 3175 3176 3176 #ifdef BUILD_PAGE_FIRST_CHUNK 3177 + #include <asm/pgalloc.h> 3178 + 3179 + #ifndef P4D_TABLE_SIZE 3180 + #define P4D_TABLE_SIZE PAGE_SIZE 3181 + #endif 3182 + 3183 + #ifndef PUD_TABLE_SIZE 3184 + #define PUD_TABLE_SIZE PAGE_SIZE 3185 + #endif 3186 + 3187 + #ifndef PMD_TABLE_SIZE 3188 + #define PMD_TABLE_SIZE PAGE_SIZE 3189 + #endif 3190 + 3191 + #ifndef PTE_TABLE_SIZE 3192 + #define PTE_TABLE_SIZE PAGE_SIZE 3193 + #endif 3194 + void __init __weak pcpu_populate_pte(unsigned long addr) 3195 + { 3196 + pgd_t *pgd = pgd_offset_k(addr); 3197 + p4d_t *p4d; 3198 + pud_t *pud; 3199 + pmd_t *pmd; 3200 + 3201 + if (pgd_none(*pgd)) { 3202 + p4d_t *new; 3203 + 3204 + new = memblock_alloc(P4D_TABLE_SIZE, P4D_TABLE_SIZE); 3205 + if (!new) 3206 + goto err_alloc; 3207 + pgd_populate(&init_mm, pgd, new); 3208 + } 3209 + 3210 + p4d = p4d_offset(pgd, addr); 3211 + if (p4d_none(*p4d)) { 3212 + pud_t *new; 3213 + 3214 + new = memblock_alloc(PUD_TABLE_SIZE, PUD_TABLE_SIZE); 3215 + if (!new) 3216 + goto err_alloc; 3217 + p4d_populate(&init_mm, p4d, new); 3218 + } 3219 + 3220 + pud = pud_offset(p4d, addr); 3221 + if (pud_none(*pud)) { 3222 + pmd_t *new; 3223 + 3224 + new = memblock_alloc(PMD_TABLE_SIZE, PMD_TABLE_SIZE); 3225 + if (!new) 3226 + goto err_alloc; 3227 + pud_populate(&init_mm, pud, new); 3228 + } 3229 + 3230 + pmd = pmd_offset(pud, addr); 3231 + if (!pmd_present(*pmd)) { 3232 + pte_t *new; 3233 + 3234 + new = memblock_alloc(PTE_TABLE_SIZE, PTE_TABLE_SIZE); 3235 + if (!new) 3236 + goto err_alloc; 3237 + pmd_populate_kernel(&init_mm, pmd, new); 3238 + } 3239 + 3240 + return; 3241 + 3242 + err_alloc: 3243 + panic("%s: Failed to allocate memory\n", __func__); 3244 + } 3245 + 3177 3246 /** 3178 3247 * pcpu_page_first_chunk - map the first chunk using PAGE_SIZE pages 3179 3248 * @reserved_size: the size of reserved percpu area in bytes 3180 3249 * @cpu_to_nd_fn: callback to convert cpu to it's node, optional 3181 - * @populate_pte_fn: function to populate pte 3182 3250 * 3183 3251 * This is a helper to ease setting up page-remapped first percpu 3184 3252 * chunk and can be called where pcpu_setup_first_chunk() is expected. ··· 3257 3189 * RETURNS: 3258 3190 * 0 on success, -errno on failure. 3259 3191 */ 3260 - int __init pcpu_page_first_chunk(size_t reserved_size, 3261 - pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn, 3262 - pcpu_fc_populate_pte_fn_t populate_pte_fn) 3192 + int __init pcpu_page_first_chunk(size_t reserved_size, pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn) 3263 3193 { 3264 3194 static struct vm_struct vm; 3265 3195 struct pcpu_alloc_info *ai; ··· 3321 3255 (unsigned long)vm.addr + unit * ai->unit_size; 3322 3256 3323 3257 for (i = 0; i < unit_pages; i++) 3324 - populate_pte_fn(unit_addr + (i << PAGE_SHIFT)); 3258 + pcpu_populate_pte(unit_addr + (i << PAGE_SHIFT)); 3325 3259 3326 3260 /* pte already populated, the following shouldn't fail */ 3327 3261 rc = __pcpu_map_pages(unit_addr, &pages[unit * unit_pages],