mm: percpu: add pcpu_fc_cpu_to_node_fn_t typedef

+9 -3

arch/mips/mm/init.c

··· 519 519 return node_distance(cpu_to_node(from), cpu_to_node(to)); 520 520 } 521 521 522 - static void * __init pcpu_fc_alloc(unsigned int cpu, size_t size, 523 - size_t align) 522 + static int __init pcpu_cpu_to_node(int cpu) 523 + { 524 + return cpu_to_node(cpu); 525 + } 526 + 527 + static void * __init pcpu_fc_alloc(unsigned int cpu, size_t size, size_t align, 528 + pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn) 524 529 { 525 530 return memblock_alloc_try_nid(size, align, __pa(MAX_DMA_ADDRESS), 526 531 MEMBLOCK_ALLOC_ACCESSIBLE, 527 - cpu_to_node(cpu)); 532 + cpu_to_nd_fn(cpu)); 528 533 } 529 534 530 535 static void __init pcpu_fc_free(void *ptr, size_t size) ··· 550 545 rc = pcpu_embed_first_chunk(PERCPU_MODULE_RESERVE, 551 546 PERCPU_DYNAMIC_RESERVE, PAGE_SIZE, 552 547 pcpu_cpu_distance, 548 + pcpu_cpu_to_node, 553 549 pcpu_fc_alloc, pcpu_fc_free); 554 550 if (rc < 0) 555 551 panic("Failed to initialize percpu areas.");

+11 -4

arch/powerpc/kernel/setup_64.c

··· 784 784 * RETURNS: 785 785 * Pointer to the allocated area on success, NULL on failure. 786 786 */ 787 - static void * __init pcpu_alloc_bootmem(unsigned int cpu, size_t size, 788 - size_t align) 787 + static void * __init pcpu_alloc_bootmem(unsigned int cpu, size_t size, size_t align, 788 + pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn) 789 789 { 790 790 const unsigned long goal = __pa(MAX_DMA_ADDRESS); 791 791 #ifdef CONFIG_NUMA 792 - int node = early_cpu_to_node(cpu); 792 + int node = cpu_to_nd_fun(cpu); 793 793 void *ptr; 794 794 795 795 if (!node_online(node) || !NODE_DATA(node)) { ··· 821 821 return LOCAL_DISTANCE; 822 822 else 823 823 return REMOTE_DISTANCE; 824 + } 825 + 826 + static __init int pcpu_cpu_to_node(int cpu) 827 + { 828 + return early_cpu_to_node(cpu); 824 829 } 825 830 826 831 unsigned long __per_cpu_offset[NR_CPUS] __read_mostly; ··· 896 891 897 892 if (pcpu_chosen_fc != PCPU_FC_PAGE) { 898 893 rc = pcpu_embed_first_chunk(0, dyn_size, atom_size, pcpu_cpu_distance, 894 + pcpu_cpu_to_node, 899 895 pcpu_alloc_bootmem, pcpu_free_bootmem); 900 896 if (rc) 901 897 pr_warn("PERCPU: %s allocator failed (%d), " ··· 905 899 } 906 900 907 901 if (rc < 0) 908 - rc = pcpu_page_first_chunk(0, pcpu_alloc_bootmem, pcpu_free_bootmem, 902 + rc = pcpu_page_first_chunk(0, pcpu_cpu_to_node, 903 + pcpu_alloc_bootmem, pcpu_free_bootmem, 909 904 pcpu_populate_pte); 910 905 if (rc < 0) 911 906 panic("cannot initialize percpu area (err=%d)", rc);

+10 -3

arch/sparc/kernel/smp_64.c

··· 1539 1539 * RETURNS: 1540 1540 * Pointer to the allocated area on success, NULL on failure. 1541 1541 */ 1542 - static void * __init pcpu_alloc_bootmem(unsigned int cpu, size_t size, 1543 - size_t align) 1542 + static void * __init pcpu_alloc_bootmem(unsigned int cpu, size_t size, size_t align, 1543 + pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn) 1544 1544 { 1545 1545 const unsigned long goal = __pa(MAX_DMA_ADDRESS); 1546 1546 #ifdef CONFIG_NUMA 1547 - int node = cpu_to_node(cpu); 1547 + int node = cpu_to_nd_fn(cpu); 1548 1548 void *ptr; 1549 1549 1550 1550 if (!node_online(node) || !NODE_DATA(node)) { ··· 1576 1576 return LOCAL_DISTANCE; 1577 1577 else 1578 1578 return REMOTE_DISTANCE; 1579 + } 1580 + 1581 + static int __init pcpu_cpu_to_node(int cpu) 1582 + { 1583 + return cpu_to_node(cpu); 1579 1584 } 1580 1585 1581 1586 static void __init pcpu_populate_pte(unsigned long addr) ··· 1646 1641 rc = pcpu_embed_first_chunk(PERCPU_MODULE_RESERVE, 1647 1642 PERCPU_DYNAMIC_RESERVE, 4 << 20, 1648 1643 pcpu_cpu_distance, 1644 + pcpu_cpu_to_node, 1649 1645 pcpu_alloc_bootmem, 1650 1646 pcpu_free_bootmem); 1651 1647 if (rc) ··· 1656 1650 } 1657 1651 if (rc < 0) 1658 1652 rc = pcpu_page_first_chunk(PERCPU_MODULE_RESERVE, 1653 + pcpu_cpu_to_node, 1659 1654 pcpu_alloc_bootmem, 1660 1655 pcpu_free_bootmem, 1661 1656 pcpu_populate_pte);

+13 -5

arch/x86/kernel/setup_percpu.c

··· 97 97 * RETURNS: 98 98 * Pointer to the allocated area on success, NULL on failure. 99 99 */ 100 - static void * __init pcpu_alloc_bootmem(unsigned int cpu, unsigned long size, 101 - unsigned long align) 100 + static void * __init pcpu_alloc_bootmem(unsigned int cpu, unsigned long size, unsigned long align, 101 + pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn) 102 102 { 103 103 const unsigned long goal = __pa(MAX_DMA_ADDRESS); 104 104 #ifdef CONFIG_NUMA 105 - int node = early_cpu_to_node(cpu); 105 + int node = cpu_to_nd_fn(cpu); 106 106 void *ptr; 107 107 108 108 if (!node_online(node) || !NODE_DATA(node)) { ··· 128 128 /* 129 129 * Helpers for first chunk memory allocation 130 130 */ 131 - static void * __init pcpu_fc_alloc(unsigned int cpu, size_t size, size_t align) 131 + static void * __init pcpu_fc_alloc(unsigned int cpu, size_t size, size_t align, 132 + pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn) 132 133 { 133 - return pcpu_alloc_bootmem(cpu, size, align); 134 + return pcpu_alloc_bootmem(cpu, size, align, cpu_to_nd_fn); 134 135 } 135 136 136 137 static void __init pcpu_fc_free(void *ptr, size_t size) ··· 149 148 #else 150 149 return LOCAL_DISTANCE; 151 150 #endif 151 + } 152 + 153 + static int __init pcpu_cpu_to_node(int cpu) 154 + { 155 + return early_cpu_to_node(cpu); 152 156 } 153 157 154 158 static void __init pcpup_populate_pte(unsigned long addr) ··· 211 205 rc = pcpu_embed_first_chunk(PERCPU_FIRST_CHUNK_RESERVE, 212 206 dyn_size, atom_size, 213 207 pcpu_cpu_distance, 208 + pcpu_cpu_to_node, 214 209 pcpu_fc_alloc, pcpu_fc_free); 215 210 if (rc < 0) 216 211 pr_warn("%s allocator failed (%d), falling back to page size\n", ··· 219 212 } 220 213 if (rc < 0) 221 214 rc = pcpu_page_first_chunk(PERCPU_FIRST_CHUNK_RESERVE, 215 + pcpu_cpu_to_node, 222 216 pcpu_fc_alloc, pcpu_fc_free, 223 217 pcpup_populate_pte); 224 218 if (rc < 0)

+5 -3

drivers/base/arch_numa.c

··· 155 155 return node_distance(early_cpu_to_node(from), early_cpu_to_node(to)); 156 156 } 157 157 158 - static void * __init pcpu_fc_alloc(unsigned int cpu, size_t size, 159 - size_t align) 158 + static void * __init pcpu_fc_alloc(unsigned int cpu, size_t size, size_t align, 159 + pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn) 160 160 { 161 - int nid = early_cpu_to_node(cpu); 161 + int nid = cpu_to_nd_fn(cpu); 162 162 163 163 return memblock_alloc_try_nid(size, align, 164 164 __pa(MAX_DMA_ADDRESS), MEMBLOCK_ALLOC_ACCESSIBLE, nid); ··· 229 229 rc = pcpu_embed_first_chunk(PERCPU_MODULE_RESERVE, 230 230 PERCPU_DYNAMIC_RESERVE, PAGE_SIZE, 231 231 pcpu_cpu_distance, 232 + early_cpu_to_node, 232 233 pcpu_fc_alloc, pcpu_fc_free); 233 234 #ifdef CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK 234 235 if (rc < 0) ··· 241 240 #ifdef CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK 242 241 if (rc < 0) 243 242 rc = pcpu_page_first_chunk(PERCPU_MODULE_RESERVE, 243 + early_cpu_to_node, 244 244 pcpu_fc_alloc, 245 245 pcpu_fc_free, 246 246 pcpu_populate_pte);

+5 -2

include/linux/percpu.h

··· 94 94 95 95 extern enum pcpu_fc pcpu_chosen_fc; 96 96 97 - typedef void * (*pcpu_fc_alloc_fn_t)(unsigned int cpu, size_t size, 98 - size_t align); 97 + typedef int (pcpu_fc_cpu_to_node_fn_t)(int cpu); 98 + typedef void * (*pcpu_fc_alloc_fn_t)(unsigned int cpu, size_t size, size_t align, 99 + pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn); 99 100 typedef void (*pcpu_fc_free_fn_t)(void *ptr, size_t size); 100 101 typedef void (*pcpu_fc_populate_pte_fn_t)(unsigned long addr); 101 102 typedef int (pcpu_fc_cpu_distance_fn_t)(unsigned int from, unsigned int to); ··· 112 111 extern int __init pcpu_embed_first_chunk(size_t reserved_size, size_t dyn_size, 113 112 size_t atom_size, 114 113 pcpu_fc_cpu_distance_fn_t cpu_distance_fn, 114 + pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn, 115 115 pcpu_fc_alloc_fn_t alloc_fn, 116 116 pcpu_fc_free_fn_t free_fn); 117 117 #endif 118 118 119 119 #ifdef CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK 120 120 extern int __init pcpu_page_first_chunk(size_t reserved_size, 121 + pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn, 121 122 pcpu_fc_alloc_fn_t alloc_fn, 122 123 pcpu_fc_free_fn_t free_fn, 123 124 pcpu_fc_populate_pte_fn_t populate_pte_fn);

+9 -5

mm/percpu.c

··· 3001 3001 * @dyn_size: minimum free size for dynamic allocation in bytes 3002 3002 * @atom_size: allocation atom size 3003 3003 * @cpu_distance_fn: callback to determine distance between cpus, optional 3004 + * @cpu_to_nd_fn: callback to convert cpu to it's node, optional 3004 3005 * @alloc_fn: function to allocate percpu page 3005 3006 * @free_fn: function to free percpu page 3006 3007 * ··· 3031 3030 int __init pcpu_embed_first_chunk(size_t reserved_size, size_t dyn_size, 3032 3031 size_t atom_size, 3033 3032 pcpu_fc_cpu_distance_fn_t cpu_distance_fn, 3033 + pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn, 3034 3034 pcpu_fc_alloc_fn_t alloc_fn, 3035 3035 pcpu_fc_free_fn_t free_fn) 3036 3036 { ··· 3068 3066 BUG_ON(cpu == NR_CPUS); 3069 3067 3070 3068 /* allocate space for the whole group */ 3071 - ptr = alloc_fn(cpu, gi->nr_units * ai->unit_size, atom_size); 3069 + ptr = alloc_fn(cpu, gi->nr_units * ai->unit_size, atom_size, cpu_to_nd_fn); 3072 3070 if (!ptr) { 3073 3071 rc = -ENOMEM; 3074 3072 goto out_free_areas; ··· 3145 3143 /** 3146 3144 * pcpu_page_first_chunk - map the first chunk using PAGE_SIZE pages 3147 3145 * @reserved_size: the size of reserved percpu area in bytes 3146 + * @cpu_to_nd_fn: callback to convert cpu to it's node, optional 3148 3147 * @alloc_fn: function to allocate percpu page, always called with PAGE_SIZE 3149 3148 * @free_fn: function to free percpu page, always called with PAGE_SIZE 3150 3149 * @populate_pte_fn: function to populate pte ··· 3160 3157 * 0 on success, -errno on failure. 3161 3158 */ 3162 3159 int __init pcpu_page_first_chunk(size_t reserved_size, 3160 + pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn, 3163 3161 pcpu_fc_alloc_fn_t alloc_fn, 3164 3162 pcpu_fc_free_fn_t free_fn, 3165 3163 pcpu_fc_populate_pte_fn_t populate_pte_fn) ··· 3205 3201 for (i = 0; i < unit_pages; i++) { 3206 3202 void *ptr; 3207 3203 3208 - ptr = alloc_fn(cpu, PAGE_SIZE, PAGE_SIZE); 3204 + ptr = alloc_fn(cpu, PAGE_SIZE, PAGE_SIZE, cpu_to_nd_fn); 3209 3205 if (!ptr) { 3210 3206 pr_warn("failed to allocate %s page for cpu%u\n", 3211 3207 psize_str, cpu); ··· 3282 3278 unsigned long __per_cpu_offset[NR_CPUS] __read_mostly; 3283 3279 EXPORT_SYMBOL(__per_cpu_offset); 3284 3280 3285 - static void * __init pcpu_dfl_fc_alloc(unsigned int cpu, size_t size, 3286 - size_t align) 3281 + static void * __init pcpu_dfl_fc_alloc(unsigned int cpu, size_t size, size_t align, 3282 + pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn) 3287 3283 { 3288 3284 return memblock_alloc_from(size, align, __pa(MAX_DMA_ADDRESS)); 3289 3285 } ··· 3304 3300 * what the legacy allocator did. 3305 3301 */ 3306 3302 rc = pcpu_embed_first_chunk(PERCPU_MODULE_RESERVE, 3307 - PERCPU_DYNAMIC_RESERVE, PAGE_SIZE, NULL, 3303 + PERCPU_DYNAMIC_RESERVE, PAGE_SIZE, NULL, NULL, 3308 3304 pcpu_dfl_fc_alloc, pcpu_dfl_fc_free); 3309 3305 if (rc < 0) 3310 3306 panic("Failed to initialize percpu areas.");

Configure Feed

Configure Feed