mm: percpu: add generic pcpu_fc_alloc/free funciton

+1 -15

arch/mips/mm/init.c

··· 524 524 return cpu_to_node(cpu); 525 525 } 526 526 527 - static void * __init pcpu_fc_alloc(unsigned int cpu, size_t size, size_t align, 528 - pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn) 529 - { 530 - return memblock_alloc_try_nid(size, align, __pa(MAX_DMA_ADDRESS), 531 - MEMBLOCK_ALLOC_ACCESSIBLE, 532 - cpu_to_nd_fn(cpu)); 533 - } 534 - 535 - static void __init pcpu_fc_free(void *ptr, size_t size) 536 - { 537 - memblock_free(ptr, size); 538 - } 539 - 540 527 void __init setup_per_cpu_areas(void) 541 528 { 542 529 unsigned long delta; ··· 537 550 rc = pcpu_embed_first_chunk(PERCPU_MODULE_RESERVE, 538 551 PERCPU_DYNAMIC_RESERVE, PAGE_SIZE, 539 552 pcpu_cpu_distance, 540 - pcpu_cpu_to_node, 541 - pcpu_fc_alloc, pcpu_fc_free); 553 + pcpu_cpu_to_node); 542 554 if (rc < 0) 543 555 panic("Failed to initialize percpu areas."); 544 556

+2 -49

arch/powerpc/kernel/setup_64.c

··· 771 771 } 772 772 773 773 #ifdef CONFIG_SMP 774 - /** 775 - * pcpu_alloc_bootmem - NUMA friendly alloc_bootmem wrapper for percpu 776 - * @cpu: cpu to allocate for 777 - * @size: size allocation in bytes 778 - * @align: alignment 779 - * 780 - * Allocate @size bytes aligned at @align for cpu @cpu. This wrapper 781 - * does the right thing for NUMA regardless of the current 782 - * configuration. 783 - * 784 - * RETURNS: 785 - * Pointer to the allocated area on success, NULL on failure. 786 - */ 787 - static void * __init pcpu_alloc_bootmem(unsigned int cpu, size_t size, size_t align, 788 - pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn) 789 - { 790 - const unsigned long goal = __pa(MAX_DMA_ADDRESS); 791 - #ifdef CONFIG_NUMA 792 - int node = cpu_to_nd_fun(cpu); 793 - void *ptr; 794 - 795 - if (!node_online(node) || !NODE_DATA(node)) { 796 - ptr = memblock_alloc_from(size, align, goal); 797 - pr_info("cpu %d has no node %d or node-local memory\n", 798 - cpu, node); 799 - pr_debug("per cpu data for cpu%d %lu bytes at %016lx\n", 800 - cpu, size, __pa(ptr)); 801 - } else { 802 - ptr = memblock_alloc_try_nid(size, align, goal, 803 - MEMBLOCK_ALLOC_ACCESSIBLE, node); 804 - pr_debug("per cpu data for cpu%d %lu bytes on node%d at " 805 - "%016lx\n", cpu, size, node, __pa(ptr)); 806 - } 807 - return ptr; 808 - #else 809 - return memblock_alloc_from(size, align, goal); 810 - #endif 811 - } 812 - 813 - static void __init pcpu_free_bootmem(void *ptr, size_t size) 814 - { 815 - memblock_free(ptr, size); 816 - } 817 - 818 774 static int pcpu_cpu_distance(unsigned int from, unsigned int to) 819 775 { 820 776 if (early_cpu_to_node(from) == early_cpu_to_node(to)) ··· 852 896 853 897 if (pcpu_chosen_fc != PCPU_FC_PAGE) { 854 898 rc = pcpu_embed_first_chunk(0, dyn_size, atom_size, pcpu_cpu_distance, 855 - pcpu_cpu_to_node, 856 - pcpu_alloc_bootmem, pcpu_free_bootmem); 899 + pcpu_cpu_to_node); 857 900 if (rc) 858 901 pr_warn("PERCPU: %s allocator failed (%d), " 859 902 "falling back to page size\n", ··· 860 905 } 861 906 862 907 if (rc < 0) 863 - rc = pcpu_page_first_chunk(0, pcpu_cpu_to_node, 864 - pcpu_alloc_bootmem, pcpu_free_bootmem, 865 - pcpu_populate_pte); 908 + rc = pcpu_page_first_chunk(0, pcpu_cpu_to_node, pcpu_populate_pte); 866 909 if (rc < 0) 867 910 panic("cannot initialize percpu area (err=%d)", rc); 868 911

+1 -49

arch/sparc/kernel/smp_64.c

··· 1526 1526 smp_call_function(stop_this_cpu, NULL, 0); 1527 1527 } 1528 1528 1529 - /** 1530 - * pcpu_alloc_bootmem - NUMA friendly alloc_bootmem wrapper for percpu 1531 - * @cpu: cpu to allocate for 1532 - * @size: size allocation in bytes 1533 - * @align: alignment 1534 - * 1535 - * Allocate @size bytes aligned at @align for cpu @cpu. This wrapper 1536 - * does the right thing for NUMA regardless of the current 1537 - * configuration. 1538 - * 1539 - * RETURNS: 1540 - * Pointer to the allocated area on success, NULL on failure. 1541 - */ 1542 - static void * __init pcpu_alloc_bootmem(unsigned int cpu, size_t size, size_t align, 1543 - pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn) 1544 - { 1545 - const unsigned long goal = __pa(MAX_DMA_ADDRESS); 1546 - #ifdef CONFIG_NUMA 1547 - int node = cpu_to_nd_fn(cpu); 1548 - void *ptr; 1549 - 1550 - if (!node_online(node) || !NODE_DATA(node)) { 1551 - ptr = memblock_alloc_from(size, align, goal); 1552 - pr_info("cpu %d has no node %d or node-local memory\n", 1553 - cpu, node); 1554 - pr_debug("per cpu data for cpu%d %lu bytes at %016lx\n", 1555 - cpu, size, __pa(ptr)); 1556 - } else { 1557 - ptr = memblock_alloc_try_nid(size, align, goal, 1558 - MEMBLOCK_ALLOC_ACCESSIBLE, node); 1559 - pr_debug("per cpu data for cpu%d %lu bytes on node%d at " 1560 - "%016lx\n", cpu, size, node, __pa(ptr)); 1561 - } 1562 - return ptr; 1563 - #else 1564 - return memblock_alloc_from(size, align, goal); 1565 - #endif 1566 - } 1567 - 1568 - static void __init pcpu_free_bootmem(void *ptr, size_t size) 1569 - { 1570 - memblock_free(ptr, size); 1571 - } 1572 - 1573 1529 static int __init pcpu_cpu_distance(unsigned int from, unsigned int to) 1574 1530 { 1575 1531 if (cpu_to_node(from) == cpu_to_node(to)) ··· 1602 1646 rc = pcpu_embed_first_chunk(PERCPU_MODULE_RESERVE, 1603 1647 PERCPU_DYNAMIC_RESERVE, 4 << 20, 1604 1648 pcpu_cpu_distance, 1605 - pcpu_cpu_to_node, 1606 - pcpu_alloc_bootmem, 1607 - pcpu_free_bootmem); 1649 + pcpu_cpu_to_node); 1608 1650 if (rc) 1609 1651 pr_warn("PERCPU: %s allocator failed (%d), " 1610 1652 "falling back to page size\n", ··· 1611 1657 if (rc < 0) 1612 1658 rc = pcpu_page_first_chunk(PERCPU_MODULE_RESERVE, 1613 1659 pcpu_cpu_to_node, 1614 - pcpu_alloc_bootmem, 1615 - pcpu_free_bootmem, 1616 1660 pcpu_populate_pte); 1617 1661 if (rc < 0) 1618 1662 panic("cannot initialize percpu area (err=%d)", rc);

+1 -58

arch/x86/kernel/setup_percpu.c

··· 84 84 } 85 85 #endif 86 86 87 - /** 88 - * pcpu_alloc_bootmem - NUMA friendly alloc_bootmem wrapper for percpu 89 - * @cpu: cpu to allocate for 90 - * @size: size allocation in bytes 91 - * @align: alignment 92 - * 93 - * Allocate @size bytes aligned at @align for cpu @cpu. This wrapper 94 - * does the right thing for NUMA regardless of the current 95 - * configuration. 96 - * 97 - * RETURNS: 98 - * Pointer to the allocated area on success, NULL on failure. 99 - */ 100 - static void * __init pcpu_alloc_bootmem(unsigned int cpu, unsigned long size, unsigned long align, 101 - pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn) 102 - { 103 - const unsigned long goal = __pa(MAX_DMA_ADDRESS); 104 - #ifdef CONFIG_NUMA 105 - int node = cpu_to_nd_fn(cpu); 106 - void *ptr; 107 - 108 - if (!node_online(node) || !NODE_DATA(node)) { 109 - ptr = memblock_alloc_from(size, align, goal); 110 - pr_info("cpu %d has no node %d or node-local memory\n", 111 - cpu, node); 112 - pr_debug("per cpu data for cpu%d %lu bytes at %016lx\n", 113 - cpu, size, __pa(ptr)); 114 - } else { 115 - ptr = memblock_alloc_try_nid(size, align, goal, 116 - MEMBLOCK_ALLOC_ACCESSIBLE, 117 - node); 118 - 119 - pr_debug("per cpu data for cpu%d %lu bytes on node%d at %016lx\n", 120 - cpu, size, node, __pa(ptr)); 121 - } 122 - return ptr; 123 - #else 124 - return memblock_alloc_from(size, align, goal); 125 - #endif 126 - } 127 - 128 - /* 129 - * Helpers for first chunk memory allocation 130 - */ 131 - static void * __init pcpu_fc_alloc(unsigned int cpu, size_t size, size_t align, 132 - pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn) 133 - { 134 - return pcpu_alloc_bootmem(cpu, size, align, cpu_to_nd_fn); 135 - } 136 - 137 - static void __init pcpu_fc_free(void *ptr, size_t size) 138 - { 139 - memblock_free(ptr, size); 140 - } 141 - 142 87 static int __init pcpu_cpu_distance(unsigned int from, unsigned int to) 143 88 { 144 89 #ifdef CONFIG_NUMA ··· 156 211 rc = pcpu_embed_first_chunk(PERCPU_FIRST_CHUNK_RESERVE, 157 212 dyn_size, atom_size, 158 213 pcpu_cpu_distance, 159 - pcpu_cpu_to_node, 160 - pcpu_fc_alloc, pcpu_fc_free); 214 + pcpu_cpu_to_node); 161 215 if (rc < 0) 162 216 pr_warn("%s allocator failed (%d), falling back to page size\n", 163 217 pcpu_fc_names[pcpu_chosen_fc], rc); ··· 164 220 if (rc < 0) 165 221 rc = pcpu_page_first_chunk(PERCPU_FIRST_CHUNK_RESERVE, 166 222 pcpu_cpu_to_node, 167 - pcpu_fc_alloc, pcpu_fc_free, 168 223 pcpup_populate_pte); 169 224 if (rc < 0) 170 225 panic("cannot initialize percpu area (err=%d)", rc);

+1 -18

drivers/base/arch_numa.c

··· 155 155 return node_distance(early_cpu_to_node(from), early_cpu_to_node(to)); 156 156 } 157 157 158 - static void * __init pcpu_fc_alloc(unsigned int cpu, size_t size, size_t align, 159 - pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn) 160 - { 161 - int nid = cpu_to_nd_fn(cpu); 162 - 163 - return memblock_alloc_try_nid(size, align, 164 - __pa(MAX_DMA_ADDRESS), MEMBLOCK_ALLOC_ACCESSIBLE, nid); 165 - } 166 - 167 - static void __init pcpu_fc_free(void *ptr, size_t size) 168 - { 169 - memblock_free(ptr, size); 170 - } 171 - 172 158 #ifdef CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK 173 159 static void __init pcpu_populate_pte(unsigned long addr) 174 160 { ··· 215 229 rc = pcpu_embed_first_chunk(PERCPU_MODULE_RESERVE, 216 230 PERCPU_DYNAMIC_RESERVE, PAGE_SIZE, 217 231 pcpu_cpu_distance, 218 - early_cpu_to_node, 219 - pcpu_fc_alloc, pcpu_fc_free); 232 + early_cpu_to_node); 220 233 #ifdef CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK 221 234 if (rc < 0) 222 235 pr_warn("PERCPU: %s allocator failed (%d), falling back to page size\n", ··· 227 242 if (rc < 0) 228 243 rc = pcpu_page_first_chunk(PERCPU_MODULE_RESERVE, 229 244 early_cpu_to_node, 230 - pcpu_fc_alloc, 231 - pcpu_fc_free, 232 245 pcpu_populate_pte); 233 246 #endif 234 247 if (rc < 0)

+1 -8

include/linux/percpu.h

··· 95 95 extern enum pcpu_fc pcpu_chosen_fc; 96 96 97 97 typedef int (pcpu_fc_cpu_to_node_fn_t)(int cpu); 98 - typedef void * (*pcpu_fc_alloc_fn_t)(unsigned int cpu, size_t size, size_t align, 99 - pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn); 100 - typedef void (*pcpu_fc_free_fn_t)(void *ptr, size_t size); 101 98 typedef void (*pcpu_fc_populate_pte_fn_t)(unsigned long addr); 102 99 typedef int (pcpu_fc_cpu_distance_fn_t)(unsigned int from, unsigned int to); 103 100 ··· 109 112 extern int __init pcpu_embed_first_chunk(size_t reserved_size, size_t dyn_size, 110 113 size_t atom_size, 111 114 pcpu_fc_cpu_distance_fn_t cpu_distance_fn, 112 - pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn, 113 - pcpu_fc_alloc_fn_t alloc_fn, 114 - pcpu_fc_free_fn_t free_fn); 115 + pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn); 115 116 #endif 116 117 117 118 #ifdef CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK 118 119 extern int __init pcpu_page_first_chunk(size_t reserved_size, 119 120 pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn, 120 - pcpu_fc_alloc_fn_t alloc_fn, 121 - pcpu_fc_free_fn_t free_fn, 122 121 pcpu_fc_populate_pte_fn_t populate_pte_fn); 123 122 #endif 124 123

+47 -31

mm/percpu.c

··· 2992 2992 2993 2993 return ai; 2994 2994 } 2995 + 2996 + static void * __init pcpu_fc_alloc(unsigned int cpu, size_t size, size_t align, 2997 + pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn) 2998 + { 2999 + const unsigned long goal = __pa(MAX_DMA_ADDRESS); 3000 + #ifdef CONFIG_NUMA 3001 + int node = NUMA_NO_NODE; 3002 + void *ptr; 3003 + 3004 + if (cpu_to_nd_fn) 3005 + node = cpu_to_nd_fn(cpu); 3006 + 3007 + if (node == NUMA_NO_NODE || !node_online(node) || !NODE_DATA(node)) { 3008 + ptr = memblock_alloc_from(size, align, goal); 3009 + pr_info("cpu %d has no node %d or node-local memory\n", 3010 + cpu, node); 3011 + pr_debug("per cpu data for cpu%d %zu bytes at 0x%llx\n", 3012 + cpu, size, (u64)__pa(ptr)); 3013 + } else { 3014 + ptr = memblock_alloc_try_nid(size, align, goal, 3015 + MEMBLOCK_ALLOC_ACCESSIBLE, 3016 + node); 3017 + 3018 + pr_debug("per cpu data for cpu%d %zu bytes on node%d at 0x%llx\n", 3019 + cpu, size, node, (u64)__pa(ptr)); 3020 + } 3021 + return ptr; 3022 + #else 3023 + return memblock_alloc_from(size, align, goal); 3024 + #endif 3025 + } 3026 + 3027 + static void __init pcpu_fc_free(void *ptr, size_t size) 3028 + { 3029 + memblock_free(ptr, size); 3030 + } 2995 3031 #endif /* BUILD_EMBED_FIRST_CHUNK || BUILD_PAGE_FIRST_CHUNK */ 2996 3032 2997 3033 #if defined(BUILD_EMBED_FIRST_CHUNK) ··· 3038 3002 * @atom_size: allocation atom size 3039 3003 * @cpu_distance_fn: callback to determine distance between cpus, optional 3040 3004 * @cpu_to_nd_fn: callback to convert cpu to it's node, optional 3041 - * @alloc_fn: function to allocate percpu page 3042 - * @free_fn: function to free percpu page 3043 3005 * 3044 3006 * This is a helper to ease setting up embedded first percpu chunk and 3045 3007 * can be called where pcpu_setup_first_chunk() is expected. 3046 3008 * 3047 3009 * If this function is used to setup the first chunk, it is allocated 3048 - * by calling @alloc_fn and used as-is without being mapped into 3010 + * by calling pcpu_fc_alloc and used as-is without being mapped into 3049 3011 * vmalloc area. Allocations are always whole multiples of @atom_size 3050 3012 * aligned to @atom_size. 3051 3013 * ··· 3057 3023 * @dyn_size specifies the minimum dynamic area size. 3058 3024 * 3059 3025 * If the needed size is smaller than the minimum or specified unit 3060 - * size, the leftover is returned using @free_fn. 3026 + * size, the leftover is returned using pcpu_fc_free. 3061 3027 * 3062 3028 * RETURNS: 3063 3029 * 0 on success, -errno on failure. ··· 3065 3031 int __init pcpu_embed_first_chunk(size_t reserved_size, size_t dyn_size, 3066 3032 size_t atom_size, 3067 3033 pcpu_fc_cpu_distance_fn_t cpu_distance_fn, 3068 - pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn, 3069 - pcpu_fc_alloc_fn_t alloc_fn, 3070 - pcpu_fc_free_fn_t free_fn) 3034 + pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn) 3071 3035 { 3072 3036 void *base = (void *)ULONG_MAX; 3073 3037 void **areas = NULL; ··· 3100 3068 BUG_ON(cpu == NR_CPUS); 3101 3069 3102 3070 /* allocate space for the whole group */ 3103 - ptr = alloc_fn(cpu, gi->nr_units * ai->unit_size, atom_size, cpu_to_nd_fn); 3071 + ptr = pcpu_fc_alloc(cpu, gi->nr_units * ai->unit_size, atom_size, cpu_to_nd_fn); 3104 3072 if (!ptr) { 3105 3073 rc = -ENOMEM; 3106 3074 goto out_free_areas; ··· 3139 3107 for (i = 0; i < gi->nr_units; i++, ptr += ai->unit_size) { 3140 3108 if (gi->cpu_map[i] == NR_CPUS) { 3141 3109 /* unused unit, free whole */ 3142 - free_fn(ptr, ai->unit_size); 3110 + pcpu_fc_free(ptr, ai->unit_size); 3143 3111 continue; 3144 3112 } 3145 3113 /* copy and return the unused part */ 3146 3114 memcpy(ptr, __per_cpu_load, ai->static_size); 3147 - free_fn(ptr + size_sum, ai->unit_size - size_sum); 3115 + pcpu_fc_free(ptr + size_sum, ai->unit_size - size_sum); 3148 3116 } 3149 3117 } 3150 3118 ··· 3163 3131 out_free_areas: 3164 3132 for (group = 0; group < ai->nr_groups; group++) 3165 3133 if (areas[group]) 3166 - free_fn(areas[group], 3134 + pcpu_fc_free(areas[group], 3167 3135 ai->groups[group].nr_units * ai->unit_size); 3168 3136 out_free: 3169 3137 pcpu_free_alloc_info(ai); ··· 3178 3146 * pcpu_page_first_chunk - map the first chunk using PAGE_SIZE pages 3179 3147 * @reserved_size: the size of reserved percpu area in bytes 3180 3148 * @cpu_to_nd_fn: callback to convert cpu to it's node, optional 3181 - * @alloc_fn: function to allocate percpu page, always called with PAGE_SIZE 3182 - * @free_fn: function to free percpu page, always called with PAGE_SIZE 3183 3149 * @populate_pte_fn: function to populate pte 3184 3150 * 3185 3151 * This is a helper to ease setting up page-remapped first percpu ··· 3191 3161 */ 3192 3162 int __init pcpu_page_first_chunk(size_t reserved_size, 3193 3163 pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn, 3194 - pcpu_fc_alloc_fn_t alloc_fn, 3195 - pcpu_fc_free_fn_t free_fn, 3196 3164 pcpu_fc_populate_pte_fn_t populate_pte_fn) 3197 3165 { 3198 3166 static struct vm_struct vm; ··· 3233 3205 for (i = 0; i < unit_pages; i++) { 3234 3206 void *ptr; 3235 3207 3236 - ptr = alloc_fn(cpu, PAGE_SIZE, PAGE_SIZE, cpu_to_nd_fn); 3208 + ptr = pcpu_fc_alloc(cpu, PAGE_SIZE, PAGE_SIZE, cpu_to_nd_fn); 3237 3209 if (!ptr) { 3238 3210 pr_warn("failed to allocate %s page for cpu%u\n", 3239 3211 psize_str, cpu); ··· 3285 3257 3286 3258 enomem: 3287 3259 while (--j >= 0) 3288 - free_fn(page_address(pages[j]), PAGE_SIZE); 3260 + pcpu_fc_free(page_address(pages[j]), PAGE_SIZE); 3289 3261 rc = -ENOMEM; 3290 3262 out_free_ar: 3291 3263 memblock_free(pages, pages_size); ··· 3310 3282 unsigned long __per_cpu_offset[NR_CPUS] __read_mostly; 3311 3283 EXPORT_SYMBOL(__per_cpu_offset); 3312 3284 3313 - static void * __init pcpu_dfl_fc_alloc(unsigned int cpu, size_t size, size_t align, 3314 - pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn) 3315 - { 3316 - return memblock_alloc_from(size, align, __pa(MAX_DMA_ADDRESS)); 3317 - } 3318 - 3319 - static void __init pcpu_dfl_fc_free(void *ptr, size_t size) 3320 - { 3321 - memblock_free(ptr, size); 3322 - } 3323 - 3324 3285 void __init setup_per_cpu_areas(void) 3325 3286 { 3326 3287 unsigned long delta; ··· 3320 3303 * Always reserve area for module percpu variables. That's 3321 3304 * what the legacy allocator did. 3322 3305 */ 3323 - rc = pcpu_embed_first_chunk(PERCPU_MODULE_RESERVE, 3324 - PERCPU_DYNAMIC_RESERVE, PAGE_SIZE, NULL, NULL, 3325 - pcpu_dfl_fc_alloc, pcpu_dfl_fc_free); 3306 + rc = pcpu_embed_first_chunk(PERCPU_MODULE_RESERVE, PERCPU_DYNAMIC_RESERVE, 3307 + PAGE_SIZE, NULL, NULL); 3326 3308 if (rc < 0) 3327 3309 panic("Failed to initialize percpu areas."); 3328 3310

Configure Feed

Configure Feed