Revert "mm/cma: manage the memory of the CMA area by using the ZONE_MOVABLE"

+1 -15

arch/arm/mm/dma-mapping.c

··· 466 466 void __init dma_contiguous_remap(void) 467 467 { 468 468 int i; 469 - 470 - if (!dma_mmu_remap_num) 471 - return; 472 - 473 - /* call flush_cache_all() since CMA area would be large enough */ 474 - flush_cache_all(); 475 469 for (i = 0; i < dma_mmu_remap_num; i++) { 476 470 phys_addr_t start = dma_mmu_remap[i].base; 477 471 phys_addr_t end = start + dma_mmu_remap[i].size; ··· 498 504 flush_tlb_kernel_range(__phys_to_virt(start), 499 505 __phys_to_virt(end)); 500 506 501 - /* 502 - * All the memory in CMA region will be on ZONE_MOVABLE. 503 - * If that zone is considered as highmem, the memory in CMA 504 - * region is also considered as highmem even if it's 505 - * physical address belong to lowmem. In this case, 506 - * re-mapping isn't required. 507 - */ 508 - if (!is_highmem_idx(ZONE_MOVABLE)) 509 - iotable_init(&map, 1); 507 + iotable_init(&map, 1); 510 508 } 511 509 } 512 510

+3

include/linux/memory_hotplug.h

··· 216 216 void mem_hotplug_begin(void); 217 217 void mem_hotplug_done(void); 218 218 219 + extern void set_zone_contiguous(struct zone *zone); 220 + extern void clear_zone_contiguous(struct zone *zone); 221 + 219 222 #else /* ! CONFIG_MEMORY_HOTPLUG */ 220 223 #define pfn_to_online_page(pfn) \ 221 224 ({ \

-1

include/linux/mm.h

··· 2109 2109 2110 2110 extern void zone_pcp_update(struct zone *zone); 2111 2111 extern void zone_pcp_reset(struct zone *zone); 2112 - extern void setup_zone_pageset(struct zone *zone); 2113 2112 2114 2113 /* page_alloc.c */ 2115 2114 extern int min_free_kbytes;

+11 -72

mm/cma.c

··· 39 39 #include <trace/events/cma.h> 40 40 41 41 #include "cma.h" 42 - #include "internal.h" 43 42 44 43 struct cma cma_areas[MAX_CMA_AREAS]; 45 44 unsigned cma_area_count; ··· 109 110 if (!cma->bitmap) 110 111 return -ENOMEM; 111 112 113 + WARN_ON_ONCE(!pfn_valid(pfn)); 114 + zone = page_zone(pfn_to_page(pfn)); 115 + 112 116 do { 113 117 unsigned j; 114 118 115 119 base_pfn = pfn; 116 - if (!pfn_valid(base_pfn)) 117 - goto err; 118 - 119 - zone = page_zone(pfn_to_page(base_pfn)); 120 120 for (j = pageblock_nr_pages; j; --j, pfn++) { 121 - if (!pfn_valid(pfn)) 122 - goto err; 123 - 121 + WARN_ON_ONCE(!pfn_valid(pfn)); 124 122 /* 125 - * In init_cma_reserved_pageblock(), present_pages 126 - * is adjusted with assumption that all pages in 127 - * the pageblock come from a single zone. 123 + * alloc_contig_range requires the pfn range 124 + * specified to be in the same zone. Make this 125 + * simple by forcing the entire CMA resv range 126 + * to be in the same zone. 128 127 */ 129 128 if (page_zone(pfn_to_page(pfn)) != zone) 130 - goto err; 129 + goto not_in_zone; 131 130 } 132 131 init_cma_reserved_pageblock(pfn_to_page(base_pfn)); 133 132 } while (--i); ··· 139 142 140 143 return 0; 141 144 142 - err: 145 + not_in_zone: 143 146 pr_err("CMA area %s could not be activated\n", cma->name); 144 147 kfree(cma->bitmap); 145 148 cma->count = 0; ··· 149 152 static int __init cma_init_reserved_areas(void) 150 153 { 151 154 int i; 152 - struct zone *zone; 153 - pg_data_t *pgdat; 154 - 155 - if (!cma_area_count) 156 - return 0; 157 - 158 - for_each_online_pgdat(pgdat) { 159 - unsigned long start_pfn = UINT_MAX, end_pfn = 0; 160 - 161 - zone = &pgdat->node_zones[ZONE_MOVABLE]; 162 - 163 - /* 164 - * In this case, we cannot adjust the zone range 165 - * since it is now maximum node span and we don't 166 - * know original zone range. 167 - */ 168 - if (populated_zone(zone)) 169 - continue; 170 - 171 - for (i = 0; i < cma_area_count; i++) { 172 - if (pfn_to_nid(cma_areas[i].base_pfn) != 173 - pgdat->node_id) 174 - continue; 175 - 176 - start_pfn = min(start_pfn, cma_areas[i].base_pfn); 177 - end_pfn = max(end_pfn, cma_areas[i].base_pfn + 178 - cma_areas[i].count); 179 - } 180 - 181 - if (!end_pfn) 182 - continue; 183 - 184 - zone->zone_start_pfn = start_pfn; 185 - zone->spanned_pages = end_pfn - start_pfn; 186 - } 187 155 188 156 for (i = 0; i < cma_area_count; i++) { 189 157 int ret = cma_activate_area(&cma_areas[i]); ··· 157 195 return ret; 158 196 } 159 197 160 - /* 161 - * Reserved pages for ZONE_MOVABLE are now activated and 162 - * this would change ZONE_MOVABLE's managed page counter and 163 - * the other zones' present counter. We need to re-calculate 164 - * various zone information that depends on this initialization. 165 - */ 166 - build_all_zonelists(NULL); 167 - for_each_populated_zone(zone) { 168 - if (zone_idx(zone) == ZONE_MOVABLE) { 169 - zone_pcp_reset(zone); 170 - setup_zone_pageset(zone); 171 - } else 172 - zone_pcp_update(zone); 173 - 174 - set_zone_contiguous(zone); 175 - } 176 - 177 - /* 178 - * We need to re-init per zone wmark by calling 179 - * init_per_zone_wmark_min() but doesn't call here because it is 180 - * registered on core_initcall and it will be called later than us. 181 - */ 182 - 183 198 return 0; 184 199 } 185 - pure_initcall(cma_init_reserved_areas); 200 + core_initcall(cma_init_reserved_areas); 186 201 187 202 /** 188 203 * cma_init_reserved_mem() - create custom contiguous area from reserved memory

+3 -1

mm/compaction.c

··· 1450 1450 * if compaction succeeds. 1451 1451 * For costly orders, we require low watermark instead of min for 1452 1452 * compaction to proceed to increase its chances. 1453 + * ALLOC_CMA is used, as pages in CMA pageblocks are considered 1454 + * suitable migration targets 1453 1455 */ 1454 1456 watermark = (order > PAGE_ALLOC_COSTLY_ORDER) ? 1455 1457 low_wmark_pages(zone) : min_wmark_pages(zone); 1456 1458 watermark += compact_gap(order); 1457 1459 if (!__zone_watermark_ok(zone, 0, watermark, classzone_idx, 1458 - 0, wmark_target)) 1460 + ALLOC_CMA, wmark_target)) 1459 1461 return COMPACT_SKIPPED; 1460 1462 1461 1463 return COMPACT_CONTINUE;

+1 -3

mm/internal.h

··· 168 168 gfp_t gfp_flags); 169 169 extern int user_min_free_kbytes; 170 170 171 - extern void set_zone_contiguous(struct zone *zone); 172 - extern void clear_zone_contiguous(struct zone *zone); 173 - 174 171 #if defined CONFIG_COMPACTION || defined CONFIG_CMA 175 172 176 173 /* ··· 495 498 #define ALLOC_HARDER 0x10 /* try to alloc harder */ 496 499 #define ALLOC_HIGH 0x20 /* __GFP_HIGH set */ 497 500 #define ALLOC_CPUSET 0x40 /* check for correct cpuset */ 501 + #define ALLOC_CMA 0x80 /* allow allocations from CMA areas */ 498 502 499 503 enum ttu_flags; 500 504 struct tlbflush_unmap_batch;

+30 -53

mm/page_alloc.c

··· 1743 1743 } 1744 1744 1745 1745 #ifdef CONFIG_CMA 1746 - static void __init adjust_present_page_count(struct page *page, long count) 1747 - { 1748 - struct zone *zone = page_zone(page); 1749 - 1750 - /* We don't need to hold a lock since it is boot-up process */ 1751 - zone->present_pages += count; 1752 - } 1753 - 1754 1746 /* Free whole pageblock and set its migration type to MIGRATE_CMA. */ 1755 1747 void __init init_cma_reserved_pageblock(struct page *page) 1756 1748 { 1757 1749 unsigned i = pageblock_nr_pages; 1758 - unsigned long pfn = page_to_pfn(page); 1759 1750 struct page *p = page; 1760 - int nid = page_to_nid(page); 1761 - 1762 - /* 1763 - * ZONE_MOVABLE will steal present pages from other zones by 1764 - * changing page links so page_zone() is changed. Before that, 1765 - * we need to adjust previous zone's page count first. 1766 - */ 1767 - adjust_present_page_count(page, -pageblock_nr_pages); 1768 1751 1769 1752 do { 1770 1753 __ClearPageReserved(p); 1771 1754 set_page_count(p, 0); 1772 - 1773 - /* Steal pages from other zones */ 1774 - set_page_links(p, ZONE_MOVABLE, nid, pfn); 1775 - } while (++p, ++pfn, --i); 1776 - 1777 - adjust_present_page_count(page, pageblock_nr_pages); 1755 + } while (++p, --i); 1778 1756 1779 1757 set_pageblock_migratetype(page, MIGRATE_CMA); 1780 1758 ··· 2867 2889 * exists. 2868 2890 */ 2869 2891 watermark = min_wmark_pages(zone) + (1UL << order); 2870 - if (!zone_watermark_ok(zone, 0, watermark, 0, 0)) 2892 + if (!zone_watermark_ok(zone, 0, watermark, 0, ALLOC_CMA)) 2871 2893 return 0; 2872 2894 2873 2895 __mod_zone_freepage_state(zone, -(1UL << order), mt); ··· 3143 3165 } 3144 3166 3145 3167 3168 + #ifdef CONFIG_CMA 3169 + /* If allocation can't use CMA areas don't use free CMA pages */ 3170 + if (!(alloc_flags & ALLOC_CMA)) 3171 + free_pages -= zone_page_state(z, NR_FREE_CMA_PAGES); 3172 + #endif 3173 + 3146 3174 /* 3147 3175 * Check watermarks for an order-0 allocation request. If these 3148 3176 * are not met, then a high-order request also cannot go ahead ··· 3175 3191 } 3176 3192 3177 3193 #ifdef CONFIG_CMA 3178 - if (!list_empty(&area->free_list[MIGRATE_CMA])) 3194 + if ((alloc_flags & ALLOC_CMA) && 3195 + !list_empty(&area->free_list[MIGRATE_CMA])) { 3179 3196 return true; 3197 + } 3180 3198 #endif 3181 3199 if (alloc_harder && 3182 3200 !list_empty(&area->free_list[MIGRATE_HIGHATOMIC])) ··· 3198 3212 unsigned long mark, int classzone_idx, unsigned int alloc_flags) 3199 3213 { 3200 3214 long free_pages = zone_page_state(z, NR_FREE_PAGES); 3215 + long cma_pages = 0; 3216 + 3217 + #ifdef CONFIG_CMA 3218 + /* If allocation can't use CMA areas don't use free CMA pages */ 3219 + if (!(alloc_flags & ALLOC_CMA)) 3220 + cma_pages = zone_page_state(z, NR_FREE_CMA_PAGES); 3221 + #endif 3201 3222 3202 3223 /* 3203 3224 * Fast check for order-0 only. If this fails then the reserves ··· 3213 3220 * the caller is !atomic then it'll uselessly search the free 3214 3221 * list. That corner case is then slower but it is harmless. 3215 3222 */ 3216 - if (!order && free_pages > mark + z->lowmem_reserve[classzone_idx]) 3223 + if (!order && (free_pages - cma_pages) > mark + z->lowmem_reserve[classzone_idx]) 3217 3224 return true; 3218 3225 3219 3226 return __zone_watermark_ok(z, order, mark, classzone_idx, alloc_flags, ··· 3849 3856 } else if (unlikely(rt_task(current)) && !in_interrupt()) 3850 3857 alloc_flags |= ALLOC_HARDER; 3851 3858 3859 + #ifdef CONFIG_CMA 3860 + if (gfpflags_to_migratetype(gfp_mask) == MIGRATE_MOVABLE) 3861 + alloc_flags |= ALLOC_CMA; 3862 + #endif 3852 3863 return alloc_flags; 3853 3864 } 3854 3865 ··· 4318 4321 4319 4322 if (should_fail_alloc_page(gfp_mask, order)) 4320 4323 return false; 4324 + 4325 + if (IS_ENABLED(CONFIG_CMA) && ac->migratetype == MIGRATE_MOVABLE) 4326 + *alloc_flags |= ALLOC_CMA; 4321 4327 4322 4328 return true; 4323 4329 } ··· 6204 6204 { 6205 6205 enum zone_type j; 6206 6206 int nid = pgdat->node_id; 6207 - unsigned long node_end_pfn = 0; 6208 6207 6209 6208 pgdat_resize_init(pgdat); 6210 6209 #ifdef CONFIG_NUMA_BALANCING ··· 6231 6232 struct zone *zone = pgdat->node_zones + j; 6232 6233 unsigned long size, realsize, freesize, memmap_pages; 6233 6234 unsigned long zone_start_pfn = zone->zone_start_pfn; 6234 - unsigned long movable_size = 0; 6235 6235 6236 6236 size = zone->spanned_pages; 6237 6237 realsize = freesize = zone->present_pages; 6238 - if (zone_end_pfn(zone) > node_end_pfn) 6239 - node_end_pfn = zone_end_pfn(zone); 6240 - 6241 6238 6242 6239 /* 6243 6240 * Adjust freesize so that it accounts for how much memory ··· 6282 6287 zone_seqlock_init(zone); 6283 6288 zone_pcp_init(zone); 6284 6289 6285 - /* 6286 - * The size of the CMA area is unknown now so we need to 6287 - * prepare the memory for the usemap at maximum. 6288 - */ 6289 - if (IS_ENABLED(CONFIG_CMA) && j == ZONE_MOVABLE && 6290 - pgdat->node_spanned_pages) { 6291 - movable_size = node_end_pfn - pgdat->node_start_pfn; 6292 - } 6293 - 6294 - if (!size && !movable_size) 6290 + if (!size) 6295 6291 continue; 6296 6292 6297 6293 set_pageblock_order(); 6298 - if (movable_size) { 6299 - zone->zone_start_pfn = pgdat->node_start_pfn; 6300 - zone->spanned_pages = movable_size; 6301 - setup_usemap(pgdat, zone, 6302 - pgdat->node_start_pfn, movable_size); 6303 - init_currently_empty_zone(zone, 6304 - pgdat->node_start_pfn, movable_size); 6305 - } else { 6306 - setup_usemap(pgdat, zone, zone_start_pfn, size); 6307 - init_currently_empty_zone(zone, zone_start_pfn, size); 6308 - } 6294 + setup_usemap(pgdat, zone, zone_start_pfn, size); 6295 + init_currently_empty_zone(zone, zone_start_pfn, size); 6309 6296 memmap_init(size, nid, j, zone_start_pfn); 6310 6297 } 6311 6298 } ··· 7928 7951 } 7929 7952 #endif 7930 7953 7931 - #if defined CONFIG_MEMORY_HOTPLUG || defined CONFIG_CMA 7954 + #ifdef CONFIG_MEMORY_HOTPLUG 7932 7955 /* 7933 7956 * The zone indicated has a new number of managed_pages; batch sizes and percpu 7934 7957 * page high values need to be recalulated.

Configure Feed

Configure Feed