Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
mm/mm_init: use deferred_init_memmap_chunk() in deferred_grow_zone()
deferred_grow_zone() initializes one or more sections in the memory map
if buddy runs out of initialized struct pages when
CONFIG_DEFERRED_STRUCT_PAGE_INIT is enabled.
It loops through memblock regions and initializes and frees pages in
MAX_ORDER_NR_PAGES chunks.
Essentially the same loop is implemented in deferred_init_memmap_chunk(),
the only actual difference is that deferred_init_memmap_chunk() does not
count initialized pages.
Make deferred_init_memmap_chunk() count the initialized pages and return
their number, wrap it with deferred_init_memmap_job() for multithreaded
initialization with padata_do_multithreaded() and replace open-coded
initialization of struct pages in deferred_grow_zone() with a call to
deferred_init_memmap_chunk().
Reviewed-by: David Hildenbrand <david@redhat.com>
Reviewed-by: Wei Yang <richard.weiyang@gmail.com>
Signed-off-by: Mike Rapoport (Microsoft) <rppt@kernel.org>
+36
-34
+36
-34
mm/mm_init.c
+36
-34
mm/mm_init.c
···
2134
2134
return nr_pages;
2135
2135
}
2136
2136
2137
-
static void __init
2137
+
static unsigned long __init
2138
2138
deferred_init_memmap_chunk(unsigned long start_pfn, unsigned long end_pfn,
2139
-
void *arg)
2139
+
struct zone *zone)
2140
2140
{
2141
+
unsigned long nr_pages = 0;
2141
2142
unsigned long spfn, epfn;
2142
-
struct zone *zone = arg;
2143
2143
u64 i = 0;
2144
2144
2145
2145
deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn, start_pfn);
···
2149
2149
* we can avoid introducing any issues with the buddy allocator.
2150
2150
*/
2151
2151
while (spfn < end_pfn) {
2152
-
deferred_init_maxorder(&i, zone, &spfn, &epfn);
2153
-
cond_resched();
2152
+
nr_pages += deferred_init_maxorder(&i, zone, &spfn, &epfn);
2153
+
if (irqs_disabled())
2154
+
touch_nmi_watchdog();
2155
+
else
2156
+
cond_resched();
2154
2157
}
2158
+
2159
+
return nr_pages;
2160
+
}
2161
+
2162
+
static void __init
2163
+
deferred_init_memmap_job(unsigned long start_pfn, unsigned long end_pfn,
2164
+
void *arg)
2165
+
{
2166
+
struct zone *zone = arg;
2167
+
2168
+
deferred_init_memmap_chunk(start_pfn, end_pfn, zone);
2155
2169
}
2156
2170
2157
2171
static unsigned int __init
···
2218
2204
while (deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn, first_init_pfn)) {
2219
2205
first_init_pfn = ALIGN(epfn, PAGES_PER_SECTION);
2220
2206
struct padata_mt_job job = {
2221
-
.thread_fn = deferred_init_memmap_chunk,
2207
+
.thread_fn = deferred_init_memmap_job,
2222
2208
.fn_arg = zone,
2223
2209
.start = spfn,
2224
2210
.size = first_init_pfn - spfn,
···
2254
2240
*/
2255
2241
bool __init deferred_grow_zone(struct zone *zone, unsigned int order)
2256
2242
{
2257
-
unsigned long nr_pages_needed = ALIGN(1 << order, PAGES_PER_SECTION);
2243
+
unsigned long nr_pages_needed = SECTION_ALIGN_UP(1 << order);
2258
2244
pg_data_t *pgdat = zone->zone_pgdat;
2259
2245
unsigned long first_deferred_pfn = pgdat->first_deferred_pfn;
2260
2246
unsigned long spfn, epfn, flags;
2261
2247
unsigned long nr_pages = 0;
2262
-
u64 i = 0;
2263
2248
2264
2249
/* Only the last zone may have deferred pages */
2265
2250
if (zone_end_pfn(zone) != pgdat_end_pfn(pgdat))
···
2275
2262
return true;
2276
2263
}
2277
2264
2278
-
/* If the zone is empty somebody else may have cleared out the zone */
2279
-
if (!deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn,
2280
-
first_deferred_pfn)) {
2281
-
pgdat->first_deferred_pfn = ULONG_MAX;
2282
-
pgdat_resize_unlock(pgdat, &flags);
2283
-
/* Retry only once. */
2284
-
return first_deferred_pfn != ULONG_MAX;
2265
+
/*
2266
+
* Initialize at least nr_pages_needed in section chunks.
2267
+
* If a section has less free memory than nr_pages_needed, the next
2268
+
* section will be also initialized.
2269
+
* Note, that it still does not guarantee that allocation of order can
2270
+
* be satisfied if the sections are fragmented because of memblock
2271
+
* allocations.
2272
+
*/
2273
+
for (spfn = first_deferred_pfn, epfn = SECTION_ALIGN_UP(spfn + 1);
2274
+
nr_pages < nr_pages_needed && spfn < zone_end_pfn(zone);
2275
+
spfn = epfn, epfn += PAGES_PER_SECTION) {
2276
+
nr_pages += deferred_init_memmap_chunk(spfn, epfn, zone);
2285
2277
}
2286
2278
2287
2279
/*
2288
-
* Initialize and free pages in MAX_PAGE_ORDER sized increments so
2289
-
* that we can avoid introducing any issues with the buddy
2290
-
* allocator.
2280
+
* There were no pages to initialize and free which means the zone's
2281
+
* memory map is completely initialized.
2291
2282
*/
2292
-
while (spfn < epfn) {
2293
-
/* update our first deferred PFN for this section */
2294
-
first_deferred_pfn = spfn;
2283
+
pgdat->first_deferred_pfn = nr_pages ? spfn : ULONG_MAX;
2295
2284
2296
-
nr_pages += deferred_init_maxorder(&i, zone, &spfn, &epfn);
2297
-
touch_nmi_watchdog();
2298
-
2299
-
/* We should only stop along section boundaries */
2300
-
if ((first_deferred_pfn ^ spfn) < PAGES_PER_SECTION)
2301
-
continue;
2302
-
2303
-
/* If our quota has been met we can stop here */
2304
-
if (nr_pages >= nr_pages_needed)
2305
-
break;
2306
-
}
2307
-
2308
-
pgdat->first_deferred_pfn = spfn;
2309
2285
pgdat_resize_unlock(pgdat, &flags);
2310
2286
2311
2287
return nr_pages > 0;