Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
mm/mm_init: drop deferred_init_maxorder()
deferred_init_memmap_chunk() calls deferred_init_maxorder() to initialize
struct pages in MAX_ORDER_NR_PAGES because according to commit 0e56acae4b4d
("mm: initialize MAX_ORDER_NR_PAGES at a time instead of doing larger
sections") this provides better cache locality than initializing the memory
map in larger sections.
The looping through free memory ranges is quite cumbersome in the current
implementation as it is divided between deferred_init_memmap_chunk() and
deferred_init_maxorder(). Besides, the latter has two loops, one that
initializes struct pages and another one that frees them.
There is no need in two loops because it is safe to free pages in groups
smaller than MAX_ORDER_NR_PAGES. Even if lookup for a buddy page will
access a struct page ahead of the pages being initialized, that page is
guaranteed to be initialized either by memmap_init_reserved_pages() or by
init_unavailable_range().
Simplify the code by moving initialization and freeing of the pages into
deferred_init_memmap_chunk() and dropping deferred_init_maxorder().
Reviewed-by: David Hildenbrand <david@redhat.com>
Reviewed-by: Wei Yang <richard.weiyang@gmail.com>
Signed-off-by: Mike Rapoport (Microsoft) <rppt@kernel.org>
+35
-98
+35
-98
mm/mm_init.c
+35
-98
mm/mm_init.c
···
2046
2046
}
2047
2047
2048
2048
/*
2049
-
* This function is meant to pre-load the iterator for the zone init from
2050
-
* a given point.
2051
-
* Specifically it walks through the ranges starting with initial index
2052
-
* passed to it until we are caught up to the first_init_pfn value and
2053
-
* exits there. If we never encounter the value we return false indicating
2054
-
* there are no valid ranges left.
2055
-
*/
2056
-
static bool __init
2057
-
deferred_init_mem_pfn_range_in_zone(u64 *i, struct zone *zone,
2058
-
unsigned long *spfn, unsigned long *epfn,
2059
-
unsigned long first_init_pfn)
2060
-
{
2061
-
u64 j = *i;
2062
-
2063
-
if (j == 0)
2064
-
__next_mem_pfn_range_in_zone(&j, zone, spfn, epfn);
2065
-
2066
-
/*
2067
-
* Start out by walking through the ranges in this zone that have
2068
-
* already been initialized. We don't need to do anything with them
2069
-
* so we just need to flush them out of the system.
2070
-
*/
2071
-
for_each_free_mem_pfn_range_in_zone_from(j, zone, spfn, epfn) {
2072
-
if (*epfn <= first_init_pfn)
2073
-
continue;
2074
-
if (*spfn < first_init_pfn)
2075
-
*spfn = first_init_pfn;
2076
-
*i = j;
2077
-
return true;
2078
-
}
2079
-
2080
-
return false;
2081
-
}
2082
-
2083
-
/*
2084
-
* Initialize and free pages. We do it in two loops: first we initialize
2085
-
* struct page, then free to buddy allocator, because while we are
2086
-
* freeing pages we can access pages that are ahead (computing buddy
2087
-
* page in __free_one_page()).
2049
+
* Initialize and free pages.
2088
2050
*
2089
-
* In order to try and keep some memory in the cache we have the loop
2090
-
* broken along max page order boundaries. This way we will not cause
2091
-
* any issues with the buddy page computation.
2051
+
* At this point reserved pages and struct pages that correspond to holes in
2052
+
* memblock.memory are already intialized so every free range has a valid
2053
+
* memory map around it.
2054
+
* This ensures that access of pages that are ahead of the range being
2055
+
* initialized (computing buddy page in __free_one_page()) always reads a valid
2056
+
* struct page.
2057
+
*
2058
+
* In order to try and improve CPU cache locality we have the loop broken along
2059
+
* max page order boundaries.
2092
2060
*/
2093
-
static unsigned long __init
2094
-
deferred_init_maxorder(u64 *i, struct zone *zone, unsigned long *start_pfn,
2095
-
unsigned long *end_pfn)
2096
-
{
2097
-
unsigned long mo_pfn = ALIGN(*start_pfn + 1, MAX_ORDER_NR_PAGES);
2098
-
unsigned long spfn = *start_pfn, epfn = *end_pfn;
2099
-
unsigned long nr_pages = 0;
2100
-
u64 j = *i;
2101
-
2102
-
/* First we loop through and initialize the page values */
2103
-
for_each_free_mem_pfn_range_in_zone_from(j, zone, start_pfn, end_pfn) {
2104
-
unsigned long t;
2105
-
2106
-
if (mo_pfn <= *start_pfn)
2107
-
break;
2108
-
2109
-
t = min(mo_pfn, *end_pfn);
2110
-
nr_pages += deferred_init_pages(zone, *start_pfn, t);
2111
-
2112
-
if (mo_pfn < *end_pfn) {
2113
-
*start_pfn = mo_pfn;
2114
-
break;
2115
-
}
2116
-
}
2117
-
2118
-
/* Reset values and now loop through freeing pages as needed */
2119
-
swap(j, *i);
2120
-
2121
-
for_each_free_mem_pfn_range_in_zone_from(j, zone, &spfn, &epfn) {
2122
-
unsigned long t;
2123
-
2124
-
if (mo_pfn <= spfn)
2125
-
break;
2126
-
2127
-
t = min(mo_pfn, epfn);
2128
-
deferred_free_pages(spfn, t - spfn);
2129
-
2130
-
if (mo_pfn <= epfn)
2131
-
break;
2132
-
}
2133
-
2134
-
return nr_pages;
2135
-
}
2136
-
2137
2061
static unsigned long __init
2138
2062
deferred_init_memmap_chunk(unsigned long start_pfn, unsigned long end_pfn,
2139
2063
struct zone *zone)
2140
2064
{
2065
+
int nid = zone_to_nid(zone);
2141
2066
unsigned long nr_pages = 0;
2142
-
unsigned long spfn, epfn;
2067
+
phys_addr_t start, end;
2143
2068
u64 i = 0;
2144
2069
2145
-
deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn, start_pfn);
2070
+
for_each_free_mem_range(i, nid, 0, &start, &end, NULL) {
2071
+
unsigned long spfn = PFN_UP(start);
2072
+
unsigned long epfn = PFN_DOWN(end);
2146
2073
2147
-
/*
2148
-
* Initialize and free pages in MAX_PAGE_ORDER sized increments so that
2149
-
* we can avoid introducing any issues with the buddy allocator.
2150
-
*/
2151
-
while (spfn < end_pfn) {
2152
-
nr_pages += deferred_init_maxorder(&i, zone, &spfn, &epfn);
2153
-
if (irqs_disabled())
2154
-
touch_nmi_watchdog();
2155
-
else
2156
-
cond_resched();
2074
+
if (spfn >= end_pfn)
2075
+
break;
2076
+
2077
+
spfn = max(spfn, start_pfn);
2078
+
epfn = min(epfn, end_pfn);
2079
+
2080
+
while (spfn < epfn) {
2081
+
unsigned long mo_pfn = ALIGN(spfn + 1, MAX_ORDER_NR_PAGES);
2082
+
unsigned long chunk_end = min(mo_pfn, epfn);
2083
+
2084
+
nr_pages += deferred_init_pages(zone, spfn, chunk_end);
2085
+
deferred_free_pages(spfn, chunk_end - spfn);
2086
+
2087
+
spfn = chunk_end;
2088
+
2089
+
if (irqs_disabled())
2090
+
touch_nmi_watchdog();
2091
+
else
2092
+
cond_resched();
2093
+
}
2157
2094
}
2158
2095
2159
2096
return nr_pages;