Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
mm/sparse: move memory hotplug bits to sparse-vmemmap.c
Let's move all memory hoptplug related code to sparse-vmemmap.c.
We only have to expose sparse_index_init(). While at it, drop the
definition of sparse_index_init() for !CONFIG_SPARSEMEM, which is unused,
and place the declaration in internal.h.
Link: https://lkml.kernel.org/r/20260320-sparsemem_cleanups-v2-15-096addc8800d@kernel.org
Signed-off-by: David Hildenbrand (Arm) <david@kernel.org>
Reviewed-by: Lorenzo Stoakes (Oracle) <ljs@kernel.org>
Reviewed-by: Mike Rapoport (Microsoft) <rppt@kernel.org>
Cc: Axel Rasmussen <axelrasmussen@google.com>
Cc: Liam Howlett <liam.howlett@oracle.com>
Cc: Michal Hocko <mhocko@suse.com>
Cc: Oscar Salvador <osalvador@suse.de>
Cc: Sidhartha Kumar <sidhartha.kumar@oracle.com>
Cc: Suren Baghdasaryan <surenb@google.com>
Cc: Vlastimil Babka <vbabka@kernel.org>
Cc: Wei Xu <weixugc@google.com>
Cc: Yuanchu Xie <yuanchu@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
authored by
David Hildenbrand (Arm)
and committed by
Andrew Morton
738de20c
08e5f77c
+310
-309
-1
include/linux/mmzone.h
-1
include/linux/mmzone.h
+4
mm/internal.h
+4
mm/internal.h
···
964
964
*/
965
965
#ifdef CONFIG_SPARSEMEM
966
966
void sparse_init(void);
967
+
int sparse_index_init(unsigned long section_nr, int nid);
967
968
968
969
static inline void sparse_init_one_section(struct mem_section *ms,
969
970
unsigned long pnum, struct page *mem_map,
···
1000
999
static inline void sparse_init(void) {}
1001
1000
#endif /* CONFIG_SPARSEMEM */
1002
1001
1002
+
/*
1003
+
* mm/sparse-vmemmap.c
1004
+
*/
1003
1005
#ifdef CONFIG_SPARSEMEM_VMEMMAP
1004
1006
void sparse_init_subsection_map(unsigned long pfn, unsigned long nr_pages);
1005
1007
#else
+304
mm/sparse-vmemmap.c
+304
mm/sparse-vmemmap.c
···
591
591
hugetlb_vmemmap_init_late(nid);
592
592
}
593
593
#endif
594
+
595
+
static void subsection_mask_set(unsigned long *map, unsigned long pfn,
596
+
unsigned long nr_pages)
597
+
{
598
+
int idx = subsection_map_index(pfn);
599
+
int end = subsection_map_index(pfn + nr_pages - 1);
600
+
601
+
bitmap_set(map, idx, end - idx + 1);
602
+
}
603
+
604
+
void __init sparse_init_subsection_map(unsigned long pfn, unsigned long nr_pages)
605
+
{
606
+
int end_sec_nr = pfn_to_section_nr(pfn + nr_pages - 1);
607
+
unsigned long nr, start_sec_nr = pfn_to_section_nr(pfn);
608
+
609
+
for (nr = start_sec_nr; nr <= end_sec_nr; nr++) {
610
+
struct mem_section *ms;
611
+
unsigned long pfns;
612
+
613
+
pfns = min(nr_pages, PAGES_PER_SECTION
614
+
- (pfn & ~PAGE_SECTION_MASK));
615
+
ms = __nr_to_section(nr);
616
+
subsection_mask_set(ms->usage->subsection_map, pfn, pfns);
617
+
618
+
pr_debug("%s: sec: %lu pfns: %lu set(%d, %d)\n", __func__, nr,
619
+
pfns, subsection_map_index(pfn),
620
+
subsection_map_index(pfn + pfns - 1));
621
+
622
+
pfn += pfns;
623
+
nr_pages -= pfns;
624
+
}
625
+
}
626
+
627
+
#ifdef CONFIG_MEMORY_HOTPLUG
628
+
629
+
/* Mark all memory sections within the pfn range as online */
630
+
void online_mem_sections(unsigned long start_pfn, unsigned long end_pfn)
631
+
{
632
+
unsigned long pfn;
633
+
634
+
for (pfn = start_pfn; pfn < end_pfn; pfn += PAGES_PER_SECTION) {
635
+
unsigned long section_nr = pfn_to_section_nr(pfn);
636
+
struct mem_section *ms = __nr_to_section(section_nr);
637
+
638
+
ms->section_mem_map |= SECTION_IS_ONLINE;
639
+
}
640
+
}
641
+
642
+
/* Mark all memory sections within the pfn range as offline */
643
+
void offline_mem_sections(unsigned long start_pfn, unsigned long end_pfn)
644
+
{
645
+
unsigned long pfn;
646
+
647
+
for (pfn = start_pfn; pfn < end_pfn; pfn += PAGES_PER_SECTION) {
648
+
unsigned long section_nr = pfn_to_section_nr(pfn);
649
+
struct mem_section *ms = __nr_to_section(section_nr);
650
+
651
+
ms->section_mem_map &= ~SECTION_IS_ONLINE;
652
+
}
653
+
}
654
+
655
+
static struct page * __meminit populate_section_memmap(unsigned long pfn,
656
+
unsigned long nr_pages, int nid, struct vmem_altmap *altmap,
657
+
struct dev_pagemap *pgmap)
658
+
{
659
+
return __populate_section_memmap(pfn, nr_pages, nid, altmap, pgmap);
660
+
}
661
+
662
+
static void depopulate_section_memmap(unsigned long pfn, unsigned long nr_pages,
663
+
struct vmem_altmap *altmap)
664
+
{
665
+
unsigned long start = (unsigned long) pfn_to_page(pfn);
666
+
unsigned long end = start + nr_pages * sizeof(struct page);
667
+
668
+
vmemmap_free(start, end, altmap);
669
+
}
670
+
static void free_map_bootmem(struct page *memmap)
671
+
{
672
+
unsigned long start = (unsigned long)memmap;
673
+
unsigned long end = (unsigned long)(memmap + PAGES_PER_SECTION);
674
+
675
+
vmemmap_free(start, end, NULL);
676
+
}
677
+
678
+
static int clear_subsection_map(unsigned long pfn, unsigned long nr_pages)
679
+
{
680
+
DECLARE_BITMAP(map, SUBSECTIONS_PER_SECTION) = { 0 };
681
+
DECLARE_BITMAP(tmp, SUBSECTIONS_PER_SECTION) = { 0 };
682
+
struct mem_section *ms = __pfn_to_section(pfn);
683
+
unsigned long *subsection_map = ms->usage
684
+
? &ms->usage->subsection_map[0] : NULL;
685
+
686
+
subsection_mask_set(map, pfn, nr_pages);
687
+
if (subsection_map)
688
+
bitmap_and(tmp, map, subsection_map, SUBSECTIONS_PER_SECTION);
689
+
690
+
if (WARN(!subsection_map || !bitmap_equal(tmp, map, SUBSECTIONS_PER_SECTION),
691
+
"section already deactivated (%#lx + %ld)\n",
692
+
pfn, nr_pages))
693
+
return -EINVAL;
694
+
695
+
bitmap_xor(subsection_map, map, subsection_map, SUBSECTIONS_PER_SECTION);
696
+
return 0;
697
+
}
698
+
699
+
static bool is_subsection_map_empty(struct mem_section *ms)
700
+
{
701
+
return bitmap_empty(&ms->usage->subsection_map[0],
702
+
SUBSECTIONS_PER_SECTION);
703
+
}
704
+
705
+
static int fill_subsection_map(unsigned long pfn, unsigned long nr_pages)
706
+
{
707
+
struct mem_section *ms = __pfn_to_section(pfn);
708
+
DECLARE_BITMAP(map, SUBSECTIONS_PER_SECTION) = { 0 };
709
+
unsigned long *subsection_map;
710
+
int rc = 0;
711
+
712
+
subsection_mask_set(map, pfn, nr_pages);
713
+
714
+
subsection_map = &ms->usage->subsection_map[0];
715
+
716
+
if (bitmap_empty(map, SUBSECTIONS_PER_SECTION))
717
+
rc = -EINVAL;
718
+
else if (bitmap_intersects(map, subsection_map, SUBSECTIONS_PER_SECTION))
719
+
rc = -EEXIST;
720
+
else
721
+
bitmap_or(subsection_map, map, subsection_map,
722
+
SUBSECTIONS_PER_SECTION);
723
+
724
+
return rc;
725
+
}
726
+
727
+
/*
728
+
* To deactivate a memory region, there are 3 cases to handle:
729
+
*
730
+
* 1. deactivation of a partial hot-added section:
731
+
* a) section was present at memory init.
732
+
* b) section was hot-added post memory init.
733
+
* 2. deactivation of a complete hot-added section.
734
+
* 3. deactivation of a complete section from memory init.
735
+
*
736
+
* For 1, when subsection_map does not empty we will not be freeing the
737
+
* usage map, but still need to free the vmemmap range.
738
+
*/
739
+
static void section_deactivate(unsigned long pfn, unsigned long nr_pages,
740
+
struct vmem_altmap *altmap)
741
+
{
742
+
struct mem_section *ms = __pfn_to_section(pfn);
743
+
bool section_is_early = early_section(ms);
744
+
struct page *memmap = NULL;
745
+
bool empty;
746
+
747
+
if (clear_subsection_map(pfn, nr_pages))
748
+
return;
749
+
750
+
empty = is_subsection_map_empty(ms);
751
+
if (empty) {
752
+
/*
753
+
* Mark the section invalid so that valid_section()
754
+
* return false. This prevents code from dereferencing
755
+
* ms->usage array.
756
+
*/
757
+
ms->section_mem_map &= ~SECTION_HAS_MEM_MAP;
758
+
759
+
/*
760
+
* When removing an early section, the usage map is kept (as the
761
+
* usage maps of other sections fall into the same page). It
762
+
* will be re-used when re-adding the section - which is then no
763
+
* longer an early section. If the usage map is PageReserved, it
764
+
* was allocated during boot.
765
+
*/
766
+
if (!PageReserved(virt_to_page(ms->usage))) {
767
+
kfree_rcu(ms->usage, rcu);
768
+
WRITE_ONCE(ms->usage, NULL);
769
+
}
770
+
memmap = pfn_to_page(SECTION_ALIGN_DOWN(pfn));
771
+
}
772
+
773
+
/*
774
+
* The memmap of early sections is always fully populated. See
775
+
* section_activate() and pfn_valid() .
776
+
*/
777
+
if (!section_is_early) {
778
+
memmap_pages_add(-1L * (DIV_ROUND_UP(nr_pages * sizeof(struct page), PAGE_SIZE)));
779
+
depopulate_section_memmap(pfn, nr_pages, altmap);
780
+
} else if (memmap) {
781
+
memmap_boot_pages_add(-1L * (DIV_ROUND_UP(nr_pages * sizeof(struct page),
782
+
PAGE_SIZE)));
783
+
free_map_bootmem(memmap);
784
+
}
785
+
786
+
if (empty)
787
+
ms->section_mem_map = (unsigned long)NULL;
788
+
}
789
+
790
+
static struct page * __meminit section_activate(int nid, unsigned long pfn,
791
+
unsigned long nr_pages, struct vmem_altmap *altmap,
792
+
struct dev_pagemap *pgmap)
793
+
{
794
+
struct mem_section *ms = __pfn_to_section(pfn);
795
+
struct mem_section_usage *usage = NULL;
796
+
struct page *memmap;
797
+
int rc;
798
+
799
+
if (!ms->usage) {
800
+
usage = kzalloc(mem_section_usage_size(), GFP_KERNEL);
801
+
if (!usage)
802
+
return ERR_PTR(-ENOMEM);
803
+
ms->usage = usage;
804
+
}
805
+
806
+
rc = fill_subsection_map(pfn, nr_pages);
807
+
if (rc) {
808
+
if (usage)
809
+
ms->usage = NULL;
810
+
kfree(usage);
811
+
return ERR_PTR(rc);
812
+
}
813
+
814
+
/*
815
+
* The early init code does not consider partially populated
816
+
* initial sections, it simply assumes that memory will never be
817
+
* referenced. If we hot-add memory into such a section then we
818
+
* do not need to populate the memmap and can simply reuse what
819
+
* is already there.
820
+
*/
821
+
if (nr_pages < PAGES_PER_SECTION && early_section(ms))
822
+
return pfn_to_page(pfn);
823
+
824
+
memmap = populate_section_memmap(pfn, nr_pages, nid, altmap, pgmap);
825
+
if (!memmap) {
826
+
section_deactivate(pfn, nr_pages, altmap);
827
+
return ERR_PTR(-ENOMEM);
828
+
}
829
+
memmap_pages_add(DIV_ROUND_UP(nr_pages * sizeof(struct page), PAGE_SIZE));
830
+
831
+
return memmap;
832
+
}
833
+
834
+
/**
835
+
* sparse_add_section - add a memory section, or populate an existing one
836
+
* @nid: The node to add section on
837
+
* @start_pfn: start pfn of the memory range
838
+
* @nr_pages: number of pfns to add in the section
839
+
* @altmap: alternate pfns to allocate the memmap backing store
840
+
* @pgmap: alternate compound page geometry for devmap mappings
841
+
*
842
+
* This is only intended for hotplug.
843
+
*
844
+
* Note that only VMEMMAP supports sub-section aligned hotplug,
845
+
* the proper alignment and size are gated by check_pfn_span().
846
+
*
847
+
*
848
+
* Return:
849
+
* * 0 - On success.
850
+
* * -EEXIST - Section has been present.
851
+
* * -ENOMEM - Out of memory.
852
+
*/
853
+
int __meminit sparse_add_section(int nid, unsigned long start_pfn,
854
+
unsigned long nr_pages, struct vmem_altmap *altmap,
855
+
struct dev_pagemap *pgmap)
856
+
{
857
+
unsigned long section_nr = pfn_to_section_nr(start_pfn);
858
+
struct mem_section *ms;
859
+
struct page *memmap;
860
+
int ret;
861
+
862
+
ret = sparse_index_init(section_nr, nid);
863
+
if (ret < 0)
864
+
return ret;
865
+
866
+
memmap = section_activate(nid, start_pfn, nr_pages, altmap, pgmap);
867
+
if (IS_ERR(memmap))
868
+
return PTR_ERR(memmap);
869
+
870
+
/*
871
+
* Poison uninitialized struct pages in order to catch invalid flags
872
+
* combinations.
873
+
*/
874
+
page_init_poison(memmap, sizeof(struct page) * nr_pages);
875
+
876
+
ms = __nr_to_section(section_nr);
877
+
__section_mark_present(ms, section_nr);
878
+
879
+
/* Align memmap to section boundary in the subsection case */
880
+
if (section_nr_to_pfn(section_nr) != start_pfn)
881
+
memmap = pfn_to_page(section_nr_to_pfn(section_nr));
882
+
sparse_init_one_section(ms, section_nr, memmap, ms->usage, 0);
883
+
884
+
return 0;
885
+
}
886
+
887
+
void sparse_remove_section(unsigned long pfn, unsigned long nr_pages,
888
+
struct vmem_altmap *altmap)
889
+
{
890
+
struct mem_section *ms = __pfn_to_section(pfn);
891
+
892
+
if (WARN_ON_ONCE(!valid_section(ms)))
893
+
return;
894
+
895
+
section_deactivate(pfn, nr_pages, altmap);
896
+
}
897
+
#endif /* CONFIG_MEMORY_HOTPLUG */
+2
-308
mm/sparse.c
+2
-308
mm/sparse.c
···
79
79
return section;
80
80
}
81
81
82
-
static int __meminit sparse_index_init(unsigned long section_nr, int nid)
82
+
int __meminit sparse_index_init(unsigned long section_nr, int nid)
83
83
{
84
84
unsigned long root = SECTION_NR_TO_ROOT(section_nr);
85
85
struct mem_section *section;
···
103
103
return 0;
104
104
}
105
105
#else /* !SPARSEMEM_EXTREME */
106
-
static inline int sparse_index_init(unsigned long section_nr, int nid)
106
+
int sparse_index_init(unsigned long section_nr, int nid)
107
107
{
108
108
return 0;
109
109
}
···
166
166
{
167
167
return next_present_section_nr(-1);
168
168
}
169
-
170
-
#ifdef CONFIG_SPARSEMEM_VMEMMAP
171
-
static void subsection_mask_set(unsigned long *map, unsigned long pfn,
172
-
unsigned long nr_pages)
173
-
{
174
-
int idx = subsection_map_index(pfn);
175
-
int end = subsection_map_index(pfn + nr_pages - 1);
176
-
177
-
bitmap_set(map, idx, end - idx + 1);
178
-
}
179
-
180
-
void __init sparse_init_subsection_map(unsigned long pfn, unsigned long nr_pages)
181
-
{
182
-
int end_sec_nr = pfn_to_section_nr(pfn + nr_pages - 1);
183
-
unsigned long nr, start_sec_nr = pfn_to_section_nr(pfn);
184
-
185
-
for (nr = start_sec_nr; nr <= end_sec_nr; nr++) {
186
-
struct mem_section *ms;
187
-
unsigned long pfns;
188
-
189
-
pfns = min(nr_pages, PAGES_PER_SECTION
190
-
- (pfn & ~PAGE_SECTION_MASK));
191
-
ms = __nr_to_section(nr);
192
-
subsection_mask_set(ms->usage->subsection_map, pfn, pfns);
193
-
194
-
pr_debug("%s: sec: %lu pfns: %lu set(%d, %d)\n", __func__, nr,
195
-
pfns, subsection_map_index(pfn),
196
-
subsection_map_index(pfn + pfns - 1));
197
-
198
-
pfn += pfns;
199
-
nr_pages -= pfns;
200
-
}
201
-
}
202
-
#endif
203
169
204
170
/* Record a memory area against a node. */
205
171
static void __init memory_present(int nid, unsigned long start, unsigned long end)
···
448
482
sparse_init_nid(nid_begin, pnum_begin, pnum_end, map_count);
449
483
vmemmap_populate_print_last();
450
484
}
451
-
452
-
#ifdef CONFIG_MEMORY_HOTPLUG
453
-
454
-
/* Mark all memory sections within the pfn range as online */
455
-
void online_mem_sections(unsigned long start_pfn, unsigned long end_pfn)
456
-
{
457
-
unsigned long pfn;
458
-
459
-
for (pfn = start_pfn; pfn < end_pfn; pfn += PAGES_PER_SECTION) {
460
-
unsigned long section_nr = pfn_to_section_nr(pfn);
461
-
struct mem_section *ms = __nr_to_section(section_nr);
462
-
463
-
ms->section_mem_map |= SECTION_IS_ONLINE;
464
-
}
465
-
}
466
-
467
-
/* Mark all memory sections within the pfn range as offline */
468
-
void offline_mem_sections(unsigned long start_pfn, unsigned long end_pfn)
469
-
{
470
-
unsigned long pfn;
471
-
472
-
for (pfn = start_pfn; pfn < end_pfn; pfn += PAGES_PER_SECTION) {
473
-
unsigned long section_nr = pfn_to_section_nr(pfn);
474
-
struct mem_section *ms = __nr_to_section(section_nr);
475
-
476
-
ms->section_mem_map &= ~SECTION_IS_ONLINE;
477
-
}
478
-
}
479
-
480
-
static struct page * __meminit populate_section_memmap(unsigned long pfn,
481
-
unsigned long nr_pages, int nid, struct vmem_altmap *altmap,
482
-
struct dev_pagemap *pgmap)
483
-
{
484
-
return __populate_section_memmap(pfn, nr_pages, nid, altmap, pgmap);
485
-
}
486
-
487
-
static void depopulate_section_memmap(unsigned long pfn, unsigned long nr_pages,
488
-
struct vmem_altmap *altmap)
489
-
{
490
-
unsigned long start = (unsigned long) pfn_to_page(pfn);
491
-
unsigned long end = start + nr_pages * sizeof(struct page);
492
-
493
-
vmemmap_free(start, end, altmap);
494
-
}
495
-
static void free_map_bootmem(struct page *memmap)
496
-
{
497
-
unsigned long start = (unsigned long)memmap;
498
-
unsigned long end = (unsigned long)(memmap + PAGES_PER_SECTION);
499
-
500
-
vmemmap_free(start, end, NULL);
501
-
}
502
-
503
-
static int clear_subsection_map(unsigned long pfn, unsigned long nr_pages)
504
-
{
505
-
DECLARE_BITMAP(map, SUBSECTIONS_PER_SECTION) = { 0 };
506
-
DECLARE_BITMAP(tmp, SUBSECTIONS_PER_SECTION) = { 0 };
507
-
struct mem_section *ms = __pfn_to_section(pfn);
508
-
unsigned long *subsection_map = ms->usage
509
-
? &ms->usage->subsection_map[0] : NULL;
510
-
511
-
subsection_mask_set(map, pfn, nr_pages);
512
-
if (subsection_map)
513
-
bitmap_and(tmp, map, subsection_map, SUBSECTIONS_PER_SECTION);
514
-
515
-
if (WARN(!subsection_map || !bitmap_equal(tmp, map, SUBSECTIONS_PER_SECTION),
516
-
"section already deactivated (%#lx + %ld)\n",
517
-
pfn, nr_pages))
518
-
return -EINVAL;
519
-
520
-
bitmap_xor(subsection_map, map, subsection_map, SUBSECTIONS_PER_SECTION);
521
-
return 0;
522
-
}
523
-
524
-
static bool is_subsection_map_empty(struct mem_section *ms)
525
-
{
526
-
return bitmap_empty(&ms->usage->subsection_map[0],
527
-
SUBSECTIONS_PER_SECTION);
528
-
}
529
-
530
-
static int fill_subsection_map(unsigned long pfn, unsigned long nr_pages)
531
-
{
532
-
struct mem_section *ms = __pfn_to_section(pfn);
533
-
DECLARE_BITMAP(map, SUBSECTIONS_PER_SECTION) = { 0 };
534
-
unsigned long *subsection_map;
535
-
int rc = 0;
536
-
537
-
subsection_mask_set(map, pfn, nr_pages);
538
-
539
-
subsection_map = &ms->usage->subsection_map[0];
540
-
541
-
if (bitmap_empty(map, SUBSECTIONS_PER_SECTION))
542
-
rc = -EINVAL;
543
-
else if (bitmap_intersects(map, subsection_map, SUBSECTIONS_PER_SECTION))
544
-
rc = -EEXIST;
545
-
else
546
-
bitmap_or(subsection_map, map, subsection_map,
547
-
SUBSECTIONS_PER_SECTION);
548
-
549
-
return rc;
550
-
}
551
-
552
-
/*
553
-
* To deactivate a memory region, there are 3 cases to handle:
554
-
*
555
-
* 1. deactivation of a partial hot-added section:
556
-
* a) section was present at memory init.
557
-
* b) section was hot-added post memory init.
558
-
* 2. deactivation of a complete hot-added section.
559
-
* 3. deactivation of a complete section from memory init.
560
-
*
561
-
* For 1, when subsection_map does not empty we will not be freeing the
562
-
* usage map, but still need to free the vmemmap range.
563
-
*/
564
-
static void section_deactivate(unsigned long pfn, unsigned long nr_pages,
565
-
struct vmem_altmap *altmap)
566
-
{
567
-
struct mem_section *ms = __pfn_to_section(pfn);
568
-
bool section_is_early = early_section(ms);
569
-
struct page *memmap = NULL;
570
-
bool empty;
571
-
572
-
if (clear_subsection_map(pfn, nr_pages))
573
-
return;
574
-
575
-
empty = is_subsection_map_empty(ms);
576
-
if (empty) {
577
-
/*
578
-
* Mark the section invalid so that valid_section()
579
-
* return false. This prevents code from dereferencing
580
-
* ms->usage array.
581
-
*/
582
-
ms->section_mem_map &= ~SECTION_HAS_MEM_MAP;
583
-
584
-
/*
585
-
* When removing an early section, the usage map is kept (as the
586
-
* usage maps of other sections fall into the same page). It
587
-
* will be re-used when re-adding the section - which is then no
588
-
* longer an early section. If the usage map is PageReserved, it
589
-
* was allocated during boot.
590
-
*/
591
-
if (!PageReserved(virt_to_page(ms->usage))) {
592
-
kfree_rcu(ms->usage, rcu);
593
-
WRITE_ONCE(ms->usage, NULL);
594
-
}
595
-
memmap = pfn_to_page(SECTION_ALIGN_DOWN(pfn));
596
-
}
597
-
598
-
/*
599
-
* The memmap of early sections is always fully populated. See
600
-
* section_activate() and pfn_valid() .
601
-
*/
602
-
if (!section_is_early) {
603
-
memmap_pages_add(-1L * (DIV_ROUND_UP(nr_pages * sizeof(struct page), PAGE_SIZE)));
604
-
depopulate_section_memmap(pfn, nr_pages, altmap);
605
-
} else if (memmap) {
606
-
memmap_boot_pages_add(-1L * (DIV_ROUND_UP(nr_pages * sizeof(struct page),
607
-
PAGE_SIZE)));
608
-
free_map_bootmem(memmap);
609
-
}
610
-
611
-
if (empty)
612
-
ms->section_mem_map = (unsigned long)NULL;
613
-
}
614
-
615
-
static struct page * __meminit section_activate(int nid, unsigned long pfn,
616
-
unsigned long nr_pages, struct vmem_altmap *altmap,
617
-
struct dev_pagemap *pgmap)
618
-
{
619
-
struct mem_section *ms = __pfn_to_section(pfn);
620
-
struct mem_section_usage *usage = NULL;
621
-
struct page *memmap;
622
-
int rc;
623
-
624
-
if (!ms->usage) {
625
-
usage = kzalloc(mem_section_usage_size(), GFP_KERNEL);
626
-
if (!usage)
627
-
return ERR_PTR(-ENOMEM);
628
-
ms->usage = usage;
629
-
}
630
-
631
-
rc = fill_subsection_map(pfn, nr_pages);
632
-
if (rc) {
633
-
if (usage)
634
-
ms->usage = NULL;
635
-
kfree(usage);
636
-
return ERR_PTR(rc);
637
-
}
638
-
639
-
/*
640
-
* The early init code does not consider partially populated
641
-
* initial sections, it simply assumes that memory will never be
642
-
* referenced. If we hot-add memory into such a section then we
643
-
* do not need to populate the memmap and can simply reuse what
644
-
* is already there.
645
-
*/
646
-
if (nr_pages < PAGES_PER_SECTION && early_section(ms))
647
-
return pfn_to_page(pfn);
648
-
649
-
memmap = populate_section_memmap(pfn, nr_pages, nid, altmap, pgmap);
650
-
if (!memmap) {
651
-
section_deactivate(pfn, nr_pages, altmap);
652
-
return ERR_PTR(-ENOMEM);
653
-
}
654
-
memmap_pages_add(DIV_ROUND_UP(nr_pages * sizeof(struct page), PAGE_SIZE));
655
-
656
-
return memmap;
657
-
}
658
-
659
-
/**
660
-
* sparse_add_section - add a memory section, or populate an existing one
661
-
* @nid: The node to add section on
662
-
* @start_pfn: start pfn of the memory range
663
-
* @nr_pages: number of pfns to add in the section
664
-
* @altmap: alternate pfns to allocate the memmap backing store
665
-
* @pgmap: alternate compound page geometry for devmap mappings
666
-
*
667
-
* This is only intended for hotplug.
668
-
*
669
-
* Note that only VMEMMAP supports sub-section aligned hotplug,
670
-
* the proper alignment and size are gated by check_pfn_span().
671
-
*
672
-
*
673
-
* Return:
674
-
* * 0 - On success.
675
-
* * -EEXIST - Section has been present.
676
-
* * -ENOMEM - Out of memory.
677
-
*/
678
-
int __meminit sparse_add_section(int nid, unsigned long start_pfn,
679
-
unsigned long nr_pages, struct vmem_altmap *altmap,
680
-
struct dev_pagemap *pgmap)
681
-
{
682
-
unsigned long section_nr = pfn_to_section_nr(start_pfn);
683
-
struct mem_section *ms;
684
-
struct page *memmap;
685
-
int ret;
686
-
687
-
ret = sparse_index_init(section_nr, nid);
688
-
if (ret < 0)
689
-
return ret;
690
-
691
-
memmap = section_activate(nid, start_pfn, nr_pages, altmap, pgmap);
692
-
if (IS_ERR(memmap))
693
-
return PTR_ERR(memmap);
694
-
695
-
/*
696
-
* Poison uninitialized struct pages in order to catch invalid flags
697
-
* combinations.
698
-
*/
699
-
page_init_poison(memmap, sizeof(struct page) * nr_pages);
700
-
701
-
ms = __nr_to_section(section_nr);
702
-
__section_mark_present(ms, section_nr);
703
-
704
-
/* Align memmap to section boundary in the subsection case */
705
-
if (section_nr_to_pfn(section_nr) != start_pfn)
706
-
memmap = pfn_to_page(section_nr_to_pfn(section_nr));
707
-
sparse_init_one_section(ms, section_nr, memmap, ms->usage, 0);
708
-
709
-
return 0;
710
-
}
711
-
712
-
void sparse_remove_section(unsigned long pfn, unsigned long nr_pages,
713
-
struct vmem_altmap *altmap)
714
-
{
715
-
struct mem_section *ms = __pfn_to_section(pfn);
716
-
717
-
if (WARN_ON_ONCE(!valid_section(ms)))
718
-
return;
719
-
720
-
section_deactivate(pfn, nr_pages, altmap);
721
-
}
722
-
#endif /* CONFIG_MEMORY_HOTPLUG */