Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
mm: cma: add cma_alloc_frozen{_compound}()
Introduce cma_alloc_frozen{_compound}() helper to alloc pages without
incrementing their refcount, then convert hugetlb cma to use the
cma_alloc_frozen_compound() and cma_release_frozen() and remove the unused
cma_{alloc,free}_folio(), also move the cma_validate_zones() into
mm/internal.h since no outside user.
The set_pages_refcounted() is only called to set non-compound pages after
above changes, so remove the processing about PageHead.
Link: https://lkml.kernel.org/r/20260109093136.1491549-6-wangkefeng.wang@huawei.com
Signed-off-by: Kefeng Wang <wangkefeng.wang@huawei.com>
Reviewed-by: Zi Yan <ziy@nvidia.com>
Cc: Brendan Jackman <jackmanb@google.com>
Cc: David Hildenbrand <david@redhat.com>
Cc: Jane Chu <jane.chu@oracle.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Matthew Wilcox (Oracle) <willy@infradead.org>
Cc: Muchun Song <muchun.song@linux.dev>
Cc: Oscar Salvador <osalvador@suse.de>
Cc: Sidhartha Kumar <sidhartha.kumar@oracle.com>
Cc: Vlastimil Babka <vbabka@suse.cz>
Cc: Claudiu Beznea <claudiu.beznea.uj@bp.renesas.com>
Cc: Mark Brown <broonie@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
authored by
Kefeng Wang
and committed by
Andrew Morton
9bda131c
e0c13267
+97
-70
+6
-20
include/linux/cma.h
+6
-20
include/linux/cma.h
···
51
51
bool no_warn);
52
52
extern bool cma_release(struct cma *cma, const struct page *pages, unsigned long count);
53
53
54
+
struct page *cma_alloc_frozen(struct cma *cma, unsigned long count,
55
+
unsigned int align, bool no_warn);
56
+
struct page *cma_alloc_frozen_compound(struct cma *cma, unsigned int order);
57
+
bool cma_release_frozen(struct cma *cma, const struct page *pages,
58
+
unsigned long count);
59
+
54
60
extern int cma_for_each_area(int (*it)(struct cma *cma, void *data), void *data);
55
61
extern bool cma_intersects(struct cma *cma, unsigned long start, unsigned long end);
56
62
57
63
extern void cma_reserve_pages_on_error(struct cma *cma);
58
-
59
-
#ifdef CONFIG_CMA
60
-
struct folio *cma_alloc_folio(struct cma *cma, int order, gfp_t gfp);
61
-
bool cma_free_folio(struct cma *cma, const struct folio *folio);
62
-
bool cma_validate_zones(struct cma *cma);
63
-
#else
64
-
static inline struct folio *cma_alloc_folio(struct cma *cma, int order, gfp_t gfp)
65
-
{
66
-
return NULL;
67
-
}
68
-
69
-
static inline bool cma_free_folio(struct cma *cma, const struct folio *folio)
70
-
{
71
-
return false;
72
-
}
73
-
static inline bool cma_validate_zones(struct cma *cma)
74
-
{
75
-
return false;
76
-
}
77
-
#endif
78
64
79
65
#endif
+72
-35
mm/cma.c
+72
-35
mm/cma.c
···
856
856
return ret;
857
857
}
858
858
859
-
static struct page *__cma_alloc(struct cma *cma, unsigned long count,
860
-
unsigned int align, gfp_t gfp)
859
+
static struct page *__cma_alloc_frozen(struct cma *cma,
860
+
unsigned long count, unsigned int align, gfp_t gfp)
861
861
{
862
862
struct page *page = NULL;
863
863
int ret = -ENOMEM, r;
···
904
904
trace_cma_alloc_finish(name, page ? page_to_pfn(page) : 0,
905
905
page, count, align, ret);
906
906
if (page) {
907
-
set_pages_refcounted(page, count);
908
907
count_vm_event(CMA_ALLOC_SUCCESS);
909
908
cma_sysfs_account_success_pages(cma, count);
910
909
} else {
···
912
913
}
913
914
914
915
return page;
916
+
}
917
+
918
+
struct page *cma_alloc_frozen(struct cma *cma, unsigned long count,
919
+
unsigned int align, bool no_warn)
920
+
{
921
+
gfp_t gfp = GFP_KERNEL | (no_warn ? __GFP_NOWARN : 0);
922
+
923
+
return __cma_alloc_frozen(cma, count, align, gfp);
924
+
}
925
+
926
+
struct page *cma_alloc_frozen_compound(struct cma *cma, unsigned int order)
927
+
{
928
+
gfp_t gfp = GFP_KERNEL | __GFP_COMP | __GFP_NOWARN;
929
+
930
+
return __cma_alloc_frozen(cma, 1 << order, order, gfp);
915
931
}
916
932
917
933
/**
···
942
928
struct page *cma_alloc(struct cma *cma, unsigned long count,
943
929
unsigned int align, bool no_warn)
944
930
{
945
-
return __cma_alloc(cma, count, align, GFP_KERNEL | (no_warn ? __GFP_NOWARN : 0));
946
-
}
947
-
948
-
struct folio *cma_alloc_folio(struct cma *cma, int order, gfp_t gfp)
949
-
{
950
931
struct page *page;
951
932
952
-
if (WARN_ON(!order || !(gfp & __GFP_COMP)))
953
-
return NULL;
933
+
page = cma_alloc_frozen(cma, count, align, no_warn);
934
+
if (page)
935
+
set_pages_refcounted(page, count);
954
936
955
-
page = __cma_alloc(cma, 1 << order, order, gfp);
956
-
957
-
return page ? page_folio(page) : NULL;
937
+
return page;
958
938
}
959
939
960
-
/**
961
-
* cma_release() - release allocated pages
962
-
* @cma: Contiguous memory region for which the allocation is performed.
963
-
* @pages: Allocated pages.
964
-
* @count: Number of allocated pages.
965
-
*
966
-
* This function releases memory allocated by cma_alloc().
967
-
* It returns false when provided pages do not belong to contiguous area and
968
-
* true otherwise.
969
-
*/
970
-
bool cma_release(struct cma *cma, const struct page *pages,
971
-
unsigned long count)
940
+
static struct cma_memrange *find_cma_memrange(struct cma *cma,
941
+
const struct page *pages, unsigned long count)
972
942
{
973
-
struct cma_memrange *cmr;
943
+
struct cma_memrange *cmr = NULL;
974
944
unsigned long pfn, end_pfn;
975
945
int r;
976
946
977
947
pr_debug("%s(page %p, count %lu)\n", __func__, (void *)pages, count);
978
948
979
949
if (!cma || !pages || count > cma->count)
980
-
return false;
950
+
return NULL;
981
951
982
952
pfn = page_to_pfn(pages);
983
953
···
979
981
if (r == cma->nranges) {
980
982
pr_debug("%s(page %p, count %lu, no cma range matches the page range)\n",
981
983
__func__, (void *)pages, count);
982
-
return false;
984
+
return NULL;
983
985
}
984
986
985
-
if (PageHead(pages))
986
-
__free_pages((struct page *)pages, compound_order(pages));
987
-
else
988
-
free_contig_range(pfn, count);
987
+
return cmr;
988
+
}
989
989
990
+
static void __cma_release_frozen(struct cma *cma, struct cma_memrange *cmr,
991
+
const struct page *pages, unsigned long count)
992
+
{
993
+
unsigned long pfn = page_to_pfn(pages);
994
+
995
+
pr_debug("%s(page %p, count %lu)\n", __func__, (void *)pages, count);
996
+
997
+
free_contig_frozen_range(pfn, count);
990
998
cma_clear_bitmap(cma, cmr, pfn, count);
991
999
cma_sysfs_account_release_pages(cma, count);
992
1000
trace_cma_release(cma->name, pfn, pages, count);
1001
+
}
1002
+
1003
+
/**
1004
+
* cma_release() - release allocated pages
1005
+
* @cma: Contiguous memory region for which the allocation is performed.
1006
+
* @pages: Allocated pages.
1007
+
* @count: Number of allocated pages.
1008
+
*
1009
+
* This function releases memory allocated by cma_alloc().
1010
+
* It returns false when provided pages do not belong to contiguous area and
1011
+
* true otherwise.
1012
+
*/
1013
+
bool cma_release(struct cma *cma, const struct page *pages,
1014
+
unsigned long count)
1015
+
{
1016
+
struct cma_memrange *cmr;
1017
+
unsigned long i, pfn;
1018
+
1019
+
cmr = find_cma_memrange(cma, pages, count);
1020
+
if (!cmr)
1021
+
return false;
1022
+
1023
+
pfn = page_to_pfn(pages);
1024
+
for (i = 0; i < count; i++, pfn++)
1025
+
VM_WARN_ON(!put_page_testzero(pfn_to_page(pfn)));
1026
+
1027
+
__cma_release_frozen(cma, cmr, pages, count);
993
1028
994
1029
return true;
995
1030
}
996
1031
997
-
bool cma_free_folio(struct cma *cma, const struct folio *folio)
1032
+
bool cma_release_frozen(struct cma *cma, const struct page *pages,
1033
+
unsigned long count)
998
1034
{
999
-
if (WARN_ON(!folio_test_large(folio)))
1035
+
struct cma_memrange *cmr;
1036
+
1037
+
cmr = find_cma_memrange(cma, pages, count);
1038
+
if (!cmr)
1000
1039
return false;
1001
1040
1002
-
return cma_release(cma, &folio->page, folio_nr_pages(folio));
1041
+
__cma_release_frozen(cma, cmr, pages, count);
1042
+
1043
+
return true;
1003
1044
}
1004
1045
1005
1046
int cma_for_each_area(int (*it)(struct cma *cma, void *data), void *data)
+14
-10
mm/hugetlb_cma.c
+14
-10
mm/hugetlb_cma.c
···
20
20
21
21
void hugetlb_cma_free_folio(struct folio *folio)
22
22
{
23
-
int nid = folio_nid(folio);
23
+
folio_ref_dec(folio);
24
24
25
-
WARN_ON_ONCE(!cma_free_folio(hugetlb_cma[nid], folio));
25
+
WARN_ON_ONCE(!cma_release_frozen(hugetlb_cma[folio_nid(folio)],
26
+
&folio->page, folio_nr_pages(folio)));
26
27
}
27
-
28
28
29
29
struct folio *hugetlb_cma_alloc_folio(int order, gfp_t gfp_mask,
30
30
int nid, nodemask_t *nodemask)
31
31
{
32
32
int node;
33
-
struct folio *folio = NULL;
33
+
struct folio *folio;
34
+
struct page *page = NULL;
34
35
35
36
if (hugetlb_cma[nid])
36
-
folio = cma_alloc_folio(hugetlb_cma[nid], order, gfp_mask);
37
+
page = cma_alloc_frozen_compound(hugetlb_cma[nid], order);
37
38
38
-
if (!folio && !(gfp_mask & __GFP_THISNODE)) {
39
+
if (!page && !(gfp_mask & __GFP_THISNODE)) {
39
40
for_each_node_mask(node, *nodemask) {
40
41
if (node == nid || !hugetlb_cma[node])
41
42
continue;
42
43
43
-
folio = cma_alloc_folio(hugetlb_cma[node], order, gfp_mask);
44
-
if (folio)
44
+
page = cma_alloc_frozen_compound(hugetlb_cma[node], order);
45
+
if (page)
45
46
break;
46
47
}
47
48
}
48
49
49
-
if (folio)
50
-
folio_set_hugetlb_cma(folio);
50
+
if (!page)
51
+
return NULL;
51
52
53
+
set_page_refcounted(page);
54
+
folio = page_folio(page);
55
+
folio_set_hugetlb_cma(folio);
52
56
return folio;
53
57
}
54
58
+5
-5
mm/internal.h
+5
-5
mm/internal.h
···
584
584
{
585
585
unsigned long pfn = page_to_pfn(page);
586
586
587
-
if (PageHead(page)) {
588
-
set_page_refcounted(page);
589
-
return;
590
-
}
591
-
592
587
for (; nr_pages--; pfn++)
593
588
set_page_refcounted(pfn_to_page(pfn));
594
589
}
···
1009
1014
struct cma;
1010
1015
1011
1016
#ifdef CONFIG_CMA
1017
+
bool cma_validate_zones(struct cma *cma);
1012
1018
void *cma_reserve_early(struct cma *cma, unsigned long size);
1013
1019
void init_cma_pageblock(struct page *page);
1014
1020
#else
1021
+
static inline bool cma_validate_zones(struct cma *cma)
1022
+
{
1023
+
return false;
1024
+
}
1015
1025
static inline void *cma_reserve_early(struct cma *cma, unsigned long size)
1016
1026
{
1017
1027
return NULL;