Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1/* SPDX-License-Identifier: GPL-2.0 */
2#ifndef _LINUX_MM_H
3#define _LINUX_MM_H
4
5#include <linux/args.h>
6#include <linux/errno.h>
7#include <linux/mmdebug.h>
8#include <linux/gfp.h>
9#include <linux/pgalloc_tag.h>
10#include <linux/bug.h>
11#include <linux/list.h>
12#include <linux/mmzone.h>
13#include <linux/rbtree.h>
14#include <linux/atomic.h>
15#include <linux/debug_locks.h>
16#include <linux/compiler.h>
17#include <linux/mm_types.h>
18#include <linux/mmap_lock.h>
19#include <linux/range.h>
20#include <linux/pfn.h>
21#include <linux/percpu-refcount.h>
22#include <linux/bit_spinlock.h>
23#include <linux/shrinker.h>
24#include <linux/resource.h>
25#include <linux/page_ext.h>
26#include <linux/err.h>
27#include <linux/page-flags.h>
28#include <linux/page_ref.h>
29#include <linux/overflow.h>
30#include <linux/sizes.h>
31#include <linux/sched.h>
32#include <linux/pgtable.h>
33#include <linux/kasan.h>
34#include <linux/memremap.h>
35#include <linux/slab.h>
36#include <linux/cacheinfo.h>
37#include <linux/rcuwait.h>
38#include <linux/bitmap.h>
39#include <linux/bitops.h>
40#include <linux/iommu-debug-pagealloc.h>
41
42struct mempolicy;
43struct anon_vma;
44struct anon_vma_chain;
45struct user_struct;
46struct pt_regs;
47struct folio_batch;
48
49void arch_mm_preinit(void);
50void mm_core_init_early(void);
51void mm_core_init(void);
52void init_mm_internals(void);
53
54extern atomic_long_t _totalram_pages;
55static inline unsigned long totalram_pages(void)
56{
57 return (unsigned long)atomic_long_read(&_totalram_pages);
58}
59
60static inline void totalram_pages_inc(void)
61{
62 atomic_long_inc(&_totalram_pages);
63}
64
65static inline void totalram_pages_dec(void)
66{
67 atomic_long_dec(&_totalram_pages);
68}
69
70static inline void totalram_pages_add(long count)
71{
72 atomic_long_add(count, &_totalram_pages);
73}
74
75extern void * high_memory;
76
77/*
78 * Convert between pages and MB
79 * 20 is the shift for 1MB (2^20 = 1MB)
80 * PAGE_SHIFT is the shift for page size (e.g., 12 for 4KB pages)
81 * So (20 - PAGE_SHIFT) converts between pages and MB
82 */
83#define PAGES_TO_MB(pages) ((pages) >> (20 - PAGE_SHIFT))
84#define MB_TO_PAGES(mb) ((mb) << (20 - PAGE_SHIFT))
85
86#ifdef CONFIG_SYSCTL
87extern int sysctl_legacy_va_layout;
88#else
89#define sysctl_legacy_va_layout 0
90#endif
91
92#ifdef CONFIG_HAVE_ARCH_MMAP_RND_BITS
93extern const int mmap_rnd_bits_min;
94extern int mmap_rnd_bits_max __ro_after_init;
95extern int mmap_rnd_bits __read_mostly;
96#endif
97#ifdef CONFIG_HAVE_ARCH_MMAP_RND_COMPAT_BITS
98extern const int mmap_rnd_compat_bits_min;
99extern const int mmap_rnd_compat_bits_max;
100extern int mmap_rnd_compat_bits __read_mostly;
101#endif
102
103#ifndef DIRECT_MAP_PHYSMEM_END
104# ifdef MAX_PHYSMEM_BITS
105# define DIRECT_MAP_PHYSMEM_END ((1ULL << MAX_PHYSMEM_BITS) - 1)
106# else
107# define DIRECT_MAP_PHYSMEM_END (((phys_addr_t)-1)&~(1ULL<<63))
108# endif
109#endif
110
111#define INVALID_PHYS_ADDR (~(phys_addr_t)0)
112
113#include <asm/page.h>
114#include <asm/processor.h>
115
116#ifndef __pa_symbol
117#define __pa_symbol(x) __pa(RELOC_HIDE((unsigned long)(x), 0))
118#endif
119
120#ifndef page_to_virt
121#define page_to_virt(x) __va(PFN_PHYS(page_to_pfn(x)))
122#endif
123
124#ifndef lm_alias
125#define lm_alias(x) __va(__pa_symbol(x))
126#endif
127
128/*
129 * To prevent common memory management code establishing
130 * a zero page mapping on a read fault.
131 * This macro should be defined within <asm/pgtable.h>.
132 * s390 does this to prevent multiplexing of hardware bits
133 * related to the physical page in case of virtualization.
134 */
135#ifndef mm_forbids_zeropage
136#define mm_forbids_zeropage(X) (0)
137#endif
138
139/*
140 * On some architectures it is expensive to call memset() for small sizes.
141 * If an architecture decides to implement their own version of
142 * mm_zero_struct_page they should wrap the defines below in a #ifndef and
143 * define their own version of this macro in <asm/pgtable.h>
144 */
145#if BITS_PER_LONG == 64
146/* This function must be updated when the size of struct page grows above 96
147 * or reduces below 56. The idea that compiler optimizes out switch()
148 * statement, and only leaves move/store instructions. Also the compiler can
149 * combine write statements if they are both assignments and can be reordered,
150 * this can result in several of the writes here being dropped.
151 */
152#define mm_zero_struct_page(pp) __mm_zero_struct_page(pp)
153static inline void __mm_zero_struct_page(struct page *page)
154{
155 unsigned long *_pp = (void *)page;
156
157 /* Check that struct page is either 56, 64, 72, 80, 88 or 96 bytes */
158 BUILD_BUG_ON(sizeof(struct page) & 7);
159 BUILD_BUG_ON(sizeof(struct page) < 56);
160 BUILD_BUG_ON(sizeof(struct page) > 96);
161
162 switch (sizeof(struct page)) {
163 case 96:
164 _pp[11] = 0;
165 fallthrough;
166 case 88:
167 _pp[10] = 0;
168 fallthrough;
169 case 80:
170 _pp[9] = 0;
171 fallthrough;
172 case 72:
173 _pp[8] = 0;
174 fallthrough;
175 case 64:
176 _pp[7] = 0;
177 fallthrough;
178 case 56:
179 _pp[6] = 0;
180 _pp[5] = 0;
181 _pp[4] = 0;
182 _pp[3] = 0;
183 _pp[2] = 0;
184 _pp[1] = 0;
185 _pp[0] = 0;
186 }
187}
188#else
189#define mm_zero_struct_page(pp) ((void)memset((pp), 0, sizeof(struct page)))
190#endif
191
192/*
193 * Default maximum number of active map areas, this limits the number of vmas
194 * per mm struct. Users can overwrite this number by sysctl but there is a
195 * problem.
196 *
197 * When a program's coredump is generated as ELF format, a section is created
198 * per a vma. In ELF, the number of sections is represented in unsigned short.
199 * This means the number of sections should be smaller than 65535 at coredump.
200 * Because the kernel adds some informative sections to a image of program at
201 * generating coredump, we need some margin. The number of extra sections is
202 * 1-3 now and depends on arch. We use "5" as safe margin, here.
203 *
204 * ELF extended numbering allows more than 65535 sections, so 16-bit bound is
205 * not a hard limit any more. Although some userspace tools can be surprised by
206 * that.
207 */
208#define MAPCOUNT_ELF_CORE_MARGIN (5)
209#define DEFAULT_MAX_MAP_COUNT (USHRT_MAX - MAPCOUNT_ELF_CORE_MARGIN)
210
211extern int sysctl_max_map_count;
212
213extern unsigned long sysctl_user_reserve_kbytes;
214extern unsigned long sysctl_admin_reserve_kbytes;
215
216#if defined(CONFIG_SPARSEMEM) && !defined(CONFIG_SPARSEMEM_VMEMMAP)
217bool page_range_contiguous(const struct page *page, unsigned long nr_pages);
218#else
219static inline bool page_range_contiguous(const struct page *page,
220 unsigned long nr_pages)
221{
222 return true;
223}
224#endif
225
226/* to align the pointer to the (next) page boundary */
227#define PAGE_ALIGN(addr) ALIGN(addr, PAGE_SIZE)
228
229/* to align the pointer to the (prev) page boundary */
230#define PAGE_ALIGN_DOWN(addr) ALIGN_DOWN(addr, PAGE_SIZE)
231
232/* test whether an address (unsigned long or pointer) is aligned to PAGE_SIZE */
233#define PAGE_ALIGNED(addr) IS_ALIGNED((unsigned long)(addr), PAGE_SIZE)
234
235/**
236 * folio_page_idx - Return the number of a page in a folio.
237 * @folio: The folio.
238 * @page: The folio page.
239 *
240 * This function expects that the page is actually part of the folio.
241 * The returned number is relative to the start of the folio.
242 */
243static inline unsigned long folio_page_idx(const struct folio *folio,
244 const struct page *page)
245{
246 return page - &folio->page;
247}
248
249static inline struct folio *lru_to_folio(struct list_head *head)
250{
251 return list_entry((head)->prev, struct folio, lru);
252}
253
254void setup_initial_init_mm(void *start_code, void *end_code,
255 void *end_data, void *brk);
256
257/*
258 * Linux kernel virtual memory manager primitives.
259 * The idea being to have a "virtual" mm in the same way
260 * we have a virtual fs - giving a cleaner interface to the
261 * mm details, and allowing different kinds of memory mappings
262 * (from shared memory to executable loading to arbitrary
263 * mmap() functions).
264 */
265
266struct vm_area_struct *vm_area_alloc(struct mm_struct *);
267struct vm_area_struct *vm_area_dup(struct vm_area_struct *);
268void vm_area_free(struct vm_area_struct *);
269
270#ifndef CONFIG_MMU
271extern struct rb_root nommu_region_tree;
272extern struct rw_semaphore nommu_region_sem;
273
274extern unsigned int kobjsize(const void *objp);
275#endif
276
277/*
278 * vm_flags in vm_area_struct, see mm_types.h.
279 * When changing, update also include/trace/events/mmflags.h
280 */
281
282#define VM_NONE 0x00000000
283
284/**
285 * typedef vma_flag_t - specifies an individual VMA flag by bit number.
286 *
287 * This value is made type safe by sparse to avoid passing invalid flag values
288 * around.
289 */
290typedef int __bitwise vma_flag_t;
291
292#define DECLARE_VMA_BIT(name, bitnum) \
293 VMA_ ## name ## _BIT = ((__force vma_flag_t)bitnum)
294#define DECLARE_VMA_BIT_ALIAS(name, aliased) \
295 VMA_ ## name ## _BIT = (VMA_ ## aliased ## _BIT)
296enum {
297 DECLARE_VMA_BIT(READ, 0),
298 DECLARE_VMA_BIT(WRITE, 1),
299 DECLARE_VMA_BIT(EXEC, 2),
300 DECLARE_VMA_BIT(SHARED, 3),
301 /* mprotect() hardcodes VM_MAYREAD >> 4 == VM_READ, and so for r/w/x bits. */
302 DECLARE_VMA_BIT(MAYREAD, 4), /* limits for mprotect() etc. */
303 DECLARE_VMA_BIT(MAYWRITE, 5),
304 DECLARE_VMA_BIT(MAYEXEC, 6),
305 DECLARE_VMA_BIT(MAYSHARE, 7),
306 DECLARE_VMA_BIT(GROWSDOWN, 8), /* general info on the segment */
307#ifdef CONFIG_MMU
308 DECLARE_VMA_BIT(UFFD_MISSING, 9),/* missing pages tracking */
309#else
310 /* nommu: R/O MAP_PRIVATE mapping that might overlay a file mapping */
311 DECLARE_VMA_BIT(MAYOVERLAY, 9),
312#endif /* CONFIG_MMU */
313 /* Page-ranges managed without "struct page", just pure PFN */
314 DECLARE_VMA_BIT(PFNMAP, 10),
315 DECLARE_VMA_BIT(MAYBE_GUARD, 11),
316 DECLARE_VMA_BIT(UFFD_WP, 12), /* wrprotect pages tracking */
317 DECLARE_VMA_BIT(LOCKED, 13),
318 DECLARE_VMA_BIT(IO, 14), /* Memory mapped I/O or similar */
319 DECLARE_VMA_BIT(SEQ_READ, 15), /* App will access data sequentially */
320 DECLARE_VMA_BIT(RAND_READ, 16), /* App will not benefit from clustered reads */
321 DECLARE_VMA_BIT(DONTCOPY, 17), /* Do not copy this vma on fork */
322 DECLARE_VMA_BIT(DONTEXPAND, 18),/* Cannot expand with mremap() */
323 DECLARE_VMA_BIT(LOCKONFAULT, 19),/* Lock pages covered when faulted in */
324 DECLARE_VMA_BIT(ACCOUNT, 20), /* Is a VM accounted object */
325 DECLARE_VMA_BIT(NORESERVE, 21), /* should the VM suppress accounting */
326 DECLARE_VMA_BIT(HUGETLB, 22), /* Huge TLB Page VM */
327 DECLARE_VMA_BIT(SYNC, 23), /* Synchronous page faults */
328 DECLARE_VMA_BIT(ARCH_1, 24), /* Architecture-specific flag */
329 DECLARE_VMA_BIT(WIPEONFORK, 25),/* Wipe VMA contents in child. */
330 DECLARE_VMA_BIT(DONTDUMP, 26), /* Do not include in the core dump */
331 DECLARE_VMA_BIT(SOFTDIRTY, 27), /* NOT soft dirty clean area */
332 DECLARE_VMA_BIT(MIXEDMAP, 28), /* Can contain struct page and pure PFN pages */
333 DECLARE_VMA_BIT(HUGEPAGE, 29), /* MADV_HUGEPAGE marked this vma */
334 DECLARE_VMA_BIT(NOHUGEPAGE, 30),/* MADV_NOHUGEPAGE marked this vma */
335 DECLARE_VMA_BIT(MERGEABLE, 31), /* KSM may merge identical pages */
336 /* These bits are reused, we define specific uses below. */
337 DECLARE_VMA_BIT(HIGH_ARCH_0, 32),
338 DECLARE_VMA_BIT(HIGH_ARCH_1, 33),
339 DECLARE_VMA_BIT(HIGH_ARCH_2, 34),
340 DECLARE_VMA_BIT(HIGH_ARCH_3, 35),
341 DECLARE_VMA_BIT(HIGH_ARCH_4, 36),
342 DECLARE_VMA_BIT(HIGH_ARCH_5, 37),
343 DECLARE_VMA_BIT(HIGH_ARCH_6, 38),
344 /*
345 * This flag is used to connect VFIO to arch specific KVM code. It
346 * indicates that the memory under this VMA is safe for use with any
347 * non-cachable memory type inside KVM. Some VFIO devices, on some
348 * platforms, are thought to be unsafe and can cause machine crashes
349 * if KVM does not lock down the memory type.
350 */
351 DECLARE_VMA_BIT(ALLOW_ANY_UNCACHED, 39),
352#ifdef CONFIG_PPC32
353 DECLARE_VMA_BIT_ALIAS(DROPPABLE, ARCH_1),
354#else
355 DECLARE_VMA_BIT(DROPPABLE, 40),
356#endif
357 DECLARE_VMA_BIT(UFFD_MINOR, 41),
358 DECLARE_VMA_BIT(SEALED, 42),
359 /* Flags that reuse flags above. */
360 DECLARE_VMA_BIT_ALIAS(PKEY_BIT0, HIGH_ARCH_0),
361 DECLARE_VMA_BIT_ALIAS(PKEY_BIT1, HIGH_ARCH_1),
362 DECLARE_VMA_BIT_ALIAS(PKEY_BIT2, HIGH_ARCH_2),
363 DECLARE_VMA_BIT_ALIAS(PKEY_BIT3, HIGH_ARCH_3),
364 DECLARE_VMA_BIT_ALIAS(PKEY_BIT4, HIGH_ARCH_4),
365#if defined(CONFIG_X86_USER_SHADOW_STACK) || defined(CONFIG_RISCV_USER_CFI)
366 /*
367 * VM_SHADOW_STACK should not be set with VM_SHARED because of lack of
368 * support core mm.
369 *
370 * These VMAs will get a single end guard page. This helps userspace
371 * protect itself from attacks. A single page is enough for current
372 * shadow stack archs (x86). See the comments near alloc_shstk() in
373 * arch/x86/kernel/shstk.c for more details on the guard size.
374 */
375 DECLARE_VMA_BIT_ALIAS(SHADOW_STACK, HIGH_ARCH_5),
376#elif defined(CONFIG_ARM64_GCS)
377 /*
378 * arm64's Guarded Control Stack implements similar functionality and
379 * has similar constraints to shadow stacks.
380 */
381 DECLARE_VMA_BIT_ALIAS(SHADOW_STACK, HIGH_ARCH_6),
382#endif
383 DECLARE_VMA_BIT_ALIAS(SAO, ARCH_1), /* Strong Access Ordering (powerpc) */
384 DECLARE_VMA_BIT_ALIAS(GROWSUP, ARCH_1), /* parisc */
385 DECLARE_VMA_BIT_ALIAS(SPARC_ADI, ARCH_1), /* sparc64 */
386 DECLARE_VMA_BIT_ALIAS(ARM64_BTI, ARCH_1), /* arm64 */
387 DECLARE_VMA_BIT_ALIAS(ARCH_CLEAR, ARCH_1), /* sparc64, arm64 */
388 DECLARE_VMA_BIT_ALIAS(MAPPED_COPY, ARCH_1), /* !CONFIG_MMU */
389 DECLARE_VMA_BIT_ALIAS(MTE, HIGH_ARCH_4), /* arm64 */
390 DECLARE_VMA_BIT_ALIAS(MTE_ALLOWED, HIGH_ARCH_5),/* arm64 */
391#ifdef CONFIG_STACK_GROWSUP
392 DECLARE_VMA_BIT_ALIAS(STACK, GROWSUP),
393 DECLARE_VMA_BIT_ALIAS(STACK_EARLY, GROWSDOWN),
394#else
395 DECLARE_VMA_BIT_ALIAS(STACK, GROWSDOWN),
396#endif
397};
398#undef DECLARE_VMA_BIT
399#undef DECLARE_VMA_BIT_ALIAS
400
401#define INIT_VM_FLAG(name) BIT((__force int) VMA_ ## name ## _BIT)
402#define VM_READ INIT_VM_FLAG(READ)
403#define VM_WRITE INIT_VM_FLAG(WRITE)
404#define VM_EXEC INIT_VM_FLAG(EXEC)
405#define VM_SHARED INIT_VM_FLAG(SHARED)
406#define VM_MAYREAD INIT_VM_FLAG(MAYREAD)
407#define VM_MAYWRITE INIT_VM_FLAG(MAYWRITE)
408#define VM_MAYEXEC INIT_VM_FLAG(MAYEXEC)
409#define VM_MAYSHARE INIT_VM_FLAG(MAYSHARE)
410#define VM_GROWSDOWN INIT_VM_FLAG(GROWSDOWN)
411#ifdef CONFIG_MMU
412#define VM_UFFD_MISSING INIT_VM_FLAG(UFFD_MISSING)
413#else
414#define VM_UFFD_MISSING VM_NONE
415#define VM_MAYOVERLAY INIT_VM_FLAG(MAYOVERLAY)
416#endif
417#define VM_PFNMAP INIT_VM_FLAG(PFNMAP)
418#define VM_MAYBE_GUARD INIT_VM_FLAG(MAYBE_GUARD)
419#define VM_UFFD_WP INIT_VM_FLAG(UFFD_WP)
420#define VM_LOCKED INIT_VM_FLAG(LOCKED)
421#define VM_IO INIT_VM_FLAG(IO)
422#define VM_SEQ_READ INIT_VM_FLAG(SEQ_READ)
423#define VM_RAND_READ INIT_VM_FLAG(RAND_READ)
424#define VM_DONTCOPY INIT_VM_FLAG(DONTCOPY)
425#define VM_DONTEXPAND INIT_VM_FLAG(DONTEXPAND)
426#define VM_LOCKONFAULT INIT_VM_FLAG(LOCKONFAULT)
427#define VM_ACCOUNT INIT_VM_FLAG(ACCOUNT)
428#define VM_NORESERVE INIT_VM_FLAG(NORESERVE)
429#define VM_HUGETLB INIT_VM_FLAG(HUGETLB)
430#define VM_SYNC INIT_VM_FLAG(SYNC)
431#define VM_ARCH_1 INIT_VM_FLAG(ARCH_1)
432#define VM_WIPEONFORK INIT_VM_FLAG(WIPEONFORK)
433#define VM_DONTDUMP INIT_VM_FLAG(DONTDUMP)
434#ifdef CONFIG_MEM_SOFT_DIRTY
435#define VM_SOFTDIRTY INIT_VM_FLAG(SOFTDIRTY)
436#else
437#define VM_SOFTDIRTY VM_NONE
438#endif
439#define VM_MIXEDMAP INIT_VM_FLAG(MIXEDMAP)
440#define VM_HUGEPAGE INIT_VM_FLAG(HUGEPAGE)
441#define VM_NOHUGEPAGE INIT_VM_FLAG(NOHUGEPAGE)
442#define VM_MERGEABLE INIT_VM_FLAG(MERGEABLE)
443#define VM_STACK INIT_VM_FLAG(STACK)
444#ifdef CONFIG_STACK_GROWSUP
445#define VM_STACK_EARLY INIT_VM_FLAG(STACK_EARLY)
446#else
447#define VM_STACK_EARLY VM_NONE
448#endif
449#ifdef CONFIG_ARCH_HAS_PKEYS
450#define VM_PKEY_SHIFT ((__force int)VMA_HIGH_ARCH_0_BIT)
451/* Despite the naming, these are FLAGS not bits. */
452#define VM_PKEY_BIT0 INIT_VM_FLAG(PKEY_BIT0)
453#define VM_PKEY_BIT1 INIT_VM_FLAG(PKEY_BIT1)
454#define VM_PKEY_BIT2 INIT_VM_FLAG(PKEY_BIT2)
455#if CONFIG_ARCH_PKEY_BITS > 3
456#define VM_PKEY_BIT3 INIT_VM_FLAG(PKEY_BIT3)
457#else
458#define VM_PKEY_BIT3 VM_NONE
459#endif /* CONFIG_ARCH_PKEY_BITS > 3 */
460#if CONFIG_ARCH_PKEY_BITS > 4
461#define VM_PKEY_BIT4 INIT_VM_FLAG(PKEY_BIT4)
462#else
463#define VM_PKEY_BIT4 VM_NONE
464#endif /* CONFIG_ARCH_PKEY_BITS > 4 */
465#endif /* CONFIG_ARCH_HAS_PKEYS */
466#if defined(CONFIG_X86_USER_SHADOW_STACK) || defined(CONFIG_ARM64_GCS) || \
467 defined(CONFIG_RISCV_USER_CFI)
468#define VM_SHADOW_STACK INIT_VM_FLAG(SHADOW_STACK)
469#else
470#define VM_SHADOW_STACK VM_NONE
471#endif
472#if defined(CONFIG_PPC64)
473#define VM_SAO INIT_VM_FLAG(SAO)
474#elif defined(CONFIG_PARISC)
475#define VM_GROWSUP INIT_VM_FLAG(GROWSUP)
476#elif defined(CONFIG_SPARC64)
477#define VM_SPARC_ADI INIT_VM_FLAG(SPARC_ADI)
478#define VM_ARCH_CLEAR INIT_VM_FLAG(ARCH_CLEAR)
479#elif defined(CONFIG_ARM64)
480#define VM_ARM64_BTI INIT_VM_FLAG(ARM64_BTI)
481#define VM_ARCH_CLEAR INIT_VM_FLAG(ARCH_CLEAR)
482#elif !defined(CONFIG_MMU)
483#define VM_MAPPED_COPY INIT_VM_FLAG(MAPPED_COPY)
484#endif
485#ifndef VM_GROWSUP
486#define VM_GROWSUP VM_NONE
487#endif
488#ifdef CONFIG_ARM64_MTE
489#define VM_MTE INIT_VM_FLAG(MTE)
490#define VM_MTE_ALLOWED INIT_VM_FLAG(MTE_ALLOWED)
491#else
492#define VM_MTE VM_NONE
493#define VM_MTE_ALLOWED VM_NONE
494#endif
495#ifdef CONFIG_HAVE_ARCH_USERFAULTFD_MINOR
496#define VM_UFFD_MINOR INIT_VM_FLAG(UFFD_MINOR)
497#else
498#define VM_UFFD_MINOR VM_NONE
499#endif
500#ifdef CONFIG_64BIT
501#define VM_ALLOW_ANY_UNCACHED INIT_VM_FLAG(ALLOW_ANY_UNCACHED)
502#define VM_SEALED INIT_VM_FLAG(SEALED)
503#else
504#define VM_ALLOW_ANY_UNCACHED VM_NONE
505#define VM_SEALED VM_NONE
506#endif
507#if defined(CONFIG_64BIT) || defined(CONFIG_PPC32)
508#define VM_DROPPABLE INIT_VM_FLAG(DROPPABLE)
509#else
510#define VM_DROPPABLE VM_NONE
511#endif
512
513/* Bits set in the VMA until the stack is in its final location */
514#define VM_STACK_INCOMPLETE_SETUP (VM_RAND_READ | VM_SEQ_READ | VM_STACK_EARLY)
515
516#define TASK_EXEC ((current->personality & READ_IMPLIES_EXEC) ? VM_EXEC : 0)
517
518/* Common data flag combinations */
519#define VM_DATA_FLAGS_TSK_EXEC (VM_READ | VM_WRITE | TASK_EXEC | \
520 VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC)
521#define VM_DATA_FLAGS_NON_EXEC (VM_READ | VM_WRITE | VM_MAYREAD | \
522 VM_MAYWRITE | VM_MAYEXEC)
523#define VM_DATA_FLAGS_EXEC (VM_READ | VM_WRITE | VM_EXEC | \
524 VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC)
525
526#ifndef VM_DATA_DEFAULT_FLAGS /* arch can override this */
527#define VM_DATA_DEFAULT_FLAGS VM_DATA_FLAGS_EXEC
528#endif
529
530#ifndef VM_STACK_DEFAULT_FLAGS /* arch can override this */
531#define VM_STACK_DEFAULT_FLAGS VM_DATA_DEFAULT_FLAGS
532#endif
533
534#define VM_STARTGAP_FLAGS (VM_GROWSDOWN | VM_SHADOW_STACK)
535
536#ifdef CONFIG_MSEAL_SYSTEM_MAPPINGS
537#define VM_SEALED_SYSMAP VM_SEALED
538#else
539#define VM_SEALED_SYSMAP VM_NONE
540#endif
541
542#define VM_STACK_FLAGS (VM_STACK | VM_STACK_DEFAULT_FLAGS | VM_ACCOUNT)
543
544/* VMA basic access permission flags */
545#define VM_ACCESS_FLAGS (VM_READ | VM_WRITE | VM_EXEC)
546
547/*
548 * Special vmas that are non-mergable, non-mlock()able.
549 */
550#define VM_SPECIAL (VM_IO | VM_DONTEXPAND | VM_PFNMAP | VM_MIXEDMAP)
551
552/*
553 * Physically remapped pages are special. Tell the
554 * rest of the world about it:
555 * IO tells people not to look at these pages
556 * (accesses can have side effects).
557 * PFNMAP tells the core MM that the base pages are just
558 * raw PFN mappings, and do not have a "struct page" associated
559 * with them.
560 * DONTEXPAND
561 * Disable vma merging and expanding with mremap().
562 * DONTDUMP
563 * Omit vma from core dump, even when VM_IO turned off.
564 */
565#define VMA_REMAP_FLAGS mk_vma_flags(VMA_IO_BIT, VMA_PFNMAP_BIT, \
566 VMA_DONTEXPAND_BIT, VMA_DONTDUMP_BIT)
567
568/* This mask prevents VMA from being scanned with khugepaged */
569#define VM_NO_KHUGEPAGED (VM_SPECIAL | VM_HUGETLB)
570
571/* This mask defines which mm->def_flags a process can inherit its parent */
572#define VM_INIT_DEF_MASK VM_NOHUGEPAGE
573
574/* This mask represents all the VMA flag bits used by mlock */
575#define VM_LOCKED_MASK (VM_LOCKED | VM_LOCKONFAULT)
576
577/* These flags can be updated atomically via VMA/mmap read lock. */
578#define VM_ATOMIC_SET_ALLOWED VM_MAYBE_GUARD
579
580/* Arch-specific flags to clear when updating VM flags on protection change */
581#ifndef VM_ARCH_CLEAR
582#define VM_ARCH_CLEAR VM_NONE
583#endif
584#define VM_FLAGS_CLEAR (ARCH_VM_PKEY_FLAGS | VM_ARCH_CLEAR)
585
586/*
587 * Flags which should be 'sticky' on merge - that is, flags which, when one VMA
588 * possesses it but the other does not, the merged VMA should nonetheless have
589 * applied to it:
590 *
591 * VM_SOFTDIRTY - if a VMA is marked soft-dirty, that is has not had its
592 * references cleared via /proc/$pid/clear_refs, any merged VMA
593 * should be considered soft-dirty also as it operates at a VMA
594 * granularity.
595 *
596 * VM_MAYBE_GUARD - If a VMA may have guard regions in place it implies that
597 * mapped page tables may contain metadata not described by the
598 * VMA and thus any merged VMA may also contain this metadata,
599 * and thus we must make this flag sticky.
600 */
601#define VM_STICKY (VM_SOFTDIRTY | VM_MAYBE_GUARD)
602
603/*
604 * VMA flags we ignore for the purposes of merge, i.e. one VMA possessing one
605 * of these flags and the other not does not preclude a merge.
606 *
607 * VM_STICKY - When merging VMAs, VMA flags must match, unless they are
608 * 'sticky'. If any sticky flags exist in either VMA, we simply
609 * set all of them on the merged VMA.
610 */
611#define VM_IGNORE_MERGE VM_STICKY
612
613/*
614 * Flags which should result in page tables being copied on fork. These are
615 * flags which indicate that the VMA maps page tables which cannot be
616 * reconsistuted upon page fault, so necessitate page table copying upon fork.
617 *
618 * Note that these flags should be compared with the DESTINATION VMA not the
619 * source, as VM_UFFD_WP may not be propagated to destination, while all other
620 * flags will be.
621 *
622 * VM_PFNMAP / VM_MIXEDMAP - These contain kernel-mapped data which cannot be
623 * reasonably reconstructed on page fault.
624 *
625 * VM_UFFD_WP - Encodes metadata about an installed uffd
626 * write protect handler, which cannot be
627 * reconstructed on page fault.
628 *
629 * We always copy pgtables when dst_vma has uffd-wp
630 * enabled even if it's file-backed
631 * (e.g. shmem). Because when uffd-wp is enabled,
632 * pgtable contains uffd-wp protection information,
633 * that's something we can't retrieve from page cache,
634 * and skip copying will lose those info.
635 *
636 * VM_MAYBE_GUARD - Could contain page guard region markers which
637 * by design are a property of the page tables
638 * only and thus cannot be reconstructed on page
639 * fault.
640 */
641#define VM_COPY_ON_FORK (VM_PFNMAP | VM_MIXEDMAP | VM_UFFD_WP | VM_MAYBE_GUARD)
642
643/*
644 * mapping from the currently active vm_flags protection bits (the
645 * low four bits) to a page protection mask..
646 */
647
648/*
649 * The default fault flags that should be used by most of the
650 * arch-specific page fault handlers.
651 */
652#define FAULT_FLAG_DEFAULT (FAULT_FLAG_ALLOW_RETRY | \
653 FAULT_FLAG_KILLABLE | \
654 FAULT_FLAG_INTERRUPTIBLE)
655
656/**
657 * fault_flag_allow_retry_first - check ALLOW_RETRY the first time
658 * @flags: Fault flags.
659 *
660 * This is mostly used for places where we want to try to avoid taking
661 * the mmap_lock for too long a time when waiting for another condition
662 * to change, in which case we can try to be polite to release the
663 * mmap_lock in the first round to avoid potential starvation of other
664 * processes that would also want the mmap_lock.
665 *
666 * Return: true if the page fault allows retry and this is the first
667 * attempt of the fault handling; false otherwise.
668 */
669static inline bool fault_flag_allow_retry_first(enum fault_flag flags)
670{
671 return (flags & FAULT_FLAG_ALLOW_RETRY) &&
672 (!(flags & FAULT_FLAG_TRIED));
673}
674
675#define FAULT_FLAG_TRACE \
676 { FAULT_FLAG_WRITE, "WRITE" }, \
677 { FAULT_FLAG_MKWRITE, "MKWRITE" }, \
678 { FAULT_FLAG_ALLOW_RETRY, "ALLOW_RETRY" }, \
679 { FAULT_FLAG_RETRY_NOWAIT, "RETRY_NOWAIT" }, \
680 { FAULT_FLAG_KILLABLE, "KILLABLE" }, \
681 { FAULT_FLAG_TRIED, "TRIED" }, \
682 { FAULT_FLAG_USER, "USER" }, \
683 { FAULT_FLAG_REMOTE, "REMOTE" }, \
684 { FAULT_FLAG_INSTRUCTION, "INSTRUCTION" }, \
685 { FAULT_FLAG_INTERRUPTIBLE, "INTERRUPTIBLE" }, \
686 { FAULT_FLAG_VMA_LOCK, "VMA_LOCK" }
687
688/*
689 * vm_fault is filled by the pagefault handler and passed to the vma's
690 * ->fault function. The vma's ->fault is responsible for returning a bitmask
691 * of VM_FAULT_xxx flags that give details about how the fault was handled.
692 *
693 * MM layer fills up gfp_mask for page allocations but fault handler might
694 * alter it if its implementation requires a different allocation context.
695 *
696 * pgoff should be used in favour of virtual_address, if possible.
697 */
698struct vm_fault {
699 const struct {
700 struct vm_area_struct *vma; /* Target VMA */
701 gfp_t gfp_mask; /* gfp mask to be used for allocations */
702 pgoff_t pgoff; /* Logical page offset based on vma */
703 unsigned long address; /* Faulting virtual address - masked */
704 unsigned long real_address; /* Faulting virtual address - unmasked */
705 };
706 enum fault_flag flags; /* FAULT_FLAG_xxx flags
707 * XXX: should really be 'const' */
708 pmd_t *pmd; /* Pointer to pmd entry matching
709 * the 'address' */
710 pud_t *pud; /* Pointer to pud entry matching
711 * the 'address'
712 */
713 union {
714 pte_t orig_pte; /* Value of PTE at the time of fault */
715 pmd_t orig_pmd; /* Value of PMD at the time of fault,
716 * used by PMD fault only.
717 */
718 };
719
720 struct page *cow_page; /* Page handler may use for COW fault */
721 struct page *page; /* ->fault handlers should return a
722 * page here, unless VM_FAULT_NOPAGE
723 * is set (which is also implied by
724 * VM_FAULT_ERROR).
725 */
726 /* These three entries are valid only while holding ptl lock */
727 pte_t *pte; /* Pointer to pte entry matching
728 * the 'address'. NULL if the page
729 * table hasn't been allocated.
730 */
731 spinlock_t *ptl; /* Page table lock.
732 * Protects pte page table if 'pte'
733 * is not NULL, otherwise pmd.
734 */
735 pgtable_t prealloc_pte; /* Pre-allocated pte page table.
736 * vm_ops->map_pages() sets up a page
737 * table from atomic context.
738 * do_fault_around() pre-allocates
739 * page table to avoid allocation from
740 * atomic context.
741 */
742};
743
744/*
745 * These are the virtual MM functions - opening of an area, closing and
746 * unmapping it (needed to keep files on disk up-to-date etc), pointer
747 * to the functions called when a no-page or a wp-page exception occurs.
748 */
749struct vm_operations_struct {
750 void (*open)(struct vm_area_struct * area);
751 /**
752 * @close: Called when the VMA is being removed from the MM.
753 * Context: User context. May sleep. Caller holds mmap_lock.
754 */
755 void (*close)(struct vm_area_struct * area);
756 /* Called any time before splitting to check if it's allowed */
757 int (*may_split)(struct vm_area_struct *area, unsigned long addr);
758 int (*mremap)(struct vm_area_struct *area);
759 /*
760 * Called by mprotect() to make driver-specific permission
761 * checks before mprotect() is finalised. The VMA must not
762 * be modified. Returns 0 if mprotect() can proceed.
763 */
764 int (*mprotect)(struct vm_area_struct *vma, unsigned long start,
765 unsigned long end, unsigned long newflags);
766 vm_fault_t (*fault)(struct vm_fault *vmf);
767 vm_fault_t (*huge_fault)(struct vm_fault *vmf, unsigned int order);
768 vm_fault_t (*map_pages)(struct vm_fault *vmf,
769 pgoff_t start_pgoff, pgoff_t end_pgoff);
770 unsigned long (*pagesize)(struct vm_area_struct * area);
771
772 /* notification that a previously read-only page is about to become
773 * writable, if an error is returned it will cause a SIGBUS */
774 vm_fault_t (*page_mkwrite)(struct vm_fault *vmf);
775
776 /* same as page_mkwrite when using VM_PFNMAP|VM_MIXEDMAP */
777 vm_fault_t (*pfn_mkwrite)(struct vm_fault *vmf);
778
779 /* called by access_process_vm when get_user_pages() fails, typically
780 * for use by special VMAs. See also generic_access_phys() for a generic
781 * implementation useful for any iomem mapping.
782 */
783 int (*access)(struct vm_area_struct *vma, unsigned long addr,
784 void *buf, int len, int write);
785
786 /* Called by the /proc/PID/maps code to ask the vma whether it
787 * has a special name. Returning non-NULL will also cause this
788 * vma to be dumped unconditionally. */
789 const char *(*name)(struct vm_area_struct *vma);
790
791#ifdef CONFIG_NUMA
792 /*
793 * set_policy() op must add a reference to any non-NULL @new mempolicy
794 * to hold the policy upon return. Caller should pass NULL @new to
795 * remove a policy and fall back to surrounding context--i.e. do not
796 * install a MPOL_DEFAULT policy, nor the task or system default
797 * mempolicy.
798 */
799 int (*set_policy)(struct vm_area_struct *vma, struct mempolicy *new);
800
801 /*
802 * get_policy() op must add reference [mpol_get()] to any policy at
803 * (vma,addr) marked as MPOL_SHARED. The shared policy infrastructure
804 * in mm/mempolicy.c will do this automatically.
805 * get_policy() must NOT add a ref if the policy at (vma,addr) is not
806 * marked as MPOL_SHARED. vma policies are protected by the mmap_lock.
807 * If no [shared/vma] mempolicy exists at the addr, get_policy() op
808 * must return NULL--i.e., do not "fallback" to task or system default
809 * policy.
810 */
811 struct mempolicy *(*get_policy)(struct vm_area_struct *vma,
812 unsigned long addr, pgoff_t *ilx);
813#endif
814#ifdef CONFIG_FIND_NORMAL_PAGE
815 /*
816 * Called by vm_normal_page() for special PTEs in @vma at @addr. This
817 * allows for returning a "normal" page from vm_normal_page() even
818 * though the PTE indicates that the "struct page" either does not exist
819 * or should not be touched: "special".
820 *
821 * Do not add new users: this really only works when a "normal" page
822 * was mapped, but then the PTE got changed to something weird (+
823 * marked special) that would not make pte_pfn() identify the originally
824 * inserted page.
825 */
826 struct page *(*find_normal_page)(struct vm_area_struct *vma,
827 unsigned long addr);
828#endif /* CONFIG_FIND_NORMAL_PAGE */
829};
830
831#ifdef CONFIG_NUMA_BALANCING
832static inline void vma_numab_state_init(struct vm_area_struct *vma)
833{
834 vma->numab_state = NULL;
835}
836static inline void vma_numab_state_free(struct vm_area_struct *vma)
837{
838 kfree(vma->numab_state);
839}
840#else
841static inline void vma_numab_state_init(struct vm_area_struct *vma) {}
842static inline void vma_numab_state_free(struct vm_area_struct *vma) {}
843#endif /* CONFIG_NUMA_BALANCING */
844
845/*
846 * These must be here rather than mmap_lock.h as dependent on vm_fault type,
847 * declared in this header.
848 */
849#ifdef CONFIG_PER_VMA_LOCK
850static inline void release_fault_lock(struct vm_fault *vmf)
851{
852 if (vmf->flags & FAULT_FLAG_VMA_LOCK)
853 vma_end_read(vmf->vma);
854 else
855 mmap_read_unlock(vmf->vma->vm_mm);
856}
857
858static inline void assert_fault_locked(const struct vm_fault *vmf)
859{
860 if (vmf->flags & FAULT_FLAG_VMA_LOCK)
861 vma_assert_locked(vmf->vma);
862 else
863 mmap_assert_locked(vmf->vma->vm_mm);
864}
865#else
866static inline void release_fault_lock(struct vm_fault *vmf)
867{
868 mmap_read_unlock(vmf->vma->vm_mm);
869}
870
871static inline void assert_fault_locked(const struct vm_fault *vmf)
872{
873 mmap_assert_locked(vmf->vma->vm_mm);
874}
875#endif /* CONFIG_PER_VMA_LOCK */
876
877static inline bool mm_flags_test(int flag, const struct mm_struct *mm)
878{
879 return test_bit(flag, ACCESS_PRIVATE(&mm->flags, __mm_flags));
880}
881
882static inline bool mm_flags_test_and_set(int flag, struct mm_struct *mm)
883{
884 return test_and_set_bit(flag, ACCESS_PRIVATE(&mm->flags, __mm_flags));
885}
886
887static inline bool mm_flags_test_and_clear(int flag, struct mm_struct *mm)
888{
889 return test_and_clear_bit(flag, ACCESS_PRIVATE(&mm->flags, __mm_flags));
890}
891
892static inline void mm_flags_set(int flag, struct mm_struct *mm)
893{
894 set_bit(flag, ACCESS_PRIVATE(&mm->flags, __mm_flags));
895}
896
897static inline void mm_flags_clear(int flag, struct mm_struct *mm)
898{
899 clear_bit(flag, ACCESS_PRIVATE(&mm->flags, __mm_flags));
900}
901
902static inline void mm_flags_clear_all(struct mm_struct *mm)
903{
904 bitmap_zero(ACCESS_PRIVATE(&mm->flags, __mm_flags), NUM_MM_FLAG_BITS);
905}
906
907extern const struct vm_operations_struct vma_dummy_vm_ops;
908
909static inline void vma_init(struct vm_area_struct *vma, struct mm_struct *mm)
910{
911 memset(vma, 0, sizeof(*vma));
912 vma->vm_mm = mm;
913 vma->vm_ops = &vma_dummy_vm_ops;
914 INIT_LIST_HEAD(&vma->anon_vma_chain);
915 vma_lock_init(vma, false);
916}
917
918/* Use when VMA is not part of the VMA tree and needs no locking */
919static inline void vm_flags_init(struct vm_area_struct *vma,
920 vm_flags_t flags)
921{
922 VM_WARN_ON_ONCE(!pgtable_supports_soft_dirty() && (flags & VM_SOFTDIRTY));
923 vma_flags_clear_all(&vma->flags);
924 vma_flags_overwrite_word(&vma->flags, flags);
925}
926
927/*
928 * Use when VMA is part of the VMA tree and modifications need coordination
929 * Note: vm_flags_reset and vm_flags_reset_once do not lock the vma and
930 * it should be locked explicitly beforehand.
931 */
932static inline void vm_flags_reset(struct vm_area_struct *vma,
933 vm_flags_t flags)
934{
935 VM_WARN_ON_ONCE(!pgtable_supports_soft_dirty() && (flags & VM_SOFTDIRTY));
936 vma_assert_write_locked(vma);
937 vm_flags_init(vma, flags);
938}
939
940static inline void vm_flags_reset_once(struct vm_area_struct *vma,
941 vm_flags_t flags)
942{
943 vma_assert_write_locked(vma);
944 /*
945 * If VMA flags exist beyond the first system word, also clear these. It
946 * is assumed the write once behaviour is required only for the first
947 * system word.
948 */
949 if (NUM_VMA_FLAG_BITS > BITS_PER_LONG) {
950 unsigned long *bitmap = vma->flags.__vma_flags;
951
952 bitmap_zero(&bitmap[1], NUM_VMA_FLAG_BITS - BITS_PER_LONG);
953 }
954
955 vma_flags_overwrite_word_once(&vma->flags, flags);
956}
957
958static inline void vm_flags_set(struct vm_area_struct *vma,
959 vm_flags_t flags)
960{
961 vma_start_write(vma);
962 vma_flags_set_word(&vma->flags, flags);
963}
964
965static inline void vm_flags_clear(struct vm_area_struct *vma,
966 vm_flags_t flags)
967{
968 VM_WARN_ON_ONCE(!pgtable_supports_soft_dirty() && (flags & VM_SOFTDIRTY));
969 vma_start_write(vma);
970 vma_flags_clear_word(&vma->flags, flags);
971}
972
973/*
974 * Use only if VMA is not part of the VMA tree or has no other users and
975 * therefore needs no locking.
976 */
977static inline void __vm_flags_mod(struct vm_area_struct *vma,
978 vm_flags_t set, vm_flags_t clear)
979{
980 vm_flags_init(vma, (vma->vm_flags | set) & ~clear);
981}
982
983/*
984 * Use only when the order of set/clear operations is unimportant, otherwise
985 * use vm_flags_{set|clear} explicitly.
986 */
987static inline void vm_flags_mod(struct vm_area_struct *vma,
988 vm_flags_t set, vm_flags_t clear)
989{
990 vma_start_write(vma);
991 __vm_flags_mod(vma, set, clear);
992}
993
994static inline bool __vma_atomic_valid_flag(struct vm_area_struct *vma, vma_flag_t bit)
995{
996 const vm_flags_t mask = BIT((__force int)bit);
997
998 /* Only specific flags are permitted */
999 if (WARN_ON_ONCE(!(mask & VM_ATOMIC_SET_ALLOWED)))
1000 return false;
1001
1002 return true;
1003}
1004
1005/*
1006 * Set VMA flag atomically. Requires only VMA/mmap read lock. Only specific
1007 * valid flags are allowed to do this.
1008 */
1009static inline void vma_set_atomic_flag(struct vm_area_struct *vma, vma_flag_t bit)
1010{
1011 unsigned long *bitmap = vma->flags.__vma_flags;
1012
1013 vma_assert_stabilised(vma);
1014 if (__vma_atomic_valid_flag(vma, bit))
1015 set_bit((__force int)bit, bitmap);
1016}
1017
1018/*
1019 * Test for VMA flag atomically. Requires no locks. Only specific valid flags
1020 * are allowed to do this.
1021 *
1022 * This is necessarily racey, so callers must ensure that serialisation is
1023 * achieved through some other means, or that races are permissible.
1024 */
1025static inline bool vma_test_atomic_flag(struct vm_area_struct *vma, vma_flag_t bit)
1026{
1027 if (__vma_atomic_valid_flag(vma, bit))
1028 return test_bit((__force int)bit, &vma->vm_flags);
1029
1030 return false;
1031}
1032
1033/* Set an individual VMA flag in flags, non-atomically. */
1034static inline void vma_flag_set(vma_flags_t *flags, vma_flag_t bit)
1035{
1036 unsigned long *bitmap = flags->__vma_flags;
1037
1038 __set_bit((__force int)bit, bitmap);
1039}
1040
1041static inline vma_flags_t __mk_vma_flags(size_t count, const vma_flag_t *bits)
1042{
1043 vma_flags_t flags;
1044 int i;
1045
1046 vma_flags_clear_all(&flags);
1047 for (i = 0; i < count; i++)
1048 vma_flag_set(&flags, bits[i]);
1049 return flags;
1050}
1051
1052/*
1053 * Helper macro which bitwise-or combines the specified input flags into a
1054 * vma_flags_t bitmap value. E.g.:
1055 *
1056 * vma_flags_t flags = mk_vma_flags(VMA_IO_BIT, VMA_PFNMAP_BIT,
1057 * VMA_DONTEXPAND_BIT, VMA_DONTDUMP_BIT);
1058 *
1059 * The compiler cleverly optimises away all of the work and this ends up being
1060 * equivalent to aggregating the values manually.
1061 */
1062#define mk_vma_flags(...) __mk_vma_flags(COUNT_ARGS(__VA_ARGS__), \
1063 (const vma_flag_t []){__VA_ARGS__})
1064
1065/* Test each of to_test flags in flags, non-atomically. */
1066static __always_inline bool vma_flags_test_mask(const vma_flags_t *flags,
1067 vma_flags_t to_test)
1068{
1069 const unsigned long *bitmap = flags->__vma_flags;
1070 const unsigned long *bitmap_to_test = to_test.__vma_flags;
1071
1072 return bitmap_intersects(bitmap_to_test, bitmap, NUM_VMA_FLAG_BITS);
1073}
1074
1075/*
1076 * Test whether any specified VMA flag is set, e.g.:
1077 *
1078 * if (vma_flags_test(flags, VMA_READ_BIT, VMA_MAYREAD_BIT)) { ... }
1079 */
1080#define vma_flags_test(flags, ...) \
1081 vma_flags_test_mask(flags, mk_vma_flags(__VA_ARGS__))
1082
1083/* Test that ALL of the to_test flags are set, non-atomically. */
1084static __always_inline bool vma_flags_test_all_mask(const vma_flags_t *flags,
1085 vma_flags_t to_test)
1086{
1087 const unsigned long *bitmap = flags->__vma_flags;
1088 const unsigned long *bitmap_to_test = to_test.__vma_flags;
1089
1090 return bitmap_subset(bitmap_to_test, bitmap, NUM_VMA_FLAG_BITS);
1091}
1092
1093/*
1094 * Test whether ALL specified VMA flags are set, e.g.:
1095 *
1096 * if (vma_flags_test_all(flags, VMA_READ_BIT, VMA_MAYREAD_BIT)) { ... }
1097 */
1098#define vma_flags_test_all(flags, ...) \
1099 vma_flags_test_all_mask(flags, mk_vma_flags(__VA_ARGS__))
1100
1101/* Set each of the to_set flags in flags, non-atomically. */
1102static __always_inline void vma_flags_set_mask(vma_flags_t *flags, vma_flags_t to_set)
1103{
1104 unsigned long *bitmap = flags->__vma_flags;
1105 const unsigned long *bitmap_to_set = to_set.__vma_flags;
1106
1107 bitmap_or(bitmap, bitmap, bitmap_to_set, NUM_VMA_FLAG_BITS);
1108}
1109
1110/*
1111 * Set all specified VMA flags, e.g.:
1112 *
1113 * vma_flags_set(&flags, VMA_READ_BIT, VMA_WRITE_BIT, VMA_EXEC_BIT);
1114 */
1115#define vma_flags_set(flags, ...) \
1116 vma_flags_set_mask(flags, mk_vma_flags(__VA_ARGS__))
1117
1118/* Clear all of the to-clear flags in flags, non-atomically. */
1119static __always_inline void vma_flags_clear_mask(vma_flags_t *flags, vma_flags_t to_clear)
1120{
1121 unsigned long *bitmap = flags->__vma_flags;
1122 const unsigned long *bitmap_to_clear = to_clear.__vma_flags;
1123
1124 bitmap_andnot(bitmap, bitmap, bitmap_to_clear, NUM_VMA_FLAG_BITS);
1125}
1126
1127/*
1128 * Clear all specified individual flags, e.g.:
1129 *
1130 * vma_flags_clear(&flags, VMA_READ_BIT, VMA_WRITE_BIT, VMA_EXEC_BIT);
1131 */
1132#define vma_flags_clear(flags, ...) \
1133 vma_flags_clear_mask(flags, mk_vma_flags(__VA_ARGS__))
1134
1135/*
1136 * Helper to test that ALL specified flags are set in a VMA.
1137 *
1138 * Note: appropriate locks must be held, this function does not acquire them for
1139 * you.
1140 */
1141static inline bool vma_test_all_flags_mask(const struct vm_area_struct *vma,
1142 vma_flags_t flags)
1143{
1144 return vma_flags_test_all_mask(&vma->flags, flags);
1145}
1146
1147/*
1148 * Helper macro for checking that ALL specified flags are set in a VMA, e.g.:
1149 *
1150 * if (vma_test_all_flags(vma, VMA_READ_BIT, VMA_MAYREAD_BIT) { ... }
1151 */
1152#define vma_test_all_flags(vma, ...) \
1153 vma_test_all_flags_mask(vma, mk_vma_flags(__VA_ARGS__))
1154
1155/*
1156 * Helper to set all VMA flags in a VMA.
1157 *
1158 * Note: appropriate locks must be held, this function does not acquire them for
1159 * you.
1160 */
1161static inline void vma_set_flags_mask(struct vm_area_struct *vma,
1162 vma_flags_t flags)
1163{
1164 vma_flags_set_mask(&vma->flags, flags);
1165}
1166
1167/*
1168 * Helper macro for specifying VMA flags in a VMA, e.g.:
1169 *
1170 * vma_set_flags(vma, VMA_IO_BIT, VMA_PFNMAP_BIT, VMA_DONTEXPAND_BIT,
1171 * VMA_DONTDUMP_BIT);
1172 *
1173 * Note: appropriate locks must be held, this function does not acquire them for
1174 * you.
1175 */
1176#define vma_set_flags(vma, ...) \
1177 vma_set_flags_mask(vma, mk_vma_flags(__VA_ARGS__))
1178
1179/* Helper to test all VMA flags in a VMA descriptor. */
1180static inline bool vma_desc_test_flags_mask(const struct vm_area_desc *desc,
1181 vma_flags_t flags)
1182{
1183 return vma_flags_test_mask(&desc->vma_flags, flags);
1184}
1185
1186/*
1187 * Helper macro for testing VMA flags for an input pointer to a struct
1188 * vm_area_desc object describing a proposed VMA, e.g.:
1189 *
1190 * if (vma_desc_test_flags(desc, VMA_IO_BIT, VMA_PFNMAP_BIT,
1191 * VMA_DONTEXPAND_BIT, VMA_DONTDUMP_BIT)) { ... }
1192 */
1193#define vma_desc_test_flags(desc, ...) \
1194 vma_desc_test_flags_mask(desc, mk_vma_flags(__VA_ARGS__))
1195
1196/* Helper to set all VMA flags in a VMA descriptor. */
1197static inline void vma_desc_set_flags_mask(struct vm_area_desc *desc,
1198 vma_flags_t flags)
1199{
1200 vma_flags_set_mask(&desc->vma_flags, flags);
1201}
1202
1203/*
1204 * Helper macro for specifying VMA flags for an input pointer to a struct
1205 * vm_area_desc object describing a proposed VMA, e.g.:
1206 *
1207 * vma_desc_set_flags(desc, VMA_IO_BIT, VMA_PFNMAP_BIT, VMA_DONTEXPAND_BIT,
1208 * VMA_DONTDUMP_BIT);
1209 */
1210#define vma_desc_set_flags(desc, ...) \
1211 vma_desc_set_flags_mask(desc, mk_vma_flags(__VA_ARGS__))
1212
1213/* Helper to clear all VMA flags in a VMA descriptor. */
1214static inline void vma_desc_clear_flags_mask(struct vm_area_desc *desc,
1215 vma_flags_t flags)
1216{
1217 vma_flags_clear_mask(&desc->vma_flags, flags);
1218}
1219
1220/*
1221 * Helper macro for clearing VMA flags for an input pointer to a struct
1222 * vm_area_desc object describing a proposed VMA, e.g.:
1223 *
1224 * vma_desc_clear_flags(desc, VMA_IO_BIT, VMA_PFNMAP_BIT, VMA_DONTEXPAND_BIT,
1225 * VMA_DONTDUMP_BIT);
1226 */
1227#define vma_desc_clear_flags(desc, ...) \
1228 vma_desc_clear_flags_mask(desc, mk_vma_flags(__VA_ARGS__))
1229
1230static inline void vma_set_anonymous(struct vm_area_struct *vma)
1231{
1232 vma->vm_ops = NULL;
1233}
1234
1235static inline bool vma_is_anonymous(struct vm_area_struct *vma)
1236{
1237 return !vma->vm_ops;
1238}
1239
1240/*
1241 * Indicate if the VMA is a heap for the given task; for
1242 * /proc/PID/maps that is the heap of the main task.
1243 */
1244static inline bool vma_is_initial_heap(const struct vm_area_struct *vma)
1245{
1246 return vma->vm_start < vma->vm_mm->brk &&
1247 vma->vm_end > vma->vm_mm->start_brk;
1248}
1249
1250/*
1251 * Indicate if the VMA is a stack for the given task; for
1252 * /proc/PID/maps that is the stack of the main task.
1253 */
1254static inline bool vma_is_initial_stack(const struct vm_area_struct *vma)
1255{
1256 /*
1257 * We make no effort to guess what a given thread considers to be
1258 * its "stack". It's not even well-defined for programs written
1259 * languages like Go.
1260 */
1261 return vma->vm_start <= vma->vm_mm->start_stack &&
1262 vma->vm_end >= vma->vm_mm->start_stack;
1263}
1264
1265static inline bool vma_is_temporary_stack(const struct vm_area_struct *vma)
1266{
1267 int maybe_stack = vma->vm_flags & (VM_GROWSDOWN | VM_GROWSUP);
1268
1269 if (!maybe_stack)
1270 return false;
1271
1272 if ((vma->vm_flags & VM_STACK_INCOMPLETE_SETUP) ==
1273 VM_STACK_INCOMPLETE_SETUP)
1274 return true;
1275
1276 return false;
1277}
1278
1279static inline bool vma_is_foreign(const struct vm_area_struct *vma)
1280{
1281 if (!current->mm)
1282 return true;
1283
1284 if (current->mm != vma->vm_mm)
1285 return true;
1286
1287 return false;
1288}
1289
1290static inline bool vma_is_accessible(const struct vm_area_struct *vma)
1291{
1292 return vma->vm_flags & VM_ACCESS_FLAGS;
1293}
1294
1295static inline bool is_shared_maywrite_vm_flags(vm_flags_t vm_flags)
1296{
1297 return (vm_flags & (VM_SHARED | VM_MAYWRITE)) ==
1298 (VM_SHARED | VM_MAYWRITE);
1299}
1300
1301static inline bool is_shared_maywrite(const vma_flags_t *flags)
1302{
1303 return vma_flags_test_all(flags, VMA_SHARED_BIT, VMA_MAYWRITE_BIT);
1304}
1305
1306static inline bool vma_is_shared_maywrite(const struct vm_area_struct *vma)
1307{
1308 return is_shared_maywrite(&vma->flags);
1309}
1310
1311static inline
1312struct vm_area_struct *vma_find(struct vma_iterator *vmi, unsigned long max)
1313{
1314 return mas_find(&vmi->mas, max - 1);
1315}
1316
1317static inline struct vm_area_struct *vma_next(struct vma_iterator *vmi)
1318{
1319 /*
1320 * Uses mas_find() to get the first VMA when the iterator starts.
1321 * Calling mas_next() could skip the first entry.
1322 */
1323 return mas_find(&vmi->mas, ULONG_MAX);
1324}
1325
1326static inline
1327struct vm_area_struct *vma_iter_next_range(struct vma_iterator *vmi)
1328{
1329 return mas_next_range(&vmi->mas, ULONG_MAX);
1330}
1331
1332
1333static inline struct vm_area_struct *vma_prev(struct vma_iterator *vmi)
1334{
1335 return mas_prev(&vmi->mas, 0);
1336}
1337
1338static inline int vma_iter_clear_gfp(struct vma_iterator *vmi,
1339 unsigned long start, unsigned long end, gfp_t gfp)
1340{
1341 __mas_set_range(&vmi->mas, start, end - 1);
1342 mas_store_gfp(&vmi->mas, NULL, gfp);
1343 if (unlikely(mas_is_err(&vmi->mas)))
1344 return -ENOMEM;
1345
1346 return 0;
1347}
1348
1349/* Free any unused preallocations */
1350static inline void vma_iter_free(struct vma_iterator *vmi)
1351{
1352 mas_destroy(&vmi->mas);
1353}
1354
1355static inline int vma_iter_bulk_store(struct vma_iterator *vmi,
1356 struct vm_area_struct *vma)
1357{
1358 vmi->mas.index = vma->vm_start;
1359 vmi->mas.last = vma->vm_end - 1;
1360 mas_store(&vmi->mas, vma);
1361 if (unlikely(mas_is_err(&vmi->mas)))
1362 return -ENOMEM;
1363
1364 vma_mark_attached(vma);
1365 return 0;
1366}
1367
1368static inline void vma_iter_invalidate(struct vma_iterator *vmi)
1369{
1370 mas_pause(&vmi->mas);
1371}
1372
1373static inline void vma_iter_set(struct vma_iterator *vmi, unsigned long addr)
1374{
1375 mas_set(&vmi->mas, addr);
1376}
1377
1378#define for_each_vma(__vmi, __vma) \
1379 while (((__vma) = vma_next(&(__vmi))) != NULL)
1380
1381/* The MM code likes to work with exclusive end addresses */
1382#define for_each_vma_range(__vmi, __vma, __end) \
1383 while (((__vma) = vma_find(&(__vmi), (__end))) != NULL)
1384
1385#ifdef CONFIG_SHMEM
1386/*
1387 * The vma_is_shmem is not inline because it is used only by slow
1388 * paths in userfault.
1389 */
1390bool vma_is_shmem(const struct vm_area_struct *vma);
1391bool vma_is_anon_shmem(const struct vm_area_struct *vma);
1392#else
1393static inline bool vma_is_shmem(const struct vm_area_struct *vma) { return false; }
1394static inline bool vma_is_anon_shmem(const struct vm_area_struct *vma) { return false; }
1395#endif
1396
1397int vma_is_stack_for_current(const struct vm_area_struct *vma);
1398
1399/* flush_tlb_range() takes a vma, not a mm, and can care about flags */
1400#define TLB_FLUSH_VMA(mm,flags) { .vm_mm = (mm), .vm_flags = (flags) }
1401
1402struct mmu_gather;
1403struct inode;
1404
1405extern void prep_compound_page(struct page *page, unsigned int order);
1406
1407static inline unsigned int folio_large_order(const struct folio *folio)
1408{
1409 return folio->_flags_1 & 0xff;
1410}
1411
1412#ifdef NR_PAGES_IN_LARGE_FOLIO
1413static inline unsigned long folio_large_nr_pages(const struct folio *folio)
1414{
1415 return folio->_nr_pages;
1416}
1417#else
1418static inline unsigned long folio_large_nr_pages(const struct folio *folio)
1419{
1420 return 1L << folio_large_order(folio);
1421}
1422#endif
1423
1424/*
1425 * compound_order() can be called without holding a reference, which means
1426 * that niceties like page_folio() don't work. These callers should be
1427 * prepared to handle wild return values. For example, PG_head may be
1428 * set before the order is initialised, or this may be a tail page.
1429 * See compaction.c for some good examples.
1430 */
1431static inline unsigned int compound_order(const struct page *page)
1432{
1433 const struct folio *folio = (struct folio *)page;
1434
1435 if (!test_bit(PG_head, &folio->flags.f))
1436 return 0;
1437 return folio_large_order(folio);
1438}
1439
1440/**
1441 * folio_order - The allocation order of a folio.
1442 * @folio: The folio.
1443 *
1444 * A folio is composed of 2^order pages. See get_order() for the definition
1445 * of order.
1446 *
1447 * Return: The order of the folio.
1448 */
1449static inline unsigned int folio_order(const struct folio *folio)
1450{
1451 if (!folio_test_large(folio))
1452 return 0;
1453 return folio_large_order(folio);
1454}
1455
1456/**
1457 * folio_reset_order - Reset the folio order and derived _nr_pages
1458 * @folio: The folio.
1459 *
1460 * Reset the order and derived _nr_pages to 0. Must only be used in the
1461 * process of splitting large folios.
1462 */
1463static inline void folio_reset_order(struct folio *folio)
1464{
1465 if (WARN_ON_ONCE(!folio_test_large(folio)))
1466 return;
1467 folio->_flags_1 &= ~0xffUL;
1468#ifdef NR_PAGES_IN_LARGE_FOLIO
1469 folio->_nr_pages = 0;
1470#endif
1471}
1472
1473#include <linux/huge_mm.h>
1474
1475/*
1476 * Methods to modify the page usage count.
1477 *
1478 * What counts for a page usage:
1479 * - cache mapping (page->mapping)
1480 * - private data (page->private)
1481 * - page mapped in a task's page tables, each mapping
1482 * is counted separately
1483 *
1484 * Also, many kernel routines increase the page count before a critical
1485 * routine so they can be sure the page doesn't go away from under them.
1486 */
1487
1488/*
1489 * Drop a ref, return true if the refcount fell to zero (the page has no users)
1490 */
1491static inline int put_page_testzero(struct page *page)
1492{
1493 VM_BUG_ON_PAGE(page_ref_count(page) == 0, page);
1494 return page_ref_dec_and_test(page);
1495}
1496
1497static inline int folio_put_testzero(struct folio *folio)
1498{
1499 return put_page_testzero(&folio->page);
1500}
1501
1502/*
1503 * Try to grab a ref unless the page has a refcount of zero, return false if
1504 * that is the case.
1505 * This can be called when MMU is off so it must not access
1506 * any of the virtual mappings.
1507 */
1508static inline bool get_page_unless_zero(struct page *page)
1509{
1510 return page_ref_add_unless(page, 1, 0);
1511}
1512
1513static inline struct folio *folio_get_nontail_page(struct page *page)
1514{
1515 if (unlikely(!get_page_unless_zero(page)))
1516 return NULL;
1517 return (struct folio *)page;
1518}
1519
1520extern int page_is_ram(unsigned long pfn);
1521
1522enum {
1523 REGION_INTERSECTS,
1524 REGION_DISJOINT,
1525 REGION_MIXED,
1526};
1527
1528int region_intersects(resource_size_t offset, size_t size, unsigned long flags,
1529 unsigned long desc);
1530
1531/* Support for virtually mapped pages */
1532struct page *vmalloc_to_page(const void *addr);
1533unsigned long vmalloc_to_pfn(const void *addr);
1534
1535/*
1536 * Determine if an address is within the vmalloc range
1537 *
1538 * On nommu, vmalloc/vfree wrap through kmalloc/kfree directly, so there
1539 * is no special casing required.
1540 */
1541#ifdef CONFIG_MMU
1542extern bool is_vmalloc_addr(const void *x);
1543extern int is_vmalloc_or_module_addr(const void *x);
1544#else
1545static inline bool is_vmalloc_addr(const void *x)
1546{
1547 return false;
1548}
1549static inline int is_vmalloc_or_module_addr(const void *x)
1550{
1551 return 0;
1552}
1553#endif
1554
1555/*
1556 * How many times the entire folio is mapped as a single unit (eg by a
1557 * PMD or PUD entry). This is probably not what you want, except for
1558 * debugging purposes or implementation of other core folio_*() primitives.
1559 */
1560static inline int folio_entire_mapcount(const struct folio *folio)
1561{
1562 VM_BUG_ON_FOLIO(!folio_test_large(folio), folio);
1563 if (!IS_ENABLED(CONFIG_64BIT) && unlikely(folio_large_order(folio) == 1))
1564 return 0;
1565 return atomic_read(&folio->_entire_mapcount) + 1;
1566}
1567
1568static inline int folio_large_mapcount(const struct folio *folio)
1569{
1570 VM_WARN_ON_FOLIO(!folio_test_large(folio), folio);
1571 return atomic_read(&folio->_large_mapcount) + 1;
1572}
1573
1574/**
1575 * folio_mapcount() - Number of mappings of this folio.
1576 * @folio: The folio.
1577 *
1578 * The folio mapcount corresponds to the number of present user page table
1579 * entries that reference any part of a folio. Each such present user page
1580 * table entry must be paired with exactly on folio reference.
1581 *
1582 * For ordindary folios, each user page table entry (PTE/PMD/PUD/...) counts
1583 * exactly once.
1584 *
1585 * For hugetlb folios, each abstracted "hugetlb" user page table entry that
1586 * references the entire folio counts exactly once, even when such special
1587 * page table entries are comprised of multiple ordinary page table entries.
1588 *
1589 * Will report 0 for pages which cannot be mapped into userspace, such as
1590 * slab, page tables and similar.
1591 *
1592 * Return: The number of times this folio is mapped.
1593 */
1594static inline int folio_mapcount(const struct folio *folio)
1595{
1596 int mapcount;
1597
1598 if (likely(!folio_test_large(folio))) {
1599 mapcount = atomic_read(&folio->_mapcount) + 1;
1600 if (page_mapcount_is_type(mapcount))
1601 mapcount = 0;
1602 return mapcount;
1603 }
1604 return folio_large_mapcount(folio);
1605}
1606
1607/**
1608 * folio_mapped - Is this folio mapped into userspace?
1609 * @folio: The folio.
1610 *
1611 * Return: True if any page in this folio is referenced by user page tables.
1612 */
1613static inline bool folio_mapped(const struct folio *folio)
1614{
1615 return folio_mapcount(folio) >= 1;
1616}
1617
1618/*
1619 * Return true if this page is mapped into pagetables.
1620 * For compound page it returns true if any sub-page of compound page is mapped,
1621 * even if this particular sub-page is not itself mapped by any PTE or PMD.
1622 */
1623static inline bool page_mapped(const struct page *page)
1624{
1625 return folio_mapped(page_folio(page));
1626}
1627
1628static inline struct page *virt_to_head_page(const void *x)
1629{
1630 struct page *page = virt_to_page(x);
1631
1632 return compound_head(page);
1633}
1634
1635static inline struct folio *virt_to_folio(const void *x)
1636{
1637 struct page *page = virt_to_page(x);
1638
1639 return page_folio(page);
1640}
1641
1642void __folio_put(struct folio *folio);
1643
1644void split_page(struct page *page, unsigned int order);
1645void folio_copy(struct folio *dst, struct folio *src);
1646int folio_mc_copy(struct folio *dst, struct folio *src);
1647
1648unsigned long nr_free_buffer_pages(void);
1649
1650/* Returns the number of bytes in this potentially compound page. */
1651static inline unsigned long page_size(const struct page *page)
1652{
1653 return PAGE_SIZE << compound_order(page);
1654}
1655
1656/* Returns the number of bits needed for the number of bytes in a page */
1657static inline unsigned int page_shift(struct page *page)
1658{
1659 return PAGE_SHIFT + compound_order(page);
1660}
1661
1662/**
1663 * thp_order - Order of a transparent huge page.
1664 * @page: Head page of a transparent huge page.
1665 */
1666static inline unsigned int thp_order(struct page *page)
1667{
1668 VM_BUG_ON_PGFLAGS(PageTail(page), page);
1669 return compound_order(page);
1670}
1671
1672/**
1673 * thp_size - Size of a transparent huge page.
1674 * @page: Head page of a transparent huge page.
1675 *
1676 * Return: Number of bytes in this page.
1677 */
1678static inline unsigned long thp_size(struct page *page)
1679{
1680 return PAGE_SIZE << thp_order(page);
1681}
1682
1683#ifdef CONFIG_MMU
1684/*
1685 * Do pte_mkwrite, but only if the vma says VM_WRITE. We do this when
1686 * servicing faults for write access. In the normal case, do always want
1687 * pte_mkwrite. But get_user_pages can cause write faults for mappings
1688 * that do not have writing enabled, when used by access_process_vm.
1689 */
1690static inline pte_t maybe_mkwrite(pte_t pte, struct vm_area_struct *vma)
1691{
1692 if (likely(vma->vm_flags & VM_WRITE))
1693 pte = pte_mkwrite(pte, vma);
1694 return pte;
1695}
1696
1697vm_fault_t do_set_pmd(struct vm_fault *vmf, struct folio *folio, struct page *page);
1698void set_pte_range(struct vm_fault *vmf, struct folio *folio,
1699 struct page *page, unsigned int nr, unsigned long addr);
1700
1701vm_fault_t finish_fault(struct vm_fault *vmf);
1702#endif
1703
1704/*
1705 * Multiple processes may "see" the same page. E.g. for untouched
1706 * mappings of /dev/null, all processes see the same page full of
1707 * zeroes, and text pages of executables and shared libraries have
1708 * only one copy in memory, at most, normally.
1709 *
1710 * For the non-reserved pages, page_count(page) denotes a reference count.
1711 * page_count() == 0 means the page is free. page->lru is then used for
1712 * freelist management in the buddy allocator.
1713 * page_count() > 0 means the page has been allocated.
1714 *
1715 * Pages are allocated by the slab allocator in order to provide memory
1716 * to kmalloc and kmem_cache_alloc. In this case, the management of the
1717 * page, and the fields in 'struct page' are the responsibility of mm/slab.c
1718 * unless a particular usage is carefully commented. (the responsibility of
1719 * freeing the kmalloc memory is the caller's, of course).
1720 *
1721 * A page may be used by anyone else who does a __get_free_page().
1722 * In this case, page_count still tracks the references, and should only
1723 * be used through the normal accessor functions. The top bits of page->flags
1724 * and page->virtual store page management information, but all other fields
1725 * are unused and could be used privately, carefully. The management of this
1726 * page is the responsibility of the one who allocated it, and those who have
1727 * subsequently been given references to it.
1728 *
1729 * The other pages (we may call them "pagecache pages") are completely
1730 * managed by the Linux memory manager: I/O, buffers, swapping etc.
1731 * The following discussion applies only to them.
1732 *
1733 * A pagecache page contains an opaque `private' member, which belongs to the
1734 * page's address_space. Usually, this is the address of a circular list of
1735 * the page's disk buffers. PG_private must be set to tell the VM to call
1736 * into the filesystem to release these pages.
1737 *
1738 * A folio may belong to an inode's memory mapping. In this case,
1739 * folio->mapping points to the inode, and folio->index is the file
1740 * offset of the folio, in units of PAGE_SIZE.
1741 *
1742 * If pagecache pages are not associated with an inode, they are said to be
1743 * anonymous pages. These may become associated with the swapcache, and in that
1744 * case PG_swapcache is set, and page->private is an offset into the swapcache.
1745 *
1746 * In either case (swapcache or inode backed), the pagecache itself holds one
1747 * reference to the page. Setting PG_private should also increment the
1748 * refcount. The each user mapping also has a reference to the page.
1749 *
1750 * The pagecache pages are stored in a per-mapping radix tree, which is
1751 * rooted at mapping->i_pages, and indexed by offset.
1752 * Where 2.4 and early 2.6 kernels kept dirty/clean pages in per-address_space
1753 * lists, we instead now tag pages as dirty/writeback in the radix tree.
1754 *
1755 * All pagecache pages may be subject to I/O:
1756 * - inode pages may need to be read from disk,
1757 * - inode pages which have been modified and are MAP_SHARED may need
1758 * to be written back to the inode on disk,
1759 * - anonymous pages (including MAP_PRIVATE file mappings) which have been
1760 * modified may need to be swapped out to swap space and (later) to be read
1761 * back into memory.
1762 */
1763
1764/* 127: arbitrary random number, small enough to assemble well */
1765#define folio_ref_zero_or_close_to_overflow(folio) \
1766 ((unsigned int) folio_ref_count(folio) + 127u <= 127u)
1767
1768/**
1769 * folio_get - Increment the reference count on a folio.
1770 * @folio: The folio.
1771 *
1772 * Context: May be called in any context, as long as you know that
1773 * you have a refcount on the folio. If you do not already have one,
1774 * folio_try_get() may be the right interface for you to use.
1775 */
1776static inline void folio_get(struct folio *folio)
1777{
1778 VM_BUG_ON_FOLIO(folio_ref_zero_or_close_to_overflow(folio), folio);
1779 folio_ref_inc(folio);
1780}
1781
1782static inline void get_page(struct page *page)
1783{
1784 struct folio *folio = page_folio(page);
1785 if (WARN_ON_ONCE(folio_test_slab(folio)))
1786 return;
1787 if (WARN_ON_ONCE(folio_test_large_kmalloc(folio)))
1788 return;
1789 folio_get(folio);
1790}
1791
1792static inline __must_check bool try_get_page(struct page *page)
1793{
1794 page = compound_head(page);
1795 if (WARN_ON_ONCE(page_ref_count(page) <= 0))
1796 return false;
1797 page_ref_inc(page);
1798 return true;
1799}
1800
1801/**
1802 * folio_put - Decrement the reference count on a folio.
1803 * @folio: The folio.
1804 *
1805 * If the folio's reference count reaches zero, the memory will be
1806 * released back to the page allocator and may be used by another
1807 * allocation immediately. Do not access the memory or the struct folio
1808 * after calling folio_put() unless you can be sure that it wasn't the
1809 * last reference.
1810 *
1811 * Context: May be called in process or interrupt context, but not in NMI
1812 * context. May be called while holding a spinlock.
1813 */
1814static inline void folio_put(struct folio *folio)
1815{
1816 if (folio_put_testzero(folio))
1817 __folio_put(folio);
1818}
1819
1820/**
1821 * folio_put_refs - Reduce the reference count on a folio.
1822 * @folio: The folio.
1823 * @refs: The amount to subtract from the folio's reference count.
1824 *
1825 * If the folio's reference count reaches zero, the memory will be
1826 * released back to the page allocator and may be used by another
1827 * allocation immediately. Do not access the memory or the struct folio
1828 * after calling folio_put_refs() unless you can be sure that these weren't
1829 * the last references.
1830 *
1831 * Context: May be called in process or interrupt context, but not in NMI
1832 * context. May be called while holding a spinlock.
1833 */
1834static inline void folio_put_refs(struct folio *folio, int refs)
1835{
1836 if (folio_ref_sub_and_test(folio, refs))
1837 __folio_put(folio);
1838}
1839
1840void folios_put_refs(struct folio_batch *folios, unsigned int *refs);
1841
1842/*
1843 * union release_pages_arg - an array of pages or folios
1844 *
1845 * release_pages() releases a simple array of multiple pages, and
1846 * accepts various different forms of said page array: either
1847 * a regular old boring array of pages, an array of folios, or
1848 * an array of encoded page pointers.
1849 *
1850 * The transparent union syntax for this kind of "any of these
1851 * argument types" is all kinds of ugly, so look away.
1852 */
1853typedef union {
1854 struct page **pages;
1855 struct folio **folios;
1856 struct encoded_page **encoded_pages;
1857} release_pages_arg __attribute__ ((__transparent_union__));
1858
1859void release_pages(release_pages_arg, int nr);
1860
1861/**
1862 * folios_put - Decrement the reference count on an array of folios.
1863 * @folios: The folios.
1864 *
1865 * Like folio_put(), but for a batch of folios. This is more efficient
1866 * than writing the loop yourself as it will optimise the locks which need
1867 * to be taken if the folios are freed. The folios batch is returned
1868 * empty and ready to be reused for another batch; there is no need to
1869 * reinitialise it.
1870 *
1871 * Context: May be called in process or interrupt context, but not in NMI
1872 * context. May be called while holding a spinlock.
1873 */
1874static inline void folios_put(struct folio_batch *folios)
1875{
1876 folios_put_refs(folios, NULL);
1877}
1878
1879static inline void put_page(struct page *page)
1880{
1881 struct folio *folio = page_folio(page);
1882
1883 if (folio_test_slab(folio) || folio_test_large_kmalloc(folio))
1884 return;
1885
1886 folio_put(folio);
1887}
1888
1889/*
1890 * GUP_PIN_COUNTING_BIAS, and the associated functions that use it, overload
1891 * the page's refcount so that two separate items are tracked: the original page
1892 * reference count, and also a new count of how many pin_user_pages() calls were
1893 * made against the page. ("gup-pinned" is another term for the latter).
1894 *
1895 * With this scheme, pin_user_pages() becomes special: such pages are marked as
1896 * distinct from normal pages. As such, the unpin_user_page() call (and its
1897 * variants) must be used in order to release gup-pinned pages.
1898 *
1899 * Choice of value:
1900 *
1901 * By making GUP_PIN_COUNTING_BIAS a power of two, debugging of page reference
1902 * counts with respect to pin_user_pages() and unpin_user_page() becomes
1903 * simpler, due to the fact that adding an even power of two to the page
1904 * refcount has the effect of using only the upper N bits, for the code that
1905 * counts up using the bias value. This means that the lower bits are left for
1906 * the exclusive use of the original code that increments and decrements by one
1907 * (or at least, by much smaller values than the bias value).
1908 *
1909 * Of course, once the lower bits overflow into the upper bits (and this is
1910 * OK, because subtraction recovers the original values), then visual inspection
1911 * no longer suffices to directly view the separate counts. However, for normal
1912 * applications that don't have huge page reference counts, this won't be an
1913 * issue.
1914 *
1915 * Locking: the lockless algorithm described in folio_try_get_rcu()
1916 * provides safe operation for get_user_pages(), folio_mkclean() and
1917 * other calls that race to set up page table entries.
1918 */
1919#define GUP_PIN_COUNTING_BIAS (1U << 10)
1920
1921void unpin_user_page(struct page *page);
1922void unpin_folio(struct folio *folio);
1923void unpin_user_pages_dirty_lock(struct page **pages, unsigned long npages,
1924 bool make_dirty);
1925void unpin_user_page_range_dirty_lock(struct page *page, unsigned long npages,
1926 bool make_dirty);
1927void unpin_user_pages(struct page **pages, unsigned long npages);
1928void unpin_user_folio(struct folio *folio, unsigned long npages);
1929void unpin_folios(struct folio **folios, unsigned long nfolios);
1930
1931static inline bool is_cow_mapping(vm_flags_t flags)
1932{
1933 return (flags & (VM_SHARED | VM_MAYWRITE)) == VM_MAYWRITE;
1934}
1935
1936static inline bool vma_desc_is_cow_mapping(struct vm_area_desc *desc)
1937{
1938 const vma_flags_t *flags = &desc->vma_flags;
1939
1940 return vma_flags_test(flags, VMA_MAYWRITE_BIT) &&
1941 !vma_flags_test(flags, VMA_SHARED_BIT);
1942}
1943
1944#ifndef CONFIG_MMU
1945static inline bool is_nommu_shared_mapping(vm_flags_t flags)
1946{
1947 /*
1948 * NOMMU shared mappings are ordinary MAP_SHARED mappings and selected
1949 * R/O MAP_PRIVATE file mappings that are an effective R/O overlay of
1950 * a file mapping. R/O MAP_PRIVATE mappings might still modify
1951 * underlying memory if ptrace is active, so this is only possible if
1952 * ptrace does not apply. Note that there is no mprotect() to upgrade
1953 * write permissions later.
1954 */
1955 return flags & (VM_MAYSHARE | VM_MAYOVERLAY);
1956}
1957
1958static inline bool is_nommu_shared_vma_flags(const vma_flags_t *flags)
1959{
1960 return vma_flags_test(flags, VMA_MAYSHARE_BIT, VMA_MAYOVERLAY_BIT);
1961}
1962#endif
1963
1964#if defined(CONFIG_SPARSEMEM) && !defined(CONFIG_SPARSEMEM_VMEMMAP)
1965#define SECTION_IN_PAGE_FLAGS
1966#endif
1967
1968/*
1969 * The identification function is mainly used by the buddy allocator for
1970 * determining if two pages could be buddies. We are not really identifying
1971 * the zone since we could be using the section number id if we do not have
1972 * node id available in page flags.
1973 * We only guarantee that it will return the same value for two combinable
1974 * pages in a zone.
1975 */
1976static inline int page_zone_id(struct page *page)
1977{
1978 return (page->flags.f >> ZONEID_PGSHIFT) & ZONEID_MASK;
1979}
1980
1981#ifdef NODE_NOT_IN_PAGE_FLAGS
1982int memdesc_nid(memdesc_flags_t mdf);
1983#else
1984static inline int memdesc_nid(memdesc_flags_t mdf)
1985{
1986 return (mdf.f >> NODES_PGSHIFT) & NODES_MASK;
1987}
1988#endif
1989
1990static inline int page_to_nid(const struct page *page)
1991{
1992 return memdesc_nid(PF_POISONED_CHECK(page)->flags);
1993}
1994
1995static inline int folio_nid(const struct folio *folio)
1996{
1997 return memdesc_nid(folio->flags);
1998}
1999
2000#ifdef CONFIG_NUMA_BALANCING
2001/* page access time bits needs to hold at least 4 seconds */
2002#define PAGE_ACCESS_TIME_MIN_BITS 12
2003#if LAST_CPUPID_SHIFT < PAGE_ACCESS_TIME_MIN_BITS
2004#define PAGE_ACCESS_TIME_BUCKETS \
2005 (PAGE_ACCESS_TIME_MIN_BITS - LAST_CPUPID_SHIFT)
2006#else
2007#define PAGE_ACCESS_TIME_BUCKETS 0
2008#endif
2009
2010#define PAGE_ACCESS_TIME_MASK \
2011 (LAST_CPUPID_MASK << PAGE_ACCESS_TIME_BUCKETS)
2012
2013static inline int cpu_pid_to_cpupid(int cpu, int pid)
2014{
2015 return ((cpu & LAST__CPU_MASK) << LAST__PID_SHIFT) | (pid & LAST__PID_MASK);
2016}
2017
2018static inline int cpupid_to_pid(int cpupid)
2019{
2020 return cpupid & LAST__PID_MASK;
2021}
2022
2023static inline int cpupid_to_cpu(int cpupid)
2024{
2025 return (cpupid >> LAST__PID_SHIFT) & LAST__CPU_MASK;
2026}
2027
2028static inline int cpupid_to_nid(int cpupid)
2029{
2030 return cpu_to_node(cpupid_to_cpu(cpupid));
2031}
2032
2033static inline bool cpupid_pid_unset(int cpupid)
2034{
2035 return cpupid_to_pid(cpupid) == (-1 & LAST__PID_MASK);
2036}
2037
2038static inline bool cpupid_cpu_unset(int cpupid)
2039{
2040 return cpupid_to_cpu(cpupid) == (-1 & LAST__CPU_MASK);
2041}
2042
2043static inline bool __cpupid_match_pid(pid_t task_pid, int cpupid)
2044{
2045 return (task_pid & LAST__PID_MASK) == cpupid_to_pid(cpupid);
2046}
2047
2048#define cpupid_match_pid(task, cpupid) __cpupid_match_pid(task->pid, cpupid)
2049#ifdef LAST_CPUPID_NOT_IN_PAGE_FLAGS
2050static inline int folio_xchg_last_cpupid(struct folio *folio, int cpupid)
2051{
2052 return xchg(&folio->_last_cpupid, cpupid & LAST_CPUPID_MASK);
2053}
2054
2055static inline int folio_last_cpupid(struct folio *folio)
2056{
2057 return folio->_last_cpupid;
2058}
2059static inline void page_cpupid_reset_last(struct page *page)
2060{
2061 page->_last_cpupid = -1 & LAST_CPUPID_MASK;
2062}
2063#else
2064static inline int folio_last_cpupid(struct folio *folio)
2065{
2066 return (folio->flags.f >> LAST_CPUPID_PGSHIFT) & LAST_CPUPID_MASK;
2067}
2068
2069int folio_xchg_last_cpupid(struct folio *folio, int cpupid);
2070
2071static inline void page_cpupid_reset_last(struct page *page)
2072{
2073 page->flags.f |= LAST_CPUPID_MASK << LAST_CPUPID_PGSHIFT;
2074}
2075#endif /* LAST_CPUPID_NOT_IN_PAGE_FLAGS */
2076
2077static inline int folio_xchg_access_time(struct folio *folio, int time)
2078{
2079 int last_time;
2080
2081 last_time = folio_xchg_last_cpupid(folio,
2082 time >> PAGE_ACCESS_TIME_BUCKETS);
2083 return last_time << PAGE_ACCESS_TIME_BUCKETS;
2084}
2085
2086static inline void vma_set_access_pid_bit(struct vm_area_struct *vma)
2087{
2088 unsigned int pid_bit;
2089
2090 pid_bit = hash_32(current->pid, ilog2(BITS_PER_LONG));
2091 if (vma->numab_state && !test_bit(pid_bit, &vma->numab_state->pids_active[1])) {
2092 __set_bit(pid_bit, &vma->numab_state->pids_active[1]);
2093 }
2094}
2095
2096bool folio_use_access_time(struct folio *folio);
2097#else /* !CONFIG_NUMA_BALANCING */
2098static inline int folio_xchg_last_cpupid(struct folio *folio, int cpupid)
2099{
2100 return folio_nid(folio); /* XXX */
2101}
2102
2103static inline int folio_xchg_access_time(struct folio *folio, int time)
2104{
2105 return 0;
2106}
2107
2108static inline int folio_last_cpupid(struct folio *folio)
2109{
2110 return folio_nid(folio); /* XXX */
2111}
2112
2113static inline int cpupid_to_nid(int cpupid)
2114{
2115 return -1;
2116}
2117
2118static inline int cpupid_to_pid(int cpupid)
2119{
2120 return -1;
2121}
2122
2123static inline int cpupid_to_cpu(int cpupid)
2124{
2125 return -1;
2126}
2127
2128static inline int cpu_pid_to_cpupid(int nid, int pid)
2129{
2130 return -1;
2131}
2132
2133static inline bool cpupid_pid_unset(int cpupid)
2134{
2135 return true;
2136}
2137
2138static inline void page_cpupid_reset_last(struct page *page)
2139{
2140}
2141
2142static inline bool cpupid_match_pid(struct task_struct *task, int cpupid)
2143{
2144 return false;
2145}
2146
2147static inline void vma_set_access_pid_bit(struct vm_area_struct *vma)
2148{
2149}
2150static inline bool folio_use_access_time(struct folio *folio)
2151{
2152 return false;
2153}
2154#endif /* CONFIG_NUMA_BALANCING */
2155
2156#if defined(CONFIG_KASAN_SW_TAGS) || defined(CONFIG_KASAN_HW_TAGS)
2157
2158/*
2159 * KASAN per-page tags are stored xor'ed with 0xff. This allows to avoid
2160 * setting tags for all pages to native kernel tag value 0xff, as the default
2161 * value 0x00 maps to 0xff.
2162 */
2163
2164static inline u8 page_kasan_tag(const struct page *page)
2165{
2166 u8 tag = KASAN_TAG_KERNEL;
2167
2168 if (kasan_enabled()) {
2169 tag = (page->flags.f >> KASAN_TAG_PGSHIFT) & KASAN_TAG_MASK;
2170 tag ^= 0xff;
2171 }
2172
2173 return tag;
2174}
2175
2176static inline void page_kasan_tag_set(struct page *page, u8 tag)
2177{
2178 unsigned long old_flags, flags;
2179
2180 if (!kasan_enabled())
2181 return;
2182
2183 tag ^= 0xff;
2184 old_flags = READ_ONCE(page->flags.f);
2185 do {
2186 flags = old_flags;
2187 flags &= ~(KASAN_TAG_MASK << KASAN_TAG_PGSHIFT);
2188 flags |= (tag & KASAN_TAG_MASK) << KASAN_TAG_PGSHIFT;
2189 } while (unlikely(!try_cmpxchg(&page->flags.f, &old_flags, flags)));
2190}
2191
2192static inline void page_kasan_tag_reset(struct page *page)
2193{
2194 if (kasan_enabled())
2195 page_kasan_tag_set(page, KASAN_TAG_KERNEL);
2196}
2197
2198#else /* CONFIG_KASAN_SW_TAGS || CONFIG_KASAN_HW_TAGS */
2199
2200static inline u8 page_kasan_tag(const struct page *page)
2201{
2202 return 0xff;
2203}
2204
2205static inline void page_kasan_tag_set(struct page *page, u8 tag) { }
2206static inline void page_kasan_tag_reset(struct page *page) { }
2207
2208#endif /* CONFIG_KASAN_SW_TAGS || CONFIG_KASAN_HW_TAGS */
2209
2210static inline struct zone *page_zone(const struct page *page)
2211{
2212 return &NODE_DATA(page_to_nid(page))->node_zones[page_zonenum(page)];
2213}
2214
2215static inline pg_data_t *page_pgdat(const struct page *page)
2216{
2217 return NODE_DATA(page_to_nid(page));
2218}
2219
2220static inline pg_data_t *folio_pgdat(const struct folio *folio)
2221{
2222 return NODE_DATA(folio_nid(folio));
2223}
2224
2225static inline struct zone *folio_zone(const struct folio *folio)
2226{
2227 return &folio_pgdat(folio)->node_zones[folio_zonenum(folio)];
2228}
2229
2230#ifdef SECTION_IN_PAGE_FLAGS
2231static inline void set_page_section(struct page *page, unsigned long section)
2232{
2233 page->flags.f &= ~(SECTIONS_MASK << SECTIONS_PGSHIFT);
2234 page->flags.f |= (section & SECTIONS_MASK) << SECTIONS_PGSHIFT;
2235}
2236
2237static inline unsigned long memdesc_section(memdesc_flags_t mdf)
2238{
2239 return (mdf.f >> SECTIONS_PGSHIFT) & SECTIONS_MASK;
2240}
2241#else /* !SECTION_IN_PAGE_FLAGS */
2242static inline unsigned long memdesc_section(memdesc_flags_t mdf)
2243{
2244 return 0;
2245}
2246#endif /* SECTION_IN_PAGE_FLAGS */
2247
2248/**
2249 * folio_pfn - Return the Page Frame Number of a folio.
2250 * @folio: The folio.
2251 *
2252 * A folio may contain multiple pages. The pages have consecutive
2253 * Page Frame Numbers.
2254 *
2255 * Return: The Page Frame Number of the first page in the folio.
2256 */
2257static inline unsigned long folio_pfn(const struct folio *folio)
2258{
2259 return page_to_pfn(&folio->page);
2260}
2261
2262static inline struct folio *pfn_folio(unsigned long pfn)
2263{
2264 return page_folio(pfn_to_page(pfn));
2265}
2266
2267#ifdef CONFIG_MMU
2268static inline pte_t mk_pte(const struct page *page, pgprot_t pgprot)
2269{
2270 return pfn_pte(page_to_pfn(page), pgprot);
2271}
2272
2273/**
2274 * folio_mk_pte - Create a PTE for this folio
2275 * @folio: The folio to create a PTE for
2276 * @pgprot: The page protection bits to use
2277 *
2278 * Create a page table entry for the first page of this folio.
2279 * This is suitable for passing to set_ptes().
2280 *
2281 * Return: A page table entry suitable for mapping this folio.
2282 */
2283static inline pte_t folio_mk_pte(const struct folio *folio, pgprot_t pgprot)
2284{
2285 return pfn_pte(folio_pfn(folio), pgprot);
2286}
2287
2288#ifdef CONFIG_TRANSPARENT_HUGEPAGE
2289/**
2290 * folio_mk_pmd - Create a PMD for this folio
2291 * @folio: The folio to create a PMD for
2292 * @pgprot: The page protection bits to use
2293 *
2294 * Create a page table entry for the first page of this folio.
2295 * This is suitable for passing to set_pmd_at().
2296 *
2297 * Return: A page table entry suitable for mapping this folio.
2298 */
2299static inline pmd_t folio_mk_pmd(const struct folio *folio, pgprot_t pgprot)
2300{
2301 return pmd_mkhuge(pfn_pmd(folio_pfn(folio), pgprot));
2302}
2303
2304#ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
2305/**
2306 * folio_mk_pud - Create a PUD for this folio
2307 * @folio: The folio to create a PUD for
2308 * @pgprot: The page protection bits to use
2309 *
2310 * Create a page table entry for the first page of this folio.
2311 * This is suitable for passing to set_pud_at().
2312 *
2313 * Return: A page table entry suitable for mapping this folio.
2314 */
2315static inline pud_t folio_mk_pud(const struct folio *folio, pgprot_t pgprot)
2316{
2317 return pud_mkhuge(pfn_pud(folio_pfn(folio), pgprot));
2318}
2319#endif /* CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */
2320#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
2321#endif /* CONFIG_MMU */
2322
2323static inline bool folio_has_pincount(const struct folio *folio)
2324{
2325 if (IS_ENABLED(CONFIG_64BIT))
2326 return folio_test_large(folio);
2327 return folio_order(folio) > 1;
2328}
2329
2330/**
2331 * folio_maybe_dma_pinned - Report if a folio may be pinned for DMA.
2332 * @folio: The folio.
2333 *
2334 * This function checks if a folio has been pinned via a call to
2335 * a function in the pin_user_pages() family.
2336 *
2337 * For small folios, the return value is partially fuzzy: false is not fuzzy,
2338 * because it means "definitely not pinned for DMA", but true means "probably
2339 * pinned for DMA, but possibly a false positive due to having at least
2340 * GUP_PIN_COUNTING_BIAS worth of normal folio references".
2341 *
2342 * False positives are OK, because: a) it's unlikely for a folio to
2343 * get that many refcounts, and b) all the callers of this routine are
2344 * expected to be able to deal gracefully with a false positive.
2345 *
2346 * For most large folios, the result will be exactly correct. That's because
2347 * we have more tracking data available: the _pincount field is used
2348 * instead of the GUP_PIN_COUNTING_BIAS scheme.
2349 *
2350 * For more information, please see Documentation/core-api/pin_user_pages.rst.
2351 *
2352 * Return: True, if it is likely that the folio has been "dma-pinned".
2353 * False, if the folio is definitely not dma-pinned.
2354 */
2355static inline bool folio_maybe_dma_pinned(struct folio *folio)
2356{
2357 if (folio_has_pincount(folio))
2358 return atomic_read(&folio->_pincount) > 0;
2359
2360 /*
2361 * folio_ref_count() is signed. If that refcount overflows, then
2362 * folio_ref_count() returns a negative value, and callers will avoid
2363 * further incrementing the refcount.
2364 *
2365 * Here, for that overflow case, use the sign bit to count a little
2366 * bit higher via unsigned math, and thus still get an accurate result.
2367 */
2368 return ((unsigned int)folio_ref_count(folio)) >=
2369 GUP_PIN_COUNTING_BIAS;
2370}
2371
2372/*
2373 * This should most likely only be called during fork() to see whether we
2374 * should break the cow immediately for an anon page on the src mm.
2375 *
2376 * The caller has to hold the PT lock and the vma->vm_mm->->write_protect_seq.
2377 */
2378static inline bool folio_needs_cow_for_dma(struct vm_area_struct *vma,
2379 struct folio *folio)
2380{
2381 VM_BUG_ON(!(raw_read_seqcount(&vma->vm_mm->write_protect_seq) & 1));
2382
2383 if (!mm_flags_test(MMF_HAS_PINNED, vma->vm_mm))
2384 return false;
2385
2386 return folio_maybe_dma_pinned(folio);
2387}
2388
2389/**
2390 * is_zero_page - Query if a page is a zero page
2391 * @page: The page to query
2392 *
2393 * This returns true if @page is one of the permanent zero pages.
2394 */
2395static inline bool is_zero_page(const struct page *page)
2396{
2397 return is_zero_pfn(page_to_pfn(page));
2398}
2399
2400/**
2401 * is_zero_folio - Query if a folio is a zero page
2402 * @folio: The folio to query
2403 *
2404 * This returns true if @folio is one of the permanent zero pages.
2405 */
2406static inline bool is_zero_folio(const struct folio *folio)
2407{
2408 return is_zero_page(&folio->page);
2409}
2410
2411/* MIGRATE_CMA and ZONE_MOVABLE do not allow pin folios */
2412#ifdef CONFIG_MIGRATION
2413static inline bool folio_is_longterm_pinnable(struct folio *folio)
2414{
2415#ifdef CONFIG_CMA
2416 int mt = folio_migratetype(folio);
2417
2418 if (mt == MIGRATE_CMA || mt == MIGRATE_ISOLATE)
2419 return false;
2420#endif
2421 /* The zero page can be "pinned" but gets special handling. */
2422 if (is_zero_folio(folio))
2423 return true;
2424
2425 /* Coherent device memory must always allow eviction. */
2426 if (folio_is_device_coherent(folio))
2427 return false;
2428
2429 /*
2430 * Filesystems can only tolerate transient delays to truncate and
2431 * hole-punch operations
2432 */
2433 if (folio_is_fsdax(folio))
2434 return false;
2435
2436 /* Otherwise, non-movable zone folios can be pinned. */
2437 return !folio_is_zone_movable(folio);
2438
2439}
2440#else
2441static inline bool folio_is_longterm_pinnable(struct folio *folio)
2442{
2443 return true;
2444}
2445#endif
2446
2447static inline void set_page_zone(struct page *page, enum zone_type zone)
2448{
2449 page->flags.f &= ~(ZONES_MASK << ZONES_PGSHIFT);
2450 page->flags.f |= (zone & ZONES_MASK) << ZONES_PGSHIFT;
2451}
2452
2453static inline void set_page_node(struct page *page, unsigned long node)
2454{
2455 page->flags.f &= ~(NODES_MASK << NODES_PGSHIFT);
2456 page->flags.f |= (node & NODES_MASK) << NODES_PGSHIFT;
2457}
2458
2459static inline void set_page_links(struct page *page, enum zone_type zone,
2460 unsigned long node, unsigned long pfn)
2461{
2462 set_page_zone(page, zone);
2463 set_page_node(page, node);
2464#ifdef SECTION_IN_PAGE_FLAGS
2465 set_page_section(page, pfn_to_section_nr(pfn));
2466#endif
2467}
2468
2469/**
2470 * folio_nr_pages - The number of pages in the folio.
2471 * @folio: The folio.
2472 *
2473 * Return: A positive power of two.
2474 */
2475static inline unsigned long folio_nr_pages(const struct folio *folio)
2476{
2477 if (!folio_test_large(folio))
2478 return 1;
2479 return folio_large_nr_pages(folio);
2480}
2481
2482#if !defined(CONFIG_HAVE_GIGANTIC_FOLIOS)
2483/*
2484 * We don't expect any folios that exceed buddy sizes (and consequently
2485 * memory sections).
2486 */
2487#define MAX_FOLIO_ORDER MAX_PAGE_ORDER
2488#elif defined(CONFIG_SPARSEMEM) && !defined(CONFIG_SPARSEMEM_VMEMMAP)
2489/*
2490 * Only pages within a single memory section are guaranteed to be
2491 * contiguous. By limiting folios to a single memory section, all folio
2492 * pages are guaranteed to be contiguous.
2493 */
2494#define MAX_FOLIO_ORDER PFN_SECTION_SHIFT
2495#elif defined(CONFIG_HUGETLB_PAGE)
2496/*
2497 * There is no real limit on the folio size. We limit them to the maximum we
2498 * currently expect (see CONFIG_HAVE_GIGANTIC_FOLIOS): with hugetlb, we expect
2499 * no folios larger than 16 GiB on 64bit and 1 GiB on 32bit.
2500 */
2501#define MAX_FOLIO_ORDER get_order(IS_ENABLED(CONFIG_64BIT) ? SZ_16G : SZ_1G)
2502#else
2503/*
2504 * Without hugetlb, gigantic folios that are bigger than a single PUD are
2505 * currently impossible.
2506 */
2507#define MAX_FOLIO_ORDER PUD_ORDER
2508#endif
2509
2510#define MAX_FOLIO_NR_PAGES (1UL << MAX_FOLIO_ORDER)
2511
2512/*
2513 * compound_nr() returns the number of pages in this potentially compound
2514 * page. compound_nr() can be called on a tail page, and is defined to
2515 * return 1 in that case.
2516 */
2517static inline unsigned long compound_nr(const struct page *page)
2518{
2519 const struct folio *folio = (struct folio *)page;
2520
2521 if (!test_bit(PG_head, &folio->flags.f))
2522 return 1;
2523 return folio_large_nr_pages(folio);
2524}
2525
2526/**
2527 * folio_next - Move to the next physical folio.
2528 * @folio: The folio we're currently operating on.
2529 *
2530 * If you have physically contiguous memory which may span more than
2531 * one folio (eg a &struct bio_vec), use this function to move from one
2532 * folio to the next. Do not use it if the memory is only virtually
2533 * contiguous as the folios are almost certainly not adjacent to each
2534 * other. This is the folio equivalent to writing ``page++``.
2535 *
2536 * Context: We assume that the folios are refcounted and/or locked at a
2537 * higher level and do not adjust the reference counts.
2538 * Return: The next struct folio.
2539 */
2540static inline struct folio *folio_next(struct folio *folio)
2541{
2542 return (struct folio *)folio_page(folio, folio_nr_pages(folio));
2543}
2544
2545/**
2546 * folio_shift - The size of the memory described by this folio.
2547 * @folio: The folio.
2548 *
2549 * A folio represents a number of bytes which is a power-of-two in size.
2550 * This function tells you which power-of-two the folio is. See also
2551 * folio_size() and folio_order().
2552 *
2553 * Context: The caller should have a reference on the folio to prevent
2554 * it from being split. It is not necessary for the folio to be locked.
2555 * Return: The base-2 logarithm of the size of this folio.
2556 */
2557static inline unsigned int folio_shift(const struct folio *folio)
2558{
2559 return PAGE_SHIFT + folio_order(folio);
2560}
2561
2562/**
2563 * folio_size - The number of bytes in a folio.
2564 * @folio: The folio.
2565 *
2566 * Context: The caller should have a reference on the folio to prevent
2567 * it from being split. It is not necessary for the folio to be locked.
2568 * Return: The number of bytes in this folio.
2569 */
2570static inline size_t folio_size(const struct folio *folio)
2571{
2572 return PAGE_SIZE << folio_order(folio);
2573}
2574
2575/**
2576 * folio_maybe_mapped_shared - Whether the folio is mapped into the page
2577 * tables of more than one MM
2578 * @folio: The folio.
2579 *
2580 * This function checks if the folio maybe currently mapped into more than one
2581 * MM ("maybe mapped shared"), or if the folio is certainly mapped into a single
2582 * MM ("mapped exclusively").
2583 *
2584 * For KSM folios, this function also returns "mapped shared" when a folio is
2585 * mapped multiple times into the same MM, because the individual page mappings
2586 * are independent.
2587 *
2588 * For small anonymous folios and anonymous hugetlb folios, the return
2589 * value will be exactly correct: non-KSM folios can only be mapped at most once
2590 * into an MM, and they cannot be partially mapped. KSM folios are
2591 * considered shared even if mapped multiple times into the same MM.
2592 *
2593 * For other folios, the result can be fuzzy:
2594 * #. For partially-mappable large folios (THP), the return value can wrongly
2595 * indicate "mapped shared" (false positive) if a folio was mapped by
2596 * more than two MMs at one point in time.
2597 * #. For pagecache folios (including hugetlb), the return value can wrongly
2598 * indicate "mapped shared" (false positive) when two VMAs in the same MM
2599 * cover the same file range.
2600 *
2601 * Further, this function only considers current page table mappings that
2602 * are tracked using the folio mapcount(s).
2603 *
2604 * This function does not consider:
2605 * #. If the folio might get mapped in the (near) future (e.g., swapcache,
2606 * pagecache, temporary unmapping for migration).
2607 * #. If the folio is mapped differently (VM_PFNMAP).
2608 * #. If hugetlb page table sharing applies. Callers might want to check
2609 * hugetlb_pmd_shared().
2610 *
2611 * Return: Whether the folio is estimated to be mapped into more than one MM.
2612 */
2613static inline bool folio_maybe_mapped_shared(struct folio *folio)
2614{
2615 int mapcount = folio_mapcount(folio);
2616
2617 /* Only partially-mappable folios require more care. */
2618 if (!folio_test_large(folio) || unlikely(folio_test_hugetlb(folio)))
2619 return mapcount > 1;
2620
2621 /*
2622 * vm_insert_page() without CONFIG_TRANSPARENT_HUGEPAGE ...
2623 * simply assume "mapped shared", nobody should really care
2624 * about this for arbitrary kernel allocations.
2625 */
2626 if (!IS_ENABLED(CONFIG_MM_ID))
2627 return true;
2628
2629 /*
2630 * A single mapping implies "mapped exclusively", even if the
2631 * folio flag says something different: it's easier to handle this
2632 * case here instead of on the RMAP hot path.
2633 */
2634 if (mapcount <= 1)
2635 return false;
2636 return test_bit(FOLIO_MM_IDS_SHARED_BITNUM, &folio->_mm_ids);
2637}
2638
2639/**
2640 * folio_expected_ref_count - calculate the expected folio refcount
2641 * @folio: the folio
2642 *
2643 * Calculate the expected folio refcount, taking references from the pagecache,
2644 * swapcache, PG_private and page table mappings into account. Useful in
2645 * combination with folio_ref_count() to detect unexpected references (e.g.,
2646 * GUP or other temporary references).
2647 *
2648 * Does currently not consider references from the LRU cache. If the folio
2649 * was isolated from the LRU (which is the case during migration or split),
2650 * the LRU cache does not apply.
2651 *
2652 * Calling this function on an unmapped folio -- !folio_mapped() -- that is
2653 * locked will return a stable result.
2654 *
2655 * Calling this function on a mapped folio will not result in a stable result,
2656 * because nothing stops additional page table mappings from coming (e.g.,
2657 * fork()) or going (e.g., munmap()).
2658 *
2659 * Calling this function without the folio lock will also not result in a
2660 * stable result: for example, the folio might get dropped from the swapcache
2661 * concurrently.
2662 *
2663 * However, even when called without the folio lock or on a mapped folio,
2664 * this function can be used to detect unexpected references early (for example,
2665 * if it makes sense to even lock the folio and unmap it).
2666 *
2667 * The caller must add any reference (e.g., from folio_try_get()) it might be
2668 * holding itself to the result.
2669 *
2670 * Returns the expected folio refcount.
2671 */
2672static inline int folio_expected_ref_count(const struct folio *folio)
2673{
2674 const int order = folio_order(folio);
2675 int ref_count = 0;
2676
2677 if (WARN_ON_ONCE(page_has_type(&folio->page) && !folio_test_hugetlb(folio)))
2678 return 0;
2679
2680 /* One reference per page from the swapcache. */
2681 ref_count += folio_test_swapcache(folio) << order;
2682
2683 if (!folio_test_anon(folio)) {
2684 /* One reference per page from the pagecache. */
2685 ref_count += !!folio->mapping << order;
2686 /* One reference from PG_private. */
2687 ref_count += folio_test_private(folio);
2688 }
2689
2690 /* One reference per page table mapping. */
2691 return ref_count + folio_mapcount(folio);
2692}
2693
2694#ifndef HAVE_ARCH_MAKE_FOLIO_ACCESSIBLE
2695static inline int arch_make_folio_accessible(struct folio *folio)
2696{
2697 return 0;
2698}
2699#endif
2700
2701/*
2702 * Some inline functions in vmstat.h depend on page_zone()
2703 */
2704#include <linux/vmstat.h>
2705
2706#if defined(CONFIG_HIGHMEM) && !defined(WANT_PAGE_VIRTUAL)
2707#define HASHED_PAGE_VIRTUAL
2708#endif
2709
2710#if defined(WANT_PAGE_VIRTUAL)
2711static inline void *page_address(const struct page *page)
2712{
2713 return page->virtual;
2714}
2715static inline void set_page_address(struct page *page, void *address)
2716{
2717 page->virtual = address;
2718}
2719#define page_address_init() do { } while(0)
2720#endif
2721
2722#if defined(HASHED_PAGE_VIRTUAL)
2723void *page_address(const struct page *page);
2724void set_page_address(struct page *page, void *virtual);
2725void page_address_init(void);
2726#endif
2727
2728static __always_inline void *lowmem_page_address(const struct page *page)
2729{
2730 return page_to_virt(page);
2731}
2732
2733#if !defined(HASHED_PAGE_VIRTUAL) && !defined(WANT_PAGE_VIRTUAL)
2734#define page_address(page) lowmem_page_address(page)
2735#define set_page_address(page, address) do { } while(0)
2736#define page_address_init() do { } while(0)
2737#endif
2738
2739static inline void *folio_address(const struct folio *folio)
2740{
2741 return page_address(&folio->page);
2742}
2743
2744/*
2745 * Return true only if the page has been allocated with
2746 * ALLOC_NO_WATERMARKS and the low watermark was not
2747 * met implying that the system is under some pressure.
2748 */
2749static inline bool page_is_pfmemalloc(const struct page *page)
2750{
2751 /*
2752 * lru.next has bit 1 set if the page is allocated from the
2753 * pfmemalloc reserves. Callers may simply overwrite it if
2754 * they do not need to preserve that information.
2755 */
2756 return (uintptr_t)page->lru.next & BIT(1);
2757}
2758
2759/*
2760 * Return true only if the folio has been allocated with
2761 * ALLOC_NO_WATERMARKS and the low watermark was not
2762 * met implying that the system is under some pressure.
2763 */
2764static inline bool folio_is_pfmemalloc(const struct folio *folio)
2765{
2766 /*
2767 * lru.next has bit 1 set if the page is allocated from the
2768 * pfmemalloc reserves. Callers may simply overwrite it if
2769 * they do not need to preserve that information.
2770 */
2771 return (uintptr_t)folio->lru.next & BIT(1);
2772}
2773
2774/*
2775 * Only to be called by the page allocator on a freshly allocated
2776 * page.
2777 */
2778static inline void set_page_pfmemalloc(struct page *page)
2779{
2780 page->lru.next = (void *)BIT(1);
2781}
2782
2783static inline void clear_page_pfmemalloc(struct page *page)
2784{
2785 page->lru.next = NULL;
2786}
2787
2788/*
2789 * Can be called by the pagefault handler when it gets a VM_FAULT_OOM.
2790 */
2791extern void pagefault_out_of_memory(void);
2792
2793#define offset_in_page(p) ((unsigned long)(p) & ~PAGE_MASK)
2794#define offset_in_folio(folio, p) ((unsigned long)(p) & (folio_size(folio) - 1))
2795
2796/*
2797 * Parameter block passed down to zap_pte_range in exceptional cases.
2798 */
2799struct zap_details {
2800 struct folio *single_folio; /* Locked folio to be unmapped */
2801 bool even_cows; /* Zap COWed private pages too? */
2802 bool reclaim_pt; /* Need reclaim page tables? */
2803 zap_flags_t zap_flags; /* Extra flags for zapping */
2804};
2805
2806/*
2807 * Whether to drop the pte markers, for example, the uffd-wp information for
2808 * file-backed memory. This should only be specified when we will completely
2809 * drop the page in the mm, either by truncation or unmapping of the vma. By
2810 * default, the flag is not set.
2811 */
2812#define ZAP_FLAG_DROP_MARKER ((__force zap_flags_t) BIT(0))
2813/* Set in unmap_vmas() to indicate a final unmap call. Only used by hugetlb */
2814#define ZAP_FLAG_UNMAP ((__force zap_flags_t) BIT(1))
2815
2816#ifdef CONFIG_MMU
2817extern bool can_do_mlock(void);
2818#else
2819static inline bool can_do_mlock(void) { return false; }
2820#endif
2821extern int user_shm_lock(size_t, struct ucounts *);
2822extern void user_shm_unlock(size_t, struct ucounts *);
2823
2824struct folio *vm_normal_folio(struct vm_area_struct *vma, unsigned long addr,
2825 pte_t pte);
2826struct page *vm_normal_page(struct vm_area_struct *vma, unsigned long addr,
2827 pte_t pte);
2828struct folio *vm_normal_folio_pmd(struct vm_area_struct *vma,
2829 unsigned long addr, pmd_t pmd);
2830struct page *vm_normal_page_pmd(struct vm_area_struct *vma, unsigned long addr,
2831 pmd_t pmd);
2832struct page *vm_normal_page_pud(struct vm_area_struct *vma, unsigned long addr,
2833 pud_t pud);
2834
2835void zap_vma_ptes(struct vm_area_struct *vma, unsigned long address,
2836 unsigned long size);
2837void zap_page_range_single(struct vm_area_struct *vma, unsigned long address,
2838 unsigned long size, struct zap_details *details);
2839static inline void zap_vma_pages(struct vm_area_struct *vma)
2840{
2841 zap_page_range_single(vma, vma->vm_start,
2842 vma->vm_end - vma->vm_start, NULL);
2843}
2844struct mmu_notifier_range;
2845
2846void free_pgd_range(struct mmu_gather *tlb, unsigned long addr,
2847 unsigned long end, unsigned long floor, unsigned long ceiling);
2848int
2849copy_page_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma);
2850int generic_access_phys(struct vm_area_struct *vma, unsigned long addr,
2851 void *buf, int len, int write);
2852
2853struct follow_pfnmap_args {
2854 /**
2855 * Inputs:
2856 * @vma: Pointer to @vm_area_struct struct
2857 * @address: the virtual address to walk
2858 */
2859 struct vm_area_struct *vma;
2860 unsigned long address;
2861 /**
2862 * Internals:
2863 *
2864 * The caller shouldn't touch any of these.
2865 */
2866 spinlock_t *lock;
2867 pte_t *ptep;
2868 /**
2869 * Outputs:
2870 *
2871 * @pfn: the PFN of the address
2872 * @addr_mask: address mask covering pfn
2873 * @pgprot: the pgprot_t of the mapping
2874 * @writable: whether the mapping is writable
2875 * @special: whether the mapping is a special mapping (real PFN maps)
2876 */
2877 unsigned long pfn;
2878 unsigned long addr_mask;
2879 pgprot_t pgprot;
2880 bool writable;
2881 bool special;
2882};
2883int follow_pfnmap_start(struct follow_pfnmap_args *args);
2884void follow_pfnmap_end(struct follow_pfnmap_args *args);
2885
2886extern void truncate_pagecache(struct inode *inode, loff_t new);
2887extern void truncate_setsize(struct inode *inode, loff_t newsize);
2888void pagecache_isize_extended(struct inode *inode, loff_t from, loff_t to);
2889void truncate_pagecache_range(struct inode *inode, loff_t offset, loff_t end);
2890int generic_error_remove_folio(struct address_space *mapping,
2891 struct folio *folio);
2892
2893struct vm_area_struct *lock_mm_and_find_vma(struct mm_struct *mm,
2894 unsigned long address, struct pt_regs *regs);
2895
2896#ifdef CONFIG_MMU
2897extern vm_fault_t handle_mm_fault(struct vm_area_struct *vma,
2898 unsigned long address, unsigned int flags,
2899 struct pt_regs *regs);
2900extern int fixup_user_fault(struct mm_struct *mm,
2901 unsigned long address, unsigned int fault_flags,
2902 bool *unlocked);
2903void unmap_mapping_pages(struct address_space *mapping,
2904 pgoff_t start, pgoff_t nr, bool even_cows);
2905void unmap_mapping_range(struct address_space *mapping,
2906 loff_t const holebegin, loff_t const holelen, int even_cows);
2907#else
2908static inline vm_fault_t handle_mm_fault(struct vm_area_struct *vma,
2909 unsigned long address, unsigned int flags,
2910 struct pt_regs *regs)
2911{
2912 /* should never happen if there's no MMU */
2913 BUG();
2914 return VM_FAULT_SIGBUS;
2915}
2916static inline int fixup_user_fault(struct mm_struct *mm, unsigned long address,
2917 unsigned int fault_flags, bool *unlocked)
2918{
2919 /* should never happen if there's no MMU */
2920 BUG();
2921 return -EFAULT;
2922}
2923static inline void unmap_mapping_pages(struct address_space *mapping,
2924 pgoff_t start, pgoff_t nr, bool even_cows) { }
2925static inline void unmap_mapping_range(struct address_space *mapping,
2926 loff_t const holebegin, loff_t const holelen, int even_cows) { }
2927#endif
2928
2929static inline void unmap_shared_mapping_range(struct address_space *mapping,
2930 loff_t const holebegin, loff_t const holelen)
2931{
2932 unmap_mapping_range(mapping, holebegin, holelen, 0);
2933}
2934
2935static inline struct vm_area_struct *vma_lookup(struct mm_struct *mm,
2936 unsigned long addr);
2937
2938extern int access_process_vm(struct task_struct *tsk, unsigned long addr,
2939 void *buf, int len, unsigned int gup_flags);
2940extern int access_remote_vm(struct mm_struct *mm, unsigned long addr,
2941 void *buf, int len, unsigned int gup_flags);
2942
2943#ifdef CONFIG_BPF_SYSCALL
2944extern int copy_remote_vm_str(struct task_struct *tsk, unsigned long addr,
2945 void *buf, int len, unsigned int gup_flags);
2946#endif
2947
2948long get_user_pages_remote(struct mm_struct *mm,
2949 unsigned long start, unsigned long nr_pages,
2950 unsigned int gup_flags, struct page **pages,
2951 int *locked);
2952long pin_user_pages_remote(struct mm_struct *mm,
2953 unsigned long start, unsigned long nr_pages,
2954 unsigned int gup_flags, struct page **pages,
2955 int *locked);
2956
2957/*
2958 * Retrieves a single page alongside its VMA. Does not support FOLL_NOWAIT.
2959 */
2960static inline struct page *get_user_page_vma_remote(struct mm_struct *mm,
2961 unsigned long addr,
2962 int gup_flags,
2963 struct vm_area_struct **vmap)
2964{
2965 struct page *page;
2966 struct vm_area_struct *vma;
2967 int got;
2968
2969 if (WARN_ON_ONCE(unlikely(gup_flags & FOLL_NOWAIT)))
2970 return ERR_PTR(-EINVAL);
2971
2972 got = get_user_pages_remote(mm, addr, 1, gup_flags, &page, NULL);
2973
2974 if (got < 0)
2975 return ERR_PTR(got);
2976
2977 vma = vma_lookup(mm, addr);
2978 if (WARN_ON_ONCE(!vma)) {
2979 put_page(page);
2980 return ERR_PTR(-EINVAL);
2981 }
2982
2983 *vmap = vma;
2984 return page;
2985}
2986
2987long get_user_pages(unsigned long start, unsigned long nr_pages,
2988 unsigned int gup_flags, struct page **pages);
2989long pin_user_pages(unsigned long start, unsigned long nr_pages,
2990 unsigned int gup_flags, struct page **pages);
2991long get_user_pages_unlocked(unsigned long start, unsigned long nr_pages,
2992 struct page **pages, unsigned int gup_flags);
2993long pin_user_pages_unlocked(unsigned long start, unsigned long nr_pages,
2994 struct page **pages, unsigned int gup_flags);
2995long memfd_pin_folios(struct file *memfd, loff_t start, loff_t end,
2996 struct folio **folios, unsigned int max_folios,
2997 pgoff_t *offset);
2998int folio_add_pins(struct folio *folio, unsigned int pins);
2999
3000int get_user_pages_fast(unsigned long start, int nr_pages,
3001 unsigned int gup_flags, struct page **pages);
3002int pin_user_pages_fast(unsigned long start, int nr_pages,
3003 unsigned int gup_flags, struct page **pages);
3004void folio_add_pin(struct folio *folio);
3005
3006int account_locked_vm(struct mm_struct *mm, unsigned long pages, bool inc);
3007int __account_locked_vm(struct mm_struct *mm, unsigned long pages, bool inc,
3008 const struct task_struct *task, bool bypass_rlim);
3009
3010struct kvec;
3011struct page *get_dump_page(unsigned long addr, int *locked);
3012
3013bool folio_mark_dirty(struct folio *folio);
3014bool folio_mark_dirty_lock(struct folio *folio);
3015bool set_page_dirty(struct page *page);
3016int set_page_dirty_lock(struct page *page);
3017
3018int get_cmdline(struct task_struct *task, char *buffer, int buflen);
3019
3020/*
3021 * Flags used by change_protection(). For now we make it a bitmap so
3022 * that we can pass in multiple flags just like parameters. However
3023 * for now all the callers are only use one of the flags at the same
3024 * time.
3025 */
3026/*
3027 * Whether we should manually check if we can map individual PTEs writable,
3028 * because something (e.g., COW, uffd-wp) blocks that from happening for all
3029 * PTEs automatically in a writable mapping.
3030 */
3031#define MM_CP_TRY_CHANGE_WRITABLE (1UL << 0)
3032/* Whether this protection change is for NUMA hints */
3033#define MM_CP_PROT_NUMA (1UL << 1)
3034/* Whether this change is for write protecting */
3035#define MM_CP_UFFD_WP (1UL << 2) /* do wp */
3036#define MM_CP_UFFD_WP_RESOLVE (1UL << 3) /* Resolve wp */
3037#define MM_CP_UFFD_WP_ALL (MM_CP_UFFD_WP | \
3038 MM_CP_UFFD_WP_RESOLVE)
3039
3040bool can_change_pte_writable(struct vm_area_struct *vma, unsigned long addr,
3041 pte_t pte);
3042extern long change_protection(struct mmu_gather *tlb,
3043 struct vm_area_struct *vma, unsigned long start,
3044 unsigned long end, unsigned long cp_flags);
3045extern int mprotect_fixup(struct vma_iterator *vmi, struct mmu_gather *tlb,
3046 struct vm_area_struct *vma, struct vm_area_struct **pprev,
3047 unsigned long start, unsigned long end, vm_flags_t newflags);
3048
3049/*
3050 * doesn't attempt to fault and will return short.
3051 */
3052int get_user_pages_fast_only(unsigned long start, int nr_pages,
3053 unsigned int gup_flags, struct page **pages);
3054
3055static inline bool get_user_page_fast_only(unsigned long addr,
3056 unsigned int gup_flags, struct page **pagep)
3057{
3058 return get_user_pages_fast_only(addr, 1, gup_flags, pagep) == 1;
3059}
3060/*
3061 * per-process(per-mm_struct) statistics.
3062 */
3063static inline unsigned long get_mm_counter(struct mm_struct *mm, int member)
3064{
3065 return percpu_counter_read_positive(&mm->rss_stat[member]);
3066}
3067
3068static inline unsigned long get_mm_counter_sum(struct mm_struct *mm, int member)
3069{
3070 return percpu_counter_sum_positive(&mm->rss_stat[member]);
3071}
3072
3073void mm_trace_rss_stat(struct mm_struct *mm, int member);
3074
3075static inline void add_mm_counter(struct mm_struct *mm, int member, long value)
3076{
3077 percpu_counter_add(&mm->rss_stat[member], value);
3078
3079 mm_trace_rss_stat(mm, member);
3080}
3081
3082static inline void inc_mm_counter(struct mm_struct *mm, int member)
3083{
3084 percpu_counter_inc(&mm->rss_stat[member]);
3085
3086 mm_trace_rss_stat(mm, member);
3087}
3088
3089static inline void dec_mm_counter(struct mm_struct *mm, int member)
3090{
3091 percpu_counter_dec(&mm->rss_stat[member]);
3092
3093 mm_trace_rss_stat(mm, member);
3094}
3095
3096/* Optimized variant when folio is already known not to be anon */
3097static inline int mm_counter_file(struct folio *folio)
3098{
3099 if (folio_test_swapbacked(folio))
3100 return MM_SHMEMPAGES;
3101 return MM_FILEPAGES;
3102}
3103
3104static inline int mm_counter(struct folio *folio)
3105{
3106 if (folio_test_anon(folio))
3107 return MM_ANONPAGES;
3108 return mm_counter_file(folio);
3109}
3110
3111static inline unsigned long get_mm_rss(struct mm_struct *mm)
3112{
3113 return get_mm_counter(mm, MM_FILEPAGES) +
3114 get_mm_counter(mm, MM_ANONPAGES) +
3115 get_mm_counter(mm, MM_SHMEMPAGES);
3116}
3117
3118static inline unsigned long get_mm_rss_sum(struct mm_struct *mm)
3119{
3120 return get_mm_counter_sum(mm, MM_FILEPAGES) +
3121 get_mm_counter_sum(mm, MM_ANONPAGES) +
3122 get_mm_counter_sum(mm, MM_SHMEMPAGES);
3123}
3124
3125static inline unsigned long get_mm_hiwater_rss(struct mm_struct *mm)
3126{
3127 return max(mm->hiwater_rss, get_mm_rss(mm));
3128}
3129
3130static inline unsigned long get_mm_hiwater_vm(struct mm_struct *mm)
3131{
3132 return max(mm->hiwater_vm, mm->total_vm);
3133}
3134
3135static inline void update_hiwater_rss(struct mm_struct *mm)
3136{
3137 unsigned long _rss = get_mm_rss(mm);
3138
3139 if (data_race(mm->hiwater_rss) < _rss)
3140 data_race(mm->hiwater_rss = _rss);
3141}
3142
3143static inline void update_hiwater_vm(struct mm_struct *mm)
3144{
3145 if (mm->hiwater_vm < mm->total_vm)
3146 mm->hiwater_vm = mm->total_vm;
3147}
3148
3149static inline void reset_mm_hiwater_rss(struct mm_struct *mm)
3150{
3151 mm->hiwater_rss = get_mm_rss(mm);
3152}
3153
3154static inline void setmax_mm_hiwater_rss(unsigned long *maxrss,
3155 struct mm_struct *mm)
3156{
3157 unsigned long hiwater_rss = get_mm_hiwater_rss(mm);
3158
3159 if (*maxrss < hiwater_rss)
3160 *maxrss = hiwater_rss;
3161}
3162
3163#ifndef CONFIG_ARCH_HAS_PTE_SPECIAL
3164static inline int pte_special(pte_t pte)
3165{
3166 return 0;
3167}
3168
3169static inline pte_t pte_mkspecial(pte_t pte)
3170{
3171 return pte;
3172}
3173#endif
3174
3175#ifndef CONFIG_ARCH_SUPPORTS_PMD_PFNMAP
3176static inline bool pmd_special(pmd_t pmd)
3177{
3178 return false;
3179}
3180
3181static inline pmd_t pmd_mkspecial(pmd_t pmd)
3182{
3183 return pmd;
3184}
3185#endif /* CONFIG_ARCH_SUPPORTS_PMD_PFNMAP */
3186
3187#ifndef CONFIG_ARCH_SUPPORTS_PUD_PFNMAP
3188static inline bool pud_special(pud_t pud)
3189{
3190 return false;
3191}
3192
3193static inline pud_t pud_mkspecial(pud_t pud)
3194{
3195 return pud;
3196}
3197#endif /* CONFIG_ARCH_SUPPORTS_PUD_PFNMAP */
3198
3199extern pte_t *get_locked_pte(struct mm_struct *mm, unsigned long addr,
3200 spinlock_t **ptl);
3201
3202#ifdef __PAGETABLE_P4D_FOLDED
3203static inline int __p4d_alloc(struct mm_struct *mm, pgd_t *pgd,
3204 unsigned long address)
3205{
3206 return 0;
3207}
3208#else
3209int __p4d_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address);
3210#endif
3211
3212#if defined(__PAGETABLE_PUD_FOLDED) || !defined(CONFIG_MMU)
3213static inline int __pud_alloc(struct mm_struct *mm, p4d_t *p4d,
3214 unsigned long address)
3215{
3216 return 0;
3217}
3218static inline void mm_inc_nr_puds(struct mm_struct *mm) {}
3219static inline void mm_dec_nr_puds(struct mm_struct *mm) {}
3220
3221#else
3222int __pud_alloc(struct mm_struct *mm, p4d_t *p4d, unsigned long address);
3223
3224static inline void mm_inc_nr_puds(struct mm_struct *mm)
3225{
3226 if (mm_pud_folded(mm))
3227 return;
3228 atomic_long_add(PTRS_PER_PUD * sizeof(pud_t), &mm->pgtables_bytes);
3229}
3230
3231static inline void mm_dec_nr_puds(struct mm_struct *mm)
3232{
3233 if (mm_pud_folded(mm))
3234 return;
3235 atomic_long_sub(PTRS_PER_PUD * sizeof(pud_t), &mm->pgtables_bytes);
3236}
3237#endif
3238
3239#if defined(__PAGETABLE_PMD_FOLDED) || !defined(CONFIG_MMU)
3240static inline int __pmd_alloc(struct mm_struct *mm, pud_t *pud,
3241 unsigned long address)
3242{
3243 return 0;
3244}
3245
3246static inline void mm_inc_nr_pmds(struct mm_struct *mm) {}
3247static inline void mm_dec_nr_pmds(struct mm_struct *mm) {}
3248
3249#else
3250int __pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address);
3251
3252static inline void mm_inc_nr_pmds(struct mm_struct *mm)
3253{
3254 if (mm_pmd_folded(mm))
3255 return;
3256 atomic_long_add(PTRS_PER_PMD * sizeof(pmd_t), &mm->pgtables_bytes);
3257}
3258
3259static inline void mm_dec_nr_pmds(struct mm_struct *mm)
3260{
3261 if (mm_pmd_folded(mm))
3262 return;
3263 atomic_long_sub(PTRS_PER_PMD * sizeof(pmd_t), &mm->pgtables_bytes);
3264}
3265#endif
3266
3267#ifdef CONFIG_MMU
3268static inline void mm_pgtables_bytes_init(struct mm_struct *mm)
3269{
3270 atomic_long_set(&mm->pgtables_bytes, 0);
3271}
3272
3273static inline unsigned long mm_pgtables_bytes(const struct mm_struct *mm)
3274{
3275 return atomic_long_read(&mm->pgtables_bytes);
3276}
3277
3278static inline void mm_inc_nr_ptes(struct mm_struct *mm)
3279{
3280 atomic_long_add(PTRS_PER_PTE * sizeof(pte_t), &mm->pgtables_bytes);
3281}
3282
3283static inline void mm_dec_nr_ptes(struct mm_struct *mm)
3284{
3285 atomic_long_sub(PTRS_PER_PTE * sizeof(pte_t), &mm->pgtables_bytes);
3286}
3287#else
3288
3289static inline void mm_pgtables_bytes_init(struct mm_struct *mm) {}
3290static inline unsigned long mm_pgtables_bytes(const struct mm_struct *mm)
3291{
3292 return 0;
3293}
3294
3295static inline void mm_inc_nr_ptes(struct mm_struct *mm) {}
3296static inline void mm_dec_nr_ptes(struct mm_struct *mm) {}
3297#endif
3298
3299int __pte_alloc(struct mm_struct *mm, pmd_t *pmd);
3300int __pte_alloc_kernel(pmd_t *pmd);
3301
3302#if defined(CONFIG_MMU)
3303
3304static inline p4d_t *p4d_alloc(struct mm_struct *mm, pgd_t *pgd,
3305 unsigned long address)
3306{
3307 return (unlikely(pgd_none(*pgd)) && __p4d_alloc(mm, pgd, address)) ?
3308 NULL : p4d_offset(pgd, address);
3309}
3310
3311static inline pud_t *pud_alloc(struct mm_struct *mm, p4d_t *p4d,
3312 unsigned long address)
3313{
3314 return (unlikely(p4d_none(*p4d)) && __pud_alloc(mm, p4d, address)) ?
3315 NULL : pud_offset(p4d, address);
3316}
3317
3318static inline pmd_t *pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address)
3319{
3320 return (unlikely(pud_none(*pud)) && __pmd_alloc(mm, pud, address))?
3321 NULL: pmd_offset(pud, address);
3322}
3323#endif /* CONFIG_MMU */
3324
3325enum pt_flags {
3326 PT_kernel = PG_referenced,
3327 PT_reserved = PG_reserved,
3328 /* High bits are used for zone/node/section */
3329};
3330
3331static inline struct ptdesc *virt_to_ptdesc(const void *x)
3332{
3333 return page_ptdesc(virt_to_page(x));
3334}
3335
3336/**
3337 * ptdesc_address - Virtual address of page table.
3338 * @pt: Page table descriptor.
3339 *
3340 * Return: The first byte of the page table described by @pt.
3341 */
3342static inline void *ptdesc_address(const struct ptdesc *pt)
3343{
3344 return folio_address(ptdesc_folio(pt));
3345}
3346
3347static inline bool pagetable_is_reserved(struct ptdesc *pt)
3348{
3349 return test_bit(PT_reserved, &pt->pt_flags.f);
3350}
3351
3352/**
3353 * ptdesc_set_kernel - Mark a ptdesc used to map the kernel
3354 * @ptdesc: The ptdesc to be marked
3355 *
3356 * Kernel page tables often need special handling. Set a flag so that
3357 * the handling code knows this ptdesc will not be used for userspace.
3358 */
3359static inline void ptdesc_set_kernel(struct ptdesc *ptdesc)
3360{
3361 set_bit(PT_kernel, &ptdesc->pt_flags.f);
3362}
3363
3364/**
3365 * ptdesc_clear_kernel - Mark a ptdesc as no longer used to map the kernel
3366 * @ptdesc: The ptdesc to be unmarked
3367 *
3368 * Use when the ptdesc is no longer used to map the kernel and no longer
3369 * needs special handling.
3370 */
3371static inline void ptdesc_clear_kernel(struct ptdesc *ptdesc)
3372{
3373 /*
3374 * Note: the 'PG_referenced' bit does not strictly need to be
3375 * cleared before freeing the page. But this is nice for
3376 * symmetry.
3377 */
3378 clear_bit(PT_kernel, &ptdesc->pt_flags.f);
3379}
3380
3381/**
3382 * ptdesc_test_kernel - Check if a ptdesc is used to map the kernel
3383 * @ptdesc: The ptdesc being tested
3384 *
3385 * Call to tell if the ptdesc used to map the kernel.
3386 */
3387static inline bool ptdesc_test_kernel(const struct ptdesc *ptdesc)
3388{
3389 return test_bit(PT_kernel, &ptdesc->pt_flags.f);
3390}
3391
3392/**
3393 * pagetable_alloc - Allocate pagetables
3394 * @gfp: GFP flags
3395 * @order: desired pagetable order
3396 *
3397 * pagetable_alloc allocates memory for page tables as well as a page table
3398 * descriptor to describe that memory.
3399 *
3400 * Return: The ptdesc describing the allocated page tables.
3401 */
3402static inline struct ptdesc *pagetable_alloc_noprof(gfp_t gfp, unsigned int order)
3403{
3404 struct page *page = alloc_pages_noprof(gfp | __GFP_COMP, order);
3405
3406 return page_ptdesc(page);
3407}
3408#define pagetable_alloc(...) alloc_hooks(pagetable_alloc_noprof(__VA_ARGS__))
3409
3410static inline void __pagetable_free(struct ptdesc *pt)
3411{
3412 struct page *page = ptdesc_page(pt);
3413
3414 __free_pages(page, compound_order(page));
3415}
3416
3417#ifdef CONFIG_ASYNC_KERNEL_PGTABLE_FREE
3418void pagetable_free_kernel(struct ptdesc *pt);
3419#else
3420static inline void pagetable_free_kernel(struct ptdesc *pt)
3421{
3422 __pagetable_free(pt);
3423}
3424#endif
3425/**
3426 * pagetable_free - Free pagetables
3427 * @pt: The page table descriptor
3428 *
3429 * pagetable_free frees the memory of all page tables described by a page
3430 * table descriptor and the memory for the descriptor itself.
3431 */
3432static inline void pagetable_free(struct ptdesc *pt)
3433{
3434 if (ptdesc_test_kernel(pt)) {
3435 ptdesc_clear_kernel(pt);
3436 pagetable_free_kernel(pt);
3437 } else {
3438 __pagetable_free(pt);
3439 }
3440}
3441
3442#if defined(CONFIG_SPLIT_PTE_PTLOCKS)
3443#if ALLOC_SPLIT_PTLOCKS
3444void __init ptlock_cache_init(void);
3445bool ptlock_alloc(struct ptdesc *ptdesc);
3446void ptlock_free(struct ptdesc *ptdesc);
3447
3448static inline spinlock_t *ptlock_ptr(struct ptdesc *ptdesc)
3449{
3450 return ptdesc->ptl;
3451}
3452#else /* ALLOC_SPLIT_PTLOCKS */
3453static inline void ptlock_cache_init(void)
3454{
3455}
3456
3457static inline bool ptlock_alloc(struct ptdesc *ptdesc)
3458{
3459 return true;
3460}
3461
3462static inline void ptlock_free(struct ptdesc *ptdesc)
3463{
3464}
3465
3466static inline spinlock_t *ptlock_ptr(struct ptdesc *ptdesc)
3467{
3468 return &ptdesc->ptl;
3469}
3470#endif /* ALLOC_SPLIT_PTLOCKS */
3471
3472static inline spinlock_t *pte_lockptr(struct mm_struct *mm, pmd_t *pmd)
3473{
3474 return ptlock_ptr(page_ptdesc(pmd_page(*pmd)));
3475}
3476
3477static inline spinlock_t *ptep_lockptr(struct mm_struct *mm, pte_t *pte)
3478{
3479 BUILD_BUG_ON(IS_ENABLED(CONFIG_HIGHPTE));
3480 BUILD_BUG_ON(MAX_PTRS_PER_PTE * sizeof(pte_t) > PAGE_SIZE);
3481 return ptlock_ptr(virt_to_ptdesc(pte));
3482}
3483
3484static inline bool ptlock_init(struct ptdesc *ptdesc)
3485{
3486 /*
3487 * prep_new_page() initialize page->private (and therefore page->ptl)
3488 * with 0. Make sure nobody took it in use in between.
3489 *
3490 * It can happen if arch try to use slab for page table allocation:
3491 * slab code uses page->slab_cache, which share storage with page->ptl.
3492 */
3493 VM_BUG_ON_PAGE(*(unsigned long *)&ptdesc->ptl, ptdesc_page(ptdesc));
3494 if (!ptlock_alloc(ptdesc))
3495 return false;
3496 spin_lock_init(ptlock_ptr(ptdesc));
3497 return true;
3498}
3499
3500#else /* !defined(CONFIG_SPLIT_PTE_PTLOCKS) */
3501/*
3502 * We use mm->page_table_lock to guard all pagetable pages of the mm.
3503 */
3504static inline spinlock_t *pte_lockptr(struct mm_struct *mm, pmd_t *pmd)
3505{
3506 return &mm->page_table_lock;
3507}
3508static inline spinlock_t *ptep_lockptr(struct mm_struct *mm, pte_t *pte)
3509{
3510 return &mm->page_table_lock;
3511}
3512static inline void ptlock_cache_init(void) {}
3513static inline bool ptlock_init(struct ptdesc *ptdesc) { return true; }
3514static inline void ptlock_free(struct ptdesc *ptdesc) {}
3515#endif /* defined(CONFIG_SPLIT_PTE_PTLOCKS) */
3516
3517static inline unsigned long ptdesc_nr_pages(const struct ptdesc *ptdesc)
3518{
3519 return compound_nr(ptdesc_page(ptdesc));
3520}
3521
3522static inline void __pagetable_ctor(struct ptdesc *ptdesc)
3523{
3524 pg_data_t *pgdat = NODE_DATA(memdesc_nid(ptdesc->pt_flags));
3525
3526 __SetPageTable(ptdesc_page(ptdesc));
3527 mod_node_page_state(pgdat, NR_PAGETABLE, ptdesc_nr_pages(ptdesc));
3528}
3529
3530static inline void pagetable_dtor(struct ptdesc *ptdesc)
3531{
3532 pg_data_t *pgdat = NODE_DATA(memdesc_nid(ptdesc->pt_flags));
3533
3534 ptlock_free(ptdesc);
3535 __ClearPageTable(ptdesc_page(ptdesc));
3536 mod_node_page_state(pgdat, NR_PAGETABLE, -ptdesc_nr_pages(ptdesc));
3537}
3538
3539static inline void pagetable_dtor_free(struct ptdesc *ptdesc)
3540{
3541 pagetable_dtor(ptdesc);
3542 pagetable_free(ptdesc);
3543}
3544
3545static inline bool pagetable_pte_ctor(struct mm_struct *mm,
3546 struct ptdesc *ptdesc)
3547{
3548 if (mm != &init_mm && !ptlock_init(ptdesc))
3549 return false;
3550 __pagetable_ctor(ptdesc);
3551 return true;
3552}
3553
3554pte_t *__pte_offset_map(pmd_t *pmd, unsigned long addr, pmd_t *pmdvalp);
3555
3556static inline pte_t *pte_offset_map(pmd_t *pmd, unsigned long addr)
3557{
3558 return __pte_offset_map(pmd, addr, NULL);
3559}
3560
3561pte_t *pte_offset_map_lock(struct mm_struct *mm, pmd_t *pmd,
3562 unsigned long addr, spinlock_t **ptlp);
3563
3564pte_t *pte_offset_map_ro_nolock(struct mm_struct *mm, pmd_t *pmd,
3565 unsigned long addr, spinlock_t **ptlp);
3566pte_t *pte_offset_map_rw_nolock(struct mm_struct *mm, pmd_t *pmd,
3567 unsigned long addr, pmd_t *pmdvalp,
3568 spinlock_t **ptlp);
3569
3570#define pte_unmap_unlock(pte, ptl) do { \
3571 spin_unlock(ptl); \
3572 pte_unmap(pte); \
3573} while (0)
3574
3575#define pte_alloc(mm, pmd) (unlikely(pmd_none(*(pmd))) && __pte_alloc(mm, pmd))
3576
3577#define pte_alloc_map(mm, pmd, address) \
3578 (pte_alloc(mm, pmd) ? NULL : pte_offset_map(pmd, address))
3579
3580#define pte_alloc_map_lock(mm, pmd, address, ptlp) \
3581 (pte_alloc(mm, pmd) ? \
3582 NULL : pte_offset_map_lock(mm, pmd, address, ptlp))
3583
3584#define pte_alloc_kernel(pmd, address) \
3585 ((unlikely(pmd_none(*(pmd))) && __pte_alloc_kernel(pmd))? \
3586 NULL: pte_offset_kernel(pmd, address))
3587
3588#if defined(CONFIG_SPLIT_PMD_PTLOCKS)
3589
3590static inline struct page *pmd_pgtable_page(pmd_t *pmd)
3591{
3592 unsigned long mask = ~(PTRS_PER_PMD * sizeof(pmd_t) - 1);
3593 return virt_to_page((void *)((unsigned long) pmd & mask));
3594}
3595
3596static inline struct ptdesc *pmd_ptdesc(pmd_t *pmd)
3597{
3598 return page_ptdesc(pmd_pgtable_page(pmd));
3599}
3600
3601static inline spinlock_t *pmd_lockptr(struct mm_struct *mm, pmd_t *pmd)
3602{
3603 return ptlock_ptr(pmd_ptdesc(pmd));
3604}
3605
3606static inline bool pmd_ptlock_init(struct ptdesc *ptdesc)
3607{
3608#ifdef CONFIG_TRANSPARENT_HUGEPAGE
3609 ptdesc->pmd_huge_pte = NULL;
3610#endif
3611 return ptlock_init(ptdesc);
3612}
3613
3614#define pmd_huge_pte(mm, pmd) (pmd_ptdesc(pmd)->pmd_huge_pte)
3615
3616#else
3617
3618static inline spinlock_t *pmd_lockptr(struct mm_struct *mm, pmd_t *pmd)
3619{
3620 return &mm->page_table_lock;
3621}
3622
3623static inline bool pmd_ptlock_init(struct ptdesc *ptdesc) { return true; }
3624
3625#define pmd_huge_pte(mm, pmd) ((mm)->pmd_huge_pte)
3626
3627#endif
3628
3629static inline spinlock_t *pmd_lock(struct mm_struct *mm, pmd_t *pmd)
3630{
3631 spinlock_t *ptl = pmd_lockptr(mm, pmd);
3632 spin_lock(ptl);
3633 return ptl;
3634}
3635
3636static inline bool pagetable_pmd_ctor(struct mm_struct *mm,
3637 struct ptdesc *ptdesc)
3638{
3639 if (mm != &init_mm && !pmd_ptlock_init(ptdesc))
3640 return false;
3641 ptdesc_pmd_pts_init(ptdesc);
3642 __pagetable_ctor(ptdesc);
3643 return true;
3644}
3645
3646/*
3647 * No scalability reason to split PUD locks yet, but follow the same pattern
3648 * as the PMD locks to make it easier if we decide to. The VM should not be
3649 * considered ready to switch to split PUD locks yet; there may be places
3650 * which need to be converted from page_table_lock.
3651 */
3652static inline spinlock_t *pud_lockptr(struct mm_struct *mm, pud_t *pud)
3653{
3654 return &mm->page_table_lock;
3655}
3656
3657static inline spinlock_t *pud_lock(struct mm_struct *mm, pud_t *pud)
3658{
3659 spinlock_t *ptl = pud_lockptr(mm, pud);
3660
3661 spin_lock(ptl);
3662 return ptl;
3663}
3664
3665static inline void pagetable_pud_ctor(struct ptdesc *ptdesc)
3666{
3667 __pagetable_ctor(ptdesc);
3668}
3669
3670static inline void pagetable_p4d_ctor(struct ptdesc *ptdesc)
3671{
3672 __pagetable_ctor(ptdesc);
3673}
3674
3675static inline void pagetable_pgd_ctor(struct ptdesc *ptdesc)
3676{
3677 __pagetable_ctor(ptdesc);
3678}
3679
3680extern void __init pagecache_init(void);
3681extern void free_initmem(void);
3682
3683/*
3684 * Free reserved pages within range [PAGE_ALIGN(start), end & PAGE_MASK)
3685 * into the buddy system. The freed pages will be poisoned with pattern
3686 * "poison" if it's within range [0, UCHAR_MAX].
3687 * Return pages freed into the buddy system.
3688 */
3689extern unsigned long free_reserved_area(void *start, void *end,
3690 int poison, const char *s);
3691
3692extern void adjust_managed_page_count(struct page *page, long count);
3693
3694extern void reserve_bootmem_region(phys_addr_t start,
3695 phys_addr_t end, int nid);
3696
3697/* Free the reserved page into the buddy system, so it gets managed. */
3698void free_reserved_page(struct page *page);
3699
3700static inline void mark_page_reserved(struct page *page)
3701{
3702 SetPageReserved(page);
3703 adjust_managed_page_count(page, -1);
3704}
3705
3706static inline void free_reserved_ptdesc(struct ptdesc *pt)
3707{
3708 free_reserved_page(ptdesc_page(pt));
3709}
3710
3711/*
3712 * Default method to free all the __init memory into the buddy system.
3713 * The freed pages will be poisoned with pattern "poison" if it's within
3714 * range [0, UCHAR_MAX].
3715 * Return pages freed into the buddy system.
3716 */
3717static inline unsigned long free_initmem_default(int poison)
3718{
3719 extern char __init_begin[], __init_end[];
3720
3721 return free_reserved_area(&__init_begin, &__init_end,
3722 poison, "unused kernel image (initmem)");
3723}
3724
3725static inline unsigned long get_num_physpages(void)
3726{
3727 int nid;
3728 unsigned long phys_pages = 0;
3729
3730 for_each_online_node(nid)
3731 phys_pages += node_present_pages(nid);
3732
3733 return phys_pages;
3734}
3735
3736/*
3737 * FIXME: Using memblock node mappings, an architecture may initialise its
3738 * zones, allocate the backing mem_map and account for memory holes in an
3739 * architecture independent manner.
3740 *
3741 * An architecture is expected to register range of page frames backed by
3742 * physical memory with memblock_add[_node]() before calling
3743 * free_area_init() passing in the PFN each zone ends at. At a basic
3744 * usage, an architecture is expected to do something like
3745 *
3746 * unsigned long max_zone_pfns[MAX_NR_ZONES] = {max_dma, max_normal_pfn,
3747 * max_highmem_pfn};
3748 * for_each_valid_physical_page_range()
3749 * memblock_add_node(base, size, nid, MEMBLOCK_NONE)
3750 * free_area_init(max_zone_pfns);
3751 */
3752void arch_zone_limits_init(unsigned long *max_zone_pfn);
3753unsigned long node_map_pfn_alignment(void);
3754extern unsigned long absent_pages_in_range(unsigned long start_pfn,
3755 unsigned long end_pfn);
3756extern void get_pfn_range_for_nid(unsigned int nid,
3757 unsigned long *start_pfn, unsigned long *end_pfn);
3758
3759#ifndef CONFIG_NUMA
3760static inline int early_pfn_to_nid(unsigned long pfn)
3761{
3762 return 0;
3763}
3764#else
3765/* please see mm/page_alloc.c */
3766extern int __meminit early_pfn_to_nid(unsigned long pfn);
3767#endif
3768
3769extern void mem_init(void);
3770extern void __init mmap_init(void);
3771
3772extern void __show_mem(unsigned int flags, nodemask_t *nodemask, int max_zone_idx);
3773static inline void show_mem(void)
3774{
3775 __show_mem(0, NULL, MAX_NR_ZONES - 1);
3776}
3777extern long si_mem_available(void);
3778extern void si_meminfo(struct sysinfo * val);
3779extern void si_meminfo_node(struct sysinfo *val, int nid);
3780
3781extern __printf(3, 4)
3782void warn_alloc(gfp_t gfp_mask, nodemask_t *nodemask, const char *fmt, ...);
3783
3784extern void setup_per_cpu_pageset(void);
3785
3786/* nommu.c */
3787extern atomic_long_t mmap_pages_allocated;
3788extern int nommu_shrink_inode_mappings(struct inode *, size_t, size_t);
3789
3790/* interval_tree.c */
3791void vma_interval_tree_insert(struct vm_area_struct *node,
3792 struct rb_root_cached *root);
3793void vma_interval_tree_insert_after(struct vm_area_struct *node,
3794 struct vm_area_struct *prev,
3795 struct rb_root_cached *root);
3796void vma_interval_tree_remove(struct vm_area_struct *node,
3797 struct rb_root_cached *root);
3798struct vm_area_struct *vma_interval_tree_subtree_search(struct vm_area_struct *node,
3799 unsigned long start, unsigned long last);
3800struct vm_area_struct *vma_interval_tree_iter_first(struct rb_root_cached *root,
3801 unsigned long start, unsigned long last);
3802struct vm_area_struct *vma_interval_tree_iter_next(struct vm_area_struct *node,
3803 unsigned long start, unsigned long last);
3804
3805#define vma_interval_tree_foreach(vma, root, start, last) \
3806 for (vma = vma_interval_tree_iter_first(root, start, last); \
3807 vma; vma = vma_interval_tree_iter_next(vma, start, last))
3808
3809void anon_vma_interval_tree_insert(struct anon_vma_chain *node,
3810 struct rb_root_cached *root);
3811void anon_vma_interval_tree_remove(struct anon_vma_chain *node,
3812 struct rb_root_cached *root);
3813struct anon_vma_chain *
3814anon_vma_interval_tree_iter_first(struct rb_root_cached *root,
3815 unsigned long start, unsigned long last);
3816struct anon_vma_chain *anon_vma_interval_tree_iter_next(
3817 struct anon_vma_chain *node, unsigned long start, unsigned long last);
3818#ifdef CONFIG_DEBUG_VM_RB
3819void anon_vma_interval_tree_verify(struct anon_vma_chain *node);
3820#endif
3821
3822#define anon_vma_interval_tree_foreach(avc, root, start, last) \
3823 for (avc = anon_vma_interval_tree_iter_first(root, start, last); \
3824 avc; avc = anon_vma_interval_tree_iter_next(avc, start, last))
3825
3826/* mmap.c */
3827extern int __vm_enough_memory(const struct mm_struct *mm, long pages, int cap_sys_admin);
3828extern int insert_vm_struct(struct mm_struct *, struct vm_area_struct *);
3829extern void exit_mmap(struct mm_struct *);
3830bool mmap_read_lock_maybe_expand(struct mm_struct *mm, struct vm_area_struct *vma,
3831 unsigned long addr, bool write);
3832
3833static inline int check_data_rlimit(unsigned long rlim,
3834 unsigned long new,
3835 unsigned long start,
3836 unsigned long end_data,
3837 unsigned long start_data)
3838{
3839 if (rlim < RLIM_INFINITY) {
3840 if (((new - start) + (end_data - start_data)) > rlim)
3841 return -ENOSPC;
3842 }
3843
3844 return 0;
3845}
3846
3847extern int mm_take_all_locks(struct mm_struct *mm);
3848extern void mm_drop_all_locks(struct mm_struct *mm);
3849
3850extern int set_mm_exe_file(struct mm_struct *mm, struct file *new_exe_file);
3851extern int replace_mm_exe_file(struct mm_struct *mm, struct file *new_exe_file);
3852extern struct file *get_mm_exe_file(struct mm_struct *mm);
3853extern struct file *get_task_exe_file(struct task_struct *task);
3854
3855extern bool may_expand_vm(struct mm_struct *, vm_flags_t, unsigned long npages);
3856extern void vm_stat_account(struct mm_struct *, vm_flags_t, long npages);
3857
3858extern bool vma_is_special_mapping(const struct vm_area_struct *vma,
3859 const struct vm_special_mapping *sm);
3860struct vm_area_struct *_install_special_mapping(struct mm_struct *mm,
3861 unsigned long addr, unsigned long len,
3862 vm_flags_t vm_flags,
3863 const struct vm_special_mapping *spec);
3864
3865unsigned long randomize_stack_top(unsigned long stack_top);
3866unsigned long randomize_page(unsigned long start, unsigned long range);
3867
3868unsigned long
3869__get_unmapped_area(struct file *file, unsigned long addr, unsigned long len,
3870 unsigned long pgoff, unsigned long flags, vm_flags_t vm_flags);
3871
3872static inline unsigned long
3873get_unmapped_area(struct file *file, unsigned long addr, unsigned long len,
3874 unsigned long pgoff, unsigned long flags)
3875{
3876 return __get_unmapped_area(file, addr, len, pgoff, flags, 0);
3877}
3878
3879extern unsigned long do_mmap(struct file *file, unsigned long addr,
3880 unsigned long len, unsigned long prot, unsigned long flags,
3881 vm_flags_t vm_flags, unsigned long pgoff, unsigned long *populate,
3882 struct list_head *uf);
3883extern int do_vmi_munmap(struct vma_iterator *vmi, struct mm_struct *mm,
3884 unsigned long start, size_t len, struct list_head *uf,
3885 bool unlock);
3886int do_vmi_align_munmap(struct vma_iterator *vmi, struct vm_area_struct *vma,
3887 struct mm_struct *mm, unsigned long start,
3888 unsigned long end, struct list_head *uf, bool unlock);
3889extern int do_munmap(struct mm_struct *, unsigned long, size_t,
3890 struct list_head *uf);
3891extern int do_madvise(struct mm_struct *mm, unsigned long start, size_t len_in, int behavior);
3892
3893#ifdef CONFIG_MMU
3894extern int __mm_populate(unsigned long addr, unsigned long len,
3895 int ignore_errors);
3896static inline void mm_populate(unsigned long addr, unsigned long len)
3897{
3898 /* Ignore errors */
3899 (void) __mm_populate(addr, len, 1);
3900}
3901#else
3902static inline void mm_populate(unsigned long addr, unsigned long len) {}
3903#endif
3904
3905/* This takes the mm semaphore itself */
3906extern int __must_check vm_brk_flags(unsigned long, unsigned long, unsigned long);
3907extern int vm_munmap(unsigned long, size_t);
3908extern unsigned long __must_check vm_mmap(struct file *, unsigned long,
3909 unsigned long, unsigned long,
3910 unsigned long, unsigned long);
3911
3912struct vm_unmapped_area_info {
3913#define VM_UNMAPPED_AREA_TOPDOWN 1
3914 unsigned long flags;
3915 unsigned long length;
3916 unsigned long low_limit;
3917 unsigned long high_limit;
3918 unsigned long align_mask;
3919 unsigned long align_offset;
3920 unsigned long start_gap;
3921};
3922
3923extern unsigned long vm_unmapped_area(struct vm_unmapped_area_info *info);
3924
3925/* truncate.c */
3926void truncate_inode_pages(struct address_space *mapping, loff_t lstart);
3927void truncate_inode_pages_range(struct address_space *mapping, loff_t lstart,
3928 uoff_t lend);
3929void truncate_inode_pages_final(struct address_space *mapping);
3930
3931/* generic vm_area_ops exported for stackable file systems */
3932extern vm_fault_t filemap_fault(struct vm_fault *vmf);
3933extern vm_fault_t filemap_map_pages(struct vm_fault *vmf,
3934 pgoff_t start_pgoff, pgoff_t end_pgoff);
3935extern vm_fault_t filemap_page_mkwrite(struct vm_fault *vmf);
3936
3937extern unsigned long stack_guard_gap;
3938/* Generic expand stack which grows the stack according to GROWS{UP,DOWN} */
3939int expand_stack_locked(struct vm_area_struct *vma, unsigned long address);
3940struct vm_area_struct *expand_stack(struct mm_struct * mm, unsigned long addr);
3941
3942/* Look up the first VMA which satisfies addr < vm_end, NULL if none. */
3943extern struct vm_area_struct * find_vma(struct mm_struct * mm, unsigned long addr);
3944extern struct vm_area_struct * find_vma_prev(struct mm_struct * mm, unsigned long addr,
3945 struct vm_area_struct **pprev);
3946
3947/*
3948 * Look up the first VMA which intersects the interval [start_addr, end_addr)
3949 * NULL if none. Assume start_addr < end_addr.
3950 */
3951struct vm_area_struct *find_vma_intersection(struct mm_struct *mm,
3952 unsigned long start_addr, unsigned long end_addr);
3953
3954/**
3955 * vma_lookup() - Find a VMA at a specific address
3956 * @mm: The process address space.
3957 * @addr: The user address.
3958 *
3959 * Return: The vm_area_struct at the given address, %NULL otherwise.
3960 */
3961static inline
3962struct vm_area_struct *vma_lookup(struct mm_struct *mm, unsigned long addr)
3963{
3964 return mtree_load(&mm->mm_mt, addr);
3965}
3966
3967static inline unsigned long stack_guard_start_gap(const struct vm_area_struct *vma)
3968{
3969 if (vma->vm_flags & VM_GROWSDOWN)
3970 return stack_guard_gap;
3971
3972 /* See reasoning around the VM_SHADOW_STACK definition */
3973 if (vma->vm_flags & VM_SHADOW_STACK)
3974 return PAGE_SIZE;
3975
3976 return 0;
3977}
3978
3979static inline unsigned long vm_start_gap(const struct vm_area_struct *vma)
3980{
3981 unsigned long gap = stack_guard_start_gap(vma);
3982 unsigned long vm_start = vma->vm_start;
3983
3984 vm_start -= gap;
3985 if (vm_start > vma->vm_start)
3986 vm_start = 0;
3987 return vm_start;
3988}
3989
3990static inline unsigned long vm_end_gap(const struct vm_area_struct *vma)
3991{
3992 unsigned long vm_end = vma->vm_end;
3993
3994 if (vma->vm_flags & VM_GROWSUP) {
3995 vm_end += stack_guard_gap;
3996 if (vm_end < vma->vm_end)
3997 vm_end = -PAGE_SIZE;
3998 }
3999 return vm_end;
4000}
4001
4002static inline unsigned long vma_pages(const struct vm_area_struct *vma)
4003{
4004 return (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;
4005}
4006
4007static inline unsigned long vma_desc_size(const struct vm_area_desc *desc)
4008{
4009 return desc->end - desc->start;
4010}
4011
4012static inline unsigned long vma_desc_pages(const struct vm_area_desc *desc)
4013{
4014 return vma_desc_size(desc) >> PAGE_SHIFT;
4015}
4016
4017/**
4018 * mmap_action_remap - helper for mmap_prepare hook to specify that a pure PFN
4019 * remap is required.
4020 * @desc: The VMA descriptor for the VMA requiring remap.
4021 * @start: The virtual address to start the remap from, must be within the VMA.
4022 * @start_pfn: The first PFN in the range to remap.
4023 * @size: The size of the range to remap, in bytes, at most spanning to the end
4024 * of the VMA.
4025 */
4026static inline void mmap_action_remap(struct vm_area_desc *desc,
4027 unsigned long start,
4028 unsigned long start_pfn,
4029 unsigned long size)
4030{
4031 struct mmap_action *action = &desc->action;
4032
4033 /* [start, start + size) must be within the VMA. */
4034 WARN_ON_ONCE(start < desc->start || start >= desc->end);
4035 WARN_ON_ONCE(start + size > desc->end);
4036
4037 action->type = MMAP_REMAP_PFN;
4038 action->remap.start = start;
4039 action->remap.start_pfn = start_pfn;
4040 action->remap.size = size;
4041 action->remap.pgprot = desc->page_prot;
4042}
4043
4044/**
4045 * mmap_action_remap_full - helper for mmap_prepare hook to specify that the
4046 * entirety of a VMA should be PFN remapped.
4047 * @desc: The VMA descriptor for the VMA requiring remap.
4048 * @start_pfn: The first PFN in the range to remap.
4049 */
4050static inline void mmap_action_remap_full(struct vm_area_desc *desc,
4051 unsigned long start_pfn)
4052{
4053 mmap_action_remap(desc, desc->start, start_pfn, vma_desc_size(desc));
4054}
4055
4056/**
4057 * mmap_action_ioremap - helper for mmap_prepare hook to specify that a pure PFN
4058 * I/O remap is required.
4059 * @desc: The VMA descriptor for the VMA requiring remap.
4060 * @start: The virtual address to start the remap from, must be within the VMA.
4061 * @start_pfn: The first PFN in the range to remap.
4062 * @size: The size of the range to remap, in bytes, at most spanning to the end
4063 * of the VMA.
4064 */
4065static inline void mmap_action_ioremap(struct vm_area_desc *desc,
4066 unsigned long start,
4067 unsigned long start_pfn,
4068 unsigned long size)
4069{
4070 mmap_action_remap(desc, start, start_pfn, size);
4071 desc->action.type = MMAP_IO_REMAP_PFN;
4072}
4073
4074/**
4075 * mmap_action_ioremap_full - helper for mmap_prepare hook to specify that the
4076 * entirety of a VMA should be PFN I/O remapped.
4077 * @desc: The VMA descriptor for the VMA requiring remap.
4078 * @start_pfn: The first PFN in the range to remap.
4079 */
4080static inline void mmap_action_ioremap_full(struct vm_area_desc *desc,
4081 unsigned long start_pfn)
4082{
4083 mmap_action_ioremap(desc, desc->start, start_pfn, vma_desc_size(desc));
4084}
4085
4086void mmap_action_prepare(struct mmap_action *action,
4087 struct vm_area_desc *desc);
4088int mmap_action_complete(struct mmap_action *action,
4089 struct vm_area_struct *vma);
4090
4091/* Look up the first VMA which exactly match the interval vm_start ... vm_end */
4092static inline struct vm_area_struct *find_exact_vma(struct mm_struct *mm,
4093 unsigned long vm_start, unsigned long vm_end)
4094{
4095 struct vm_area_struct *vma = vma_lookup(mm, vm_start);
4096
4097 if (vma && (vma->vm_start != vm_start || vma->vm_end != vm_end))
4098 vma = NULL;
4099
4100 return vma;
4101}
4102
4103static inline bool range_in_vma(const struct vm_area_struct *vma,
4104 unsigned long start, unsigned long end)
4105{
4106 return (vma && vma->vm_start <= start && end <= vma->vm_end);
4107}
4108
4109#ifdef CONFIG_MMU
4110pgprot_t vm_get_page_prot(vm_flags_t vm_flags);
4111void vma_set_page_prot(struct vm_area_struct *vma);
4112#else
4113static inline pgprot_t vm_get_page_prot(vm_flags_t vm_flags)
4114{
4115 return __pgprot(0);
4116}
4117static inline void vma_set_page_prot(struct vm_area_struct *vma)
4118{
4119 vma->vm_page_prot = vm_get_page_prot(vma->vm_flags);
4120}
4121#endif
4122
4123void vma_set_file(struct vm_area_struct *vma, struct file *file);
4124
4125#ifdef CONFIG_NUMA_BALANCING
4126unsigned long change_prot_numa(struct vm_area_struct *vma,
4127 unsigned long start, unsigned long end);
4128#endif
4129
4130struct vm_area_struct *find_extend_vma_locked(struct mm_struct *,
4131 unsigned long addr);
4132int remap_pfn_range(struct vm_area_struct *vma, unsigned long addr,
4133 unsigned long pfn, unsigned long size, pgprot_t pgprot);
4134
4135int vm_insert_page(struct vm_area_struct *, unsigned long addr, struct page *);
4136int vm_insert_pages(struct vm_area_struct *vma, unsigned long addr,
4137 struct page **pages, unsigned long *num);
4138int vm_map_pages(struct vm_area_struct *vma, struct page **pages,
4139 unsigned long num);
4140int vm_map_pages_zero(struct vm_area_struct *vma, struct page **pages,
4141 unsigned long num);
4142vm_fault_t vmf_insert_page_mkwrite(struct vm_fault *vmf, struct page *page,
4143 bool write);
4144vm_fault_t vmf_insert_pfn(struct vm_area_struct *vma, unsigned long addr,
4145 unsigned long pfn);
4146vm_fault_t vmf_insert_pfn_prot(struct vm_area_struct *vma, unsigned long addr,
4147 unsigned long pfn, pgprot_t pgprot);
4148vm_fault_t vmf_insert_mixed(struct vm_area_struct *vma, unsigned long addr,
4149 unsigned long pfn);
4150vm_fault_t vmf_insert_mixed_mkwrite(struct vm_area_struct *vma,
4151 unsigned long addr, unsigned long pfn);
4152int vm_iomap_memory(struct vm_area_struct *vma, phys_addr_t start, unsigned long len);
4153
4154static inline vm_fault_t vmf_insert_page(struct vm_area_struct *vma,
4155 unsigned long addr, struct page *page)
4156{
4157 int err = vm_insert_page(vma, addr, page);
4158
4159 if (err == -ENOMEM)
4160 return VM_FAULT_OOM;
4161 if (err < 0 && err != -EBUSY)
4162 return VM_FAULT_SIGBUS;
4163
4164 return VM_FAULT_NOPAGE;
4165}
4166
4167#ifndef io_remap_pfn_range_pfn
4168static inline unsigned long io_remap_pfn_range_pfn(unsigned long pfn,
4169 unsigned long size)
4170{
4171 return pfn;
4172}
4173#endif
4174
4175static inline int io_remap_pfn_range(struct vm_area_struct *vma,
4176 unsigned long addr, unsigned long orig_pfn,
4177 unsigned long size, pgprot_t orig_prot)
4178{
4179 const unsigned long pfn = io_remap_pfn_range_pfn(orig_pfn, size);
4180 const pgprot_t prot = pgprot_decrypted(orig_prot);
4181
4182 return remap_pfn_range(vma, addr, pfn, size, prot);
4183}
4184
4185static inline vm_fault_t vmf_error(int err)
4186{
4187 if (err == -ENOMEM)
4188 return VM_FAULT_OOM;
4189 else if (err == -EHWPOISON)
4190 return VM_FAULT_HWPOISON;
4191 return VM_FAULT_SIGBUS;
4192}
4193
4194/*
4195 * Convert errno to return value for ->page_mkwrite() calls.
4196 *
4197 * This should eventually be merged with vmf_error() above, but will need a
4198 * careful audit of all vmf_error() callers.
4199 */
4200static inline vm_fault_t vmf_fs_error(int err)
4201{
4202 if (err == 0)
4203 return VM_FAULT_LOCKED;
4204 if (err == -EFAULT || err == -EAGAIN)
4205 return VM_FAULT_NOPAGE;
4206 if (err == -ENOMEM)
4207 return VM_FAULT_OOM;
4208 /* -ENOSPC, -EDQUOT, -EIO ... */
4209 return VM_FAULT_SIGBUS;
4210}
4211
4212static inline int vm_fault_to_errno(vm_fault_t vm_fault, int foll_flags)
4213{
4214 if (vm_fault & VM_FAULT_OOM)
4215 return -ENOMEM;
4216 if (vm_fault & (VM_FAULT_HWPOISON | VM_FAULT_HWPOISON_LARGE))
4217 return (foll_flags & FOLL_HWPOISON) ? -EHWPOISON : -EFAULT;
4218 if (vm_fault & (VM_FAULT_SIGBUS | VM_FAULT_SIGSEGV))
4219 return -EFAULT;
4220 return 0;
4221}
4222
4223/*
4224 * Indicates whether GUP can follow a PROT_NONE mapped page, or whether
4225 * a (NUMA hinting) fault is required.
4226 */
4227static inline bool gup_can_follow_protnone(const struct vm_area_struct *vma,
4228 unsigned int flags)
4229{
4230 /*
4231 * If callers don't want to honor NUMA hinting faults, no need to
4232 * determine if we would actually have to trigger a NUMA hinting fault.
4233 */
4234 if (!(flags & FOLL_HONOR_NUMA_FAULT))
4235 return true;
4236
4237 /*
4238 * NUMA hinting faults don't apply in inaccessible (PROT_NONE) VMAs.
4239 *
4240 * Requiring a fault here even for inaccessible VMAs would mean that
4241 * FOLL_FORCE cannot make any progress, because handle_mm_fault()
4242 * refuses to process NUMA hinting faults in inaccessible VMAs.
4243 */
4244 return !vma_is_accessible(vma);
4245}
4246
4247typedef int (*pte_fn_t)(pte_t *pte, unsigned long addr, void *data);
4248extern int apply_to_page_range(struct mm_struct *mm, unsigned long address,
4249 unsigned long size, pte_fn_t fn, void *data);
4250extern int apply_to_existing_page_range(struct mm_struct *mm,
4251 unsigned long address, unsigned long size,
4252 pte_fn_t fn, void *data);
4253
4254#ifdef CONFIG_PAGE_POISONING
4255extern void __kernel_poison_pages(struct page *page, int numpages);
4256extern void __kernel_unpoison_pages(struct page *page, int numpages);
4257extern bool _page_poisoning_enabled_early;
4258DECLARE_STATIC_KEY_FALSE(_page_poisoning_enabled);
4259static inline bool page_poisoning_enabled(void)
4260{
4261 return _page_poisoning_enabled_early;
4262}
4263/*
4264 * For use in fast paths after init_mem_debugging() has run, or when a
4265 * false negative result is not harmful when called too early.
4266 */
4267static inline bool page_poisoning_enabled_static(void)
4268{
4269 return static_branch_unlikely(&_page_poisoning_enabled);
4270}
4271static inline void kernel_poison_pages(struct page *page, int numpages)
4272{
4273 if (page_poisoning_enabled_static())
4274 __kernel_poison_pages(page, numpages);
4275}
4276static inline void kernel_unpoison_pages(struct page *page, int numpages)
4277{
4278 if (page_poisoning_enabled_static())
4279 __kernel_unpoison_pages(page, numpages);
4280}
4281#else
4282static inline bool page_poisoning_enabled(void) { return false; }
4283static inline bool page_poisoning_enabled_static(void) { return false; }
4284static inline void __kernel_poison_pages(struct page *page, int nunmpages) { }
4285static inline void kernel_poison_pages(struct page *page, int numpages) { }
4286static inline void kernel_unpoison_pages(struct page *page, int numpages) { }
4287#endif
4288
4289DECLARE_STATIC_KEY_MAYBE(CONFIG_INIT_ON_ALLOC_DEFAULT_ON, init_on_alloc);
4290static inline bool want_init_on_alloc(gfp_t flags)
4291{
4292 if (static_branch_maybe(CONFIG_INIT_ON_ALLOC_DEFAULT_ON,
4293 &init_on_alloc))
4294 return true;
4295 return flags & __GFP_ZERO;
4296}
4297
4298DECLARE_STATIC_KEY_MAYBE(CONFIG_INIT_ON_FREE_DEFAULT_ON, init_on_free);
4299static inline bool want_init_on_free(void)
4300{
4301 return static_branch_maybe(CONFIG_INIT_ON_FREE_DEFAULT_ON,
4302 &init_on_free);
4303}
4304
4305extern bool _debug_pagealloc_enabled_early;
4306DECLARE_STATIC_KEY_FALSE(_debug_pagealloc_enabled);
4307
4308static inline bool debug_pagealloc_enabled(void)
4309{
4310 return IS_ENABLED(CONFIG_DEBUG_PAGEALLOC) &&
4311 _debug_pagealloc_enabled_early;
4312}
4313
4314/*
4315 * For use in fast paths after mem_debugging_and_hardening_init() has run,
4316 * or when a false negative result is not harmful when called too early.
4317 */
4318static inline bool debug_pagealloc_enabled_static(void)
4319{
4320 if (!IS_ENABLED(CONFIG_DEBUG_PAGEALLOC))
4321 return false;
4322
4323 return static_branch_unlikely(&_debug_pagealloc_enabled);
4324}
4325
4326/*
4327 * To support DEBUG_PAGEALLOC architecture must ensure that
4328 * __kernel_map_pages() never fails
4329 */
4330extern void __kernel_map_pages(struct page *page, int numpages, int enable);
4331#ifdef CONFIG_DEBUG_PAGEALLOC
4332static inline void debug_pagealloc_map_pages(struct page *page, int numpages)
4333{
4334 iommu_debug_check_unmapped(page, numpages);
4335
4336 if (debug_pagealloc_enabled_static())
4337 __kernel_map_pages(page, numpages, 1);
4338}
4339
4340static inline void debug_pagealloc_unmap_pages(struct page *page, int numpages)
4341{
4342 iommu_debug_check_unmapped(page, numpages);
4343
4344 if (debug_pagealloc_enabled_static())
4345 __kernel_map_pages(page, numpages, 0);
4346}
4347
4348extern unsigned int _debug_guardpage_minorder;
4349DECLARE_STATIC_KEY_FALSE(_debug_guardpage_enabled);
4350
4351static inline unsigned int debug_guardpage_minorder(void)
4352{
4353 return _debug_guardpage_minorder;
4354}
4355
4356static inline bool debug_guardpage_enabled(void)
4357{
4358 return static_branch_unlikely(&_debug_guardpage_enabled);
4359}
4360
4361static inline bool page_is_guard(const struct page *page)
4362{
4363 if (!debug_guardpage_enabled())
4364 return false;
4365
4366 return PageGuard(page);
4367}
4368
4369bool __set_page_guard(struct zone *zone, struct page *page, unsigned int order);
4370static inline bool set_page_guard(struct zone *zone, struct page *page,
4371 unsigned int order)
4372{
4373 if (!debug_guardpage_enabled())
4374 return false;
4375 return __set_page_guard(zone, page, order);
4376}
4377
4378void __clear_page_guard(struct zone *zone, struct page *page, unsigned int order);
4379static inline void clear_page_guard(struct zone *zone, struct page *page,
4380 unsigned int order)
4381{
4382 if (!debug_guardpage_enabled())
4383 return;
4384 __clear_page_guard(zone, page, order);
4385}
4386
4387#else /* CONFIG_DEBUG_PAGEALLOC */
4388static inline void debug_pagealloc_map_pages(struct page *page, int numpages) {}
4389static inline void debug_pagealloc_unmap_pages(struct page *page, int numpages) {}
4390static inline unsigned int debug_guardpage_minorder(void) { return 0; }
4391static inline bool debug_guardpage_enabled(void) { return false; }
4392static inline bool page_is_guard(const struct page *page) { return false; }
4393static inline bool set_page_guard(struct zone *zone, struct page *page,
4394 unsigned int order) { return false; }
4395static inline void clear_page_guard(struct zone *zone, struct page *page,
4396 unsigned int order) {}
4397#endif /* CONFIG_DEBUG_PAGEALLOC */
4398
4399#ifndef clear_pages
4400/**
4401 * clear_pages() - clear a page range for kernel-internal use.
4402 * @addr: start address
4403 * @npages: number of pages
4404 *
4405 * Use clear_user_pages() instead when clearing a page range to be
4406 * mapped to user space.
4407 *
4408 * Does absolutely no exception handling.
4409 *
4410 * Note that even though the clearing operation is preemptible, clear_pages()
4411 * does not (and on architectures where it reduces to a few long-running
4412 * instructions, might not be able to) call cond_resched() to check if
4413 * rescheduling is required.
4414 *
4415 * When running under preemptible models this is not a problem. Under
4416 * cooperatively scheduled models, however, the caller is expected to
4417 * limit @npages to no more than PROCESS_PAGES_NON_PREEMPT_BATCH.
4418 */
4419static inline void clear_pages(void *addr, unsigned int npages)
4420{
4421 do {
4422 clear_page(addr);
4423 addr += PAGE_SIZE;
4424 } while (--npages);
4425}
4426#endif
4427
4428#ifndef PROCESS_PAGES_NON_PREEMPT_BATCH
4429#ifdef clear_pages
4430/*
4431 * The architecture defines clear_pages(), and we assume that it is
4432 * generally "fast". So choose a batch size large enough to allow the processor
4433 * headroom for optimizing the operation and yet small enough that we see
4434 * reasonable preemption latency for when this optimization is not possible
4435 * (ex. slow microarchitectures, memory bandwidth saturation.)
4436 *
4437 * With a value of 32MB and assuming a memory bandwidth of ~10GBps, this should
4438 * result in worst case preemption latency of around 3ms when clearing pages.
4439 *
4440 * (See comment above clear_pages() for why preemption latency is a concern
4441 * here.)
4442 */
4443#define PROCESS_PAGES_NON_PREEMPT_BATCH (SZ_32M >> PAGE_SHIFT)
4444#else /* !clear_pages */
4445/*
4446 * The architecture does not provide a clear_pages() implementation. Assume
4447 * that clear_page() -- which clear_pages() will fallback to -- is relatively
4448 * slow and choose a small value for PROCESS_PAGES_NON_PREEMPT_BATCH.
4449 */
4450#define PROCESS_PAGES_NON_PREEMPT_BATCH 1
4451#endif
4452#endif
4453
4454#ifdef __HAVE_ARCH_GATE_AREA
4455extern struct vm_area_struct *get_gate_vma(struct mm_struct *mm);
4456extern int in_gate_area_no_mm(unsigned long addr);
4457extern int in_gate_area(struct mm_struct *mm, unsigned long addr);
4458#else
4459static inline struct vm_area_struct *get_gate_vma(struct mm_struct *mm)
4460{
4461 return NULL;
4462}
4463static inline int in_gate_area_no_mm(unsigned long addr) { return 0; }
4464static inline int in_gate_area(struct mm_struct *mm, unsigned long addr)
4465{
4466 return 0;
4467}
4468#endif /* __HAVE_ARCH_GATE_AREA */
4469
4470bool process_shares_mm(const struct task_struct *p, const struct mm_struct *mm);
4471
4472void drop_slab(void);
4473
4474#ifndef CONFIG_MMU
4475#define randomize_va_space 0
4476#else
4477extern int randomize_va_space;
4478#endif
4479
4480const char * arch_vma_name(struct vm_area_struct *vma);
4481#ifdef CONFIG_MMU
4482void print_vma_addr(char *prefix, unsigned long rip);
4483#else
4484static inline void print_vma_addr(char *prefix, unsigned long rip)
4485{
4486}
4487#endif
4488
4489void *sparse_buffer_alloc(unsigned long size);
4490unsigned long section_map_size(void);
4491struct page * __populate_section_memmap(unsigned long pfn,
4492 unsigned long nr_pages, int nid, struct vmem_altmap *altmap,
4493 struct dev_pagemap *pgmap);
4494pgd_t *vmemmap_pgd_populate(unsigned long addr, int node);
4495p4d_t *vmemmap_p4d_populate(pgd_t *pgd, unsigned long addr, int node);
4496pud_t *vmemmap_pud_populate(p4d_t *p4d, unsigned long addr, int node);
4497pmd_t *vmemmap_pmd_populate(pud_t *pud, unsigned long addr, int node);
4498pte_t *vmemmap_pte_populate(pmd_t *pmd, unsigned long addr, int node,
4499 struct vmem_altmap *altmap, unsigned long ptpfn,
4500 unsigned long flags);
4501void *vmemmap_alloc_block(unsigned long size, int node);
4502struct vmem_altmap;
4503void *vmemmap_alloc_block_buf(unsigned long size, int node,
4504 struct vmem_altmap *altmap);
4505void vmemmap_verify(pte_t *, int, unsigned long, unsigned long);
4506void vmemmap_set_pmd(pmd_t *pmd, void *p, int node,
4507 unsigned long addr, unsigned long next);
4508int vmemmap_check_pmd(pmd_t *pmd, int node,
4509 unsigned long addr, unsigned long next);
4510int vmemmap_populate_basepages(unsigned long start, unsigned long end,
4511 int node, struct vmem_altmap *altmap);
4512int vmemmap_populate_hugepages(unsigned long start, unsigned long end,
4513 int node, struct vmem_altmap *altmap);
4514int vmemmap_populate(unsigned long start, unsigned long end, int node,
4515 struct vmem_altmap *altmap);
4516int vmemmap_populate_hvo(unsigned long start, unsigned long end, int node,
4517 unsigned long headsize);
4518int vmemmap_undo_hvo(unsigned long start, unsigned long end, int node,
4519 unsigned long headsize);
4520void vmemmap_wrprotect_hvo(unsigned long start, unsigned long end, int node,
4521 unsigned long headsize);
4522void vmemmap_populate_print_last(void);
4523#ifdef CONFIG_MEMORY_HOTPLUG
4524void vmemmap_free(unsigned long start, unsigned long end,
4525 struct vmem_altmap *altmap);
4526#endif
4527
4528#ifdef CONFIG_SPARSEMEM_VMEMMAP
4529static inline unsigned long vmem_altmap_offset(const struct vmem_altmap *altmap)
4530{
4531 /* number of pfns from base where pfn_to_page() is valid */
4532 if (altmap)
4533 return altmap->reserve + altmap->free;
4534 return 0;
4535}
4536
4537static inline void vmem_altmap_free(struct vmem_altmap *altmap,
4538 unsigned long nr_pfns)
4539{
4540 altmap->alloc -= nr_pfns;
4541}
4542#else
4543static inline unsigned long vmem_altmap_offset(const struct vmem_altmap *altmap)
4544{
4545 return 0;
4546}
4547
4548static inline void vmem_altmap_free(struct vmem_altmap *altmap,
4549 unsigned long nr_pfns)
4550{
4551}
4552#endif
4553
4554#define VMEMMAP_RESERVE_NR 2
4555#ifdef CONFIG_ARCH_WANT_OPTIMIZE_DAX_VMEMMAP
4556static inline bool __vmemmap_can_optimize(struct vmem_altmap *altmap,
4557 struct dev_pagemap *pgmap)
4558{
4559 unsigned long nr_pages;
4560 unsigned long nr_vmemmap_pages;
4561
4562 if (!pgmap || !is_power_of_2(sizeof(struct page)))
4563 return false;
4564
4565 nr_pages = pgmap_vmemmap_nr(pgmap);
4566 nr_vmemmap_pages = ((nr_pages * sizeof(struct page)) >> PAGE_SHIFT);
4567 /*
4568 * For vmemmap optimization with DAX we need minimum 2 vmemmap
4569 * pages. See layout diagram in Documentation/mm/vmemmap_dedup.rst
4570 */
4571 return !altmap && (nr_vmemmap_pages > VMEMMAP_RESERVE_NR);
4572}
4573/*
4574 * If we don't have an architecture override, use the generic rule
4575 */
4576#ifndef vmemmap_can_optimize
4577#define vmemmap_can_optimize __vmemmap_can_optimize
4578#endif
4579
4580#else
4581static inline bool vmemmap_can_optimize(struct vmem_altmap *altmap,
4582 struct dev_pagemap *pgmap)
4583{
4584 return false;
4585}
4586#endif
4587
4588enum mf_flags {
4589 MF_COUNT_INCREASED = 1 << 0,
4590 MF_ACTION_REQUIRED = 1 << 1,
4591 MF_MUST_KILL = 1 << 2,
4592 MF_SOFT_OFFLINE = 1 << 3,
4593 MF_UNPOISON = 1 << 4,
4594 MF_SW_SIMULATED = 1 << 5,
4595 MF_NO_RETRY = 1 << 6,
4596 MF_MEM_PRE_REMOVE = 1 << 7,
4597};
4598int mf_dax_kill_procs(struct address_space *mapping, pgoff_t index,
4599 unsigned long count, int mf_flags);
4600extern int memory_failure(unsigned long pfn, int flags);
4601extern int unpoison_memory(unsigned long pfn);
4602extern atomic_long_t num_poisoned_pages __read_mostly;
4603extern int soft_offline_page(unsigned long pfn, int flags);
4604#ifdef CONFIG_MEMORY_FAILURE
4605/*
4606 * Sysfs entries for memory failure handling statistics.
4607 */
4608extern const struct attribute_group memory_failure_attr_group;
4609extern void memory_failure_queue(unsigned long pfn, int flags);
4610extern int __get_huge_page_for_hwpoison(unsigned long pfn, int flags,
4611 bool *migratable_cleared);
4612void num_poisoned_pages_inc(unsigned long pfn);
4613void num_poisoned_pages_sub(unsigned long pfn, long i);
4614#else
4615static inline void memory_failure_queue(unsigned long pfn, int flags)
4616{
4617}
4618
4619static inline int __get_huge_page_for_hwpoison(unsigned long pfn, int flags,
4620 bool *migratable_cleared)
4621{
4622 return 0;
4623}
4624
4625static inline void num_poisoned_pages_inc(unsigned long pfn)
4626{
4627}
4628
4629static inline void num_poisoned_pages_sub(unsigned long pfn, long i)
4630{
4631}
4632#endif
4633
4634#if defined(CONFIG_MEMORY_FAILURE) && defined(CONFIG_MEMORY_HOTPLUG)
4635extern void memblk_nr_poison_inc(unsigned long pfn);
4636extern void memblk_nr_poison_sub(unsigned long pfn, long i);
4637#else
4638static inline void memblk_nr_poison_inc(unsigned long pfn)
4639{
4640}
4641
4642static inline void memblk_nr_poison_sub(unsigned long pfn, long i)
4643{
4644}
4645#endif
4646
4647#ifndef arch_memory_failure
4648static inline int arch_memory_failure(unsigned long pfn, int flags)
4649{
4650 return -ENXIO;
4651}
4652#endif
4653
4654#ifndef arch_is_platform_page
4655static inline bool arch_is_platform_page(u64 paddr)
4656{
4657 return false;
4658}
4659#endif
4660
4661/*
4662 * Error handlers for various types of pages.
4663 */
4664enum mf_result {
4665 MF_IGNORED, /* Error: cannot be handled */
4666 MF_FAILED, /* Error: handling failed */
4667 MF_DELAYED, /* Will be handled later */
4668 MF_RECOVERED, /* Successfully recovered */
4669};
4670
4671enum mf_action_page_type {
4672 MF_MSG_KERNEL,
4673 MF_MSG_KERNEL_HIGH_ORDER,
4674 MF_MSG_DIFFERENT_COMPOUND,
4675 MF_MSG_HUGE,
4676 MF_MSG_FREE_HUGE,
4677 MF_MSG_GET_HWPOISON,
4678 MF_MSG_UNMAP_FAILED,
4679 MF_MSG_DIRTY_SWAPCACHE,
4680 MF_MSG_CLEAN_SWAPCACHE,
4681 MF_MSG_DIRTY_MLOCKED_LRU,
4682 MF_MSG_CLEAN_MLOCKED_LRU,
4683 MF_MSG_DIRTY_UNEVICTABLE_LRU,
4684 MF_MSG_CLEAN_UNEVICTABLE_LRU,
4685 MF_MSG_DIRTY_LRU,
4686 MF_MSG_CLEAN_LRU,
4687 MF_MSG_TRUNCATED_LRU,
4688 MF_MSG_BUDDY,
4689 MF_MSG_DAX,
4690 MF_MSG_UNSPLIT_THP,
4691 MF_MSG_ALREADY_POISONED,
4692 MF_MSG_PFN_MAP,
4693 MF_MSG_UNKNOWN,
4694};
4695
4696#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_HUGETLBFS)
4697void folio_zero_user(struct folio *folio, unsigned long addr_hint);
4698int copy_user_large_folio(struct folio *dst, struct folio *src,
4699 unsigned long addr_hint,
4700 struct vm_area_struct *vma);
4701long copy_folio_from_user(struct folio *dst_folio,
4702 const void __user *usr_src,
4703 bool allow_pagefault);
4704
4705/**
4706 * vma_is_special_huge - Are transhuge page-table entries considered special?
4707 * @vma: Pointer to the struct vm_area_struct to consider
4708 *
4709 * Whether transhuge page-table entries are considered "special" following
4710 * the definition in vm_normal_page().
4711 *
4712 * Return: true if transhuge page-table entries should be considered special,
4713 * false otherwise.
4714 */
4715static inline bool vma_is_special_huge(const struct vm_area_struct *vma)
4716{
4717 return vma_is_dax(vma) || (vma->vm_file &&
4718 (vma->vm_flags & (VM_PFNMAP | VM_MIXEDMAP)));
4719}
4720
4721#endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_HUGETLBFS */
4722
4723#if MAX_NUMNODES > 1
4724void __init setup_nr_node_ids(void);
4725#else
4726static inline void setup_nr_node_ids(void) {}
4727#endif
4728
4729extern int memcmp_pages(struct page *page1, struct page *page2);
4730
4731static inline int pages_identical(struct page *page1, struct page *page2)
4732{
4733 return !memcmp_pages(page1, page2);
4734}
4735
4736#ifdef CONFIG_MAPPING_DIRTY_HELPERS
4737unsigned long clean_record_shared_mapping_range(struct address_space *mapping,
4738 pgoff_t first_index, pgoff_t nr,
4739 pgoff_t bitmap_pgoff,
4740 unsigned long *bitmap,
4741 pgoff_t *start,
4742 pgoff_t *end);
4743
4744unsigned long wp_shared_mapping_range(struct address_space *mapping,
4745 pgoff_t first_index, pgoff_t nr);
4746#endif
4747
4748#ifdef CONFIG_ANON_VMA_NAME
4749int set_anon_vma_name(unsigned long addr, unsigned long size,
4750 const char __user *uname);
4751#else
4752static inline
4753int set_anon_vma_name(unsigned long addr, unsigned long size,
4754 const char __user *uname)
4755{
4756 return -EINVAL;
4757}
4758#endif
4759
4760#ifdef CONFIG_UNACCEPTED_MEMORY
4761
4762bool range_contains_unaccepted_memory(phys_addr_t start, unsigned long size);
4763void accept_memory(phys_addr_t start, unsigned long size);
4764
4765#else
4766
4767static inline bool range_contains_unaccepted_memory(phys_addr_t start,
4768 unsigned long size)
4769{
4770 return false;
4771}
4772
4773static inline void accept_memory(phys_addr_t start, unsigned long size)
4774{
4775}
4776
4777#endif
4778
4779static inline bool pfn_is_unaccepted_memory(unsigned long pfn)
4780{
4781 return range_contains_unaccepted_memory(pfn << PAGE_SHIFT, PAGE_SIZE);
4782}
4783
4784void vma_pgtable_walk_begin(struct vm_area_struct *vma);
4785void vma_pgtable_walk_end(struct vm_area_struct *vma);
4786
4787int reserve_mem_find_by_name(const char *name, phys_addr_t *start, phys_addr_t *size);
4788int reserve_mem_release_by_name(const char *name);
4789
4790#ifdef CONFIG_64BIT
4791int do_mseal(unsigned long start, size_t len_in, unsigned long flags);
4792#else
4793static inline int do_mseal(unsigned long start, size_t len_in, unsigned long flags)
4794{
4795 /* noop on 32 bit */
4796 return 0;
4797}
4798#endif
4799
4800/*
4801 * user_alloc_needs_zeroing checks if a user folio from page allocator needs to
4802 * be zeroed or not.
4803 */
4804static inline bool user_alloc_needs_zeroing(void)
4805{
4806 /*
4807 * for user folios, arch with cache aliasing requires cache flush and
4808 * arc changes folio->flags to make icache coherent with dcache, so
4809 * always return false to make caller use
4810 * clear_user_page()/clear_user_highpage().
4811 */
4812 return cpu_dcache_is_aliasing() || cpu_icache_is_aliasing() ||
4813 !static_branch_maybe(CONFIG_INIT_ON_ALLOC_DEFAULT_ON,
4814 &init_on_alloc);
4815}
4816
4817int arch_get_shadow_stack_status(struct task_struct *t, unsigned long __user *status);
4818int arch_set_shadow_stack_status(struct task_struct *t, unsigned long status);
4819int arch_lock_shadow_stack_status(struct task_struct *t, unsigned long status);
4820
4821/*
4822 * DMA mapping IDs for page_pool
4823 *
4824 * When DMA-mapping a page, page_pool allocates an ID (from an xarray) and
4825 * stashes it in the upper bits of page->pp_magic. We always want to be able to
4826 * unambiguously identify page pool pages (using page_pool_page_is_pp()). Non-PP
4827 * pages can have arbitrary kernel pointers stored in the same field as pp_magic
4828 * (since it overlaps with page->lru.next), so we must ensure that we cannot
4829 * mistake a valid kernel pointer with any of the values we write into this
4830 * field.
4831 *
4832 * On architectures that set POISON_POINTER_DELTA, this is already ensured,
4833 * since this value becomes part of PP_SIGNATURE; meaning we can just use the
4834 * space between the PP_SIGNATURE value (without POISON_POINTER_DELTA), and the
4835 * lowest bits of POISON_POINTER_DELTA. On arches where POISON_POINTER_DELTA is
4836 * 0, we use the lowest bit of PAGE_OFFSET as the boundary if that value is
4837 * known at compile-time.
4838 *
4839 * If the value of PAGE_OFFSET is not known at compile time, or if it is too
4840 * small to leave at least 8 bits available above PP_SIGNATURE, we define the
4841 * number of bits to be 0, which turns off the DMA index tracking altogether
4842 * (see page_pool_register_dma_index()).
4843 */
4844#define PP_DMA_INDEX_SHIFT (1 + __fls(PP_SIGNATURE - POISON_POINTER_DELTA))
4845#if POISON_POINTER_DELTA > 0
4846/* PP_SIGNATURE includes POISON_POINTER_DELTA, so limit the size of the DMA
4847 * index to not overlap with that if set
4848 */
4849#define PP_DMA_INDEX_BITS MIN(32, __ffs(POISON_POINTER_DELTA) - PP_DMA_INDEX_SHIFT)
4850#else
4851/* Use the lowest bit of PAGE_OFFSET if there's at least 8 bits available; see above */
4852#define PP_DMA_INDEX_MIN_OFFSET (1 << (PP_DMA_INDEX_SHIFT + 8))
4853#define PP_DMA_INDEX_BITS ((__builtin_constant_p(PAGE_OFFSET) && \
4854 PAGE_OFFSET >= PP_DMA_INDEX_MIN_OFFSET && \
4855 !(PAGE_OFFSET & (PP_DMA_INDEX_MIN_OFFSET - 1))) ? \
4856 MIN(32, __ffs(PAGE_OFFSET) - PP_DMA_INDEX_SHIFT) : 0)
4857
4858#endif
4859
4860#define PP_DMA_INDEX_MASK GENMASK(PP_DMA_INDEX_BITS + PP_DMA_INDEX_SHIFT - 1, \
4861 PP_DMA_INDEX_SHIFT)
4862
4863/* Mask used for checking in page_pool_page_is_pp() below. page->pp_magic is
4864 * OR'ed with PP_SIGNATURE after the allocation in order to preserve bit 0 for
4865 * the head page of compound page and bit 1 for pfmemalloc page, as well as the
4866 * bits used for the DMA index. page_is_pfmemalloc() is checked in
4867 * __page_pool_put_page() to avoid recycling the pfmemalloc page.
4868 */
4869#define PP_MAGIC_MASK ~(PP_DMA_INDEX_MASK | 0x3UL)
4870
4871#ifdef CONFIG_PAGE_POOL
4872static inline bool page_pool_page_is_pp(const struct page *page)
4873{
4874 return (page->pp_magic & PP_MAGIC_MASK) == PP_SIGNATURE;
4875}
4876#else
4877static inline bool page_pool_page_is_pp(const struct page *page)
4878{
4879 return false;
4880}
4881#endif
4882
4883#define PAGE_SNAPSHOT_FAITHFUL (1 << 0)
4884#define PAGE_SNAPSHOT_PG_BUDDY (1 << 1)
4885#define PAGE_SNAPSHOT_PG_IDLE (1 << 2)
4886
4887struct page_snapshot {
4888 struct folio folio_snapshot;
4889 struct page page_snapshot;
4890 unsigned long pfn;
4891 unsigned long idx;
4892 unsigned long flags;
4893};
4894
4895static inline bool snapshot_page_is_faithful(const struct page_snapshot *ps)
4896{
4897 return ps->flags & PAGE_SNAPSHOT_FAITHFUL;
4898}
4899
4900void snapshot_page(struct page_snapshot *ps, const struct page *page);
4901
4902#endif /* _LINUX_MM_H */