Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * Describes operations that can be performed on software-defined page table
4 * leaf entries. These are abstracted from the hardware page table entries
5 * themselves by the softleaf_t type, see mm_types.h.
6 */
7#ifndef _LINUX_LEAFOPS_H
8#define _LINUX_LEAFOPS_H
9
10#include <linux/mm_types.h>
11#include <linux/swapops.h>
12#include <linux/swap.h>
13
14#ifdef CONFIG_MMU
15
16/* Temporary until swp_entry_t eliminated. */
17#define LEAF_TYPE_SHIFT SWP_TYPE_SHIFT
18
19enum softleaf_type {
20 /* Fundamental types. */
21 SOFTLEAF_NONE,
22 SOFTLEAF_SWAP,
23 /* Migration types. */
24 SOFTLEAF_MIGRATION_READ,
25 SOFTLEAF_MIGRATION_READ_EXCLUSIVE,
26 SOFTLEAF_MIGRATION_WRITE,
27 /* Device types. */
28 SOFTLEAF_DEVICE_PRIVATE_READ,
29 SOFTLEAF_DEVICE_PRIVATE_WRITE,
30 SOFTLEAF_DEVICE_EXCLUSIVE,
31 /* H/W posion types. */
32 SOFTLEAF_HWPOISON,
33 /* Marker types. */
34 SOFTLEAF_MARKER,
35};
36
37/**
38 * softleaf_mk_none() - Create an empty ('none') leaf entry.
39 * Returns: empty leaf entry.
40 */
41static inline softleaf_t softleaf_mk_none(void)
42{
43 return ((softleaf_t) { 0 });
44}
45
46/**
47 * softleaf_from_pte() - Obtain a leaf entry from a PTE entry.
48 * @pte: PTE entry.
49 *
50 * If @pte is present (therefore not a leaf entry) the function returns an empty
51 * leaf entry. Otherwise, it returns a leaf entry.
52 *
53 * Returns: Leaf entry.
54 */
55static inline softleaf_t softleaf_from_pte(pte_t pte)
56{
57 softleaf_t arch_entry;
58
59 if (pte_present(pte) || pte_none(pte))
60 return softleaf_mk_none();
61
62 pte = pte_swp_clear_flags(pte);
63 arch_entry = __pte_to_swp_entry(pte);
64
65 /* Temporary until swp_entry_t eliminated. */
66 return swp_entry(__swp_type(arch_entry), __swp_offset(arch_entry));
67}
68
69/**
70 * softleaf_to_pte() - Obtain a PTE entry from a leaf entry.
71 * @entry: Leaf entry.
72 *
73 * This generates an architecture-specific PTE entry that can be utilised to
74 * encode the metadata the leaf entry encodes.
75 *
76 * Returns: Architecture-specific PTE entry encoding leaf entry.
77 */
78static inline pte_t softleaf_to_pte(softleaf_t entry)
79{
80 /* Temporary until swp_entry_t eliminated. */
81 return swp_entry_to_pte(entry);
82}
83
84#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
85/**
86 * softleaf_from_pmd() - Obtain a leaf entry from a PMD entry.
87 * @pmd: PMD entry.
88 *
89 * If @pmd is present (therefore not a leaf entry) the function returns an empty
90 * leaf entry. Otherwise, it returns a leaf entry.
91 *
92 * Returns: Leaf entry.
93 */
94static inline softleaf_t softleaf_from_pmd(pmd_t pmd)
95{
96 softleaf_t arch_entry;
97
98 if (pmd_present(pmd) || pmd_none(pmd))
99 return softleaf_mk_none();
100
101 if (pmd_swp_soft_dirty(pmd))
102 pmd = pmd_swp_clear_soft_dirty(pmd);
103 if (pmd_swp_uffd_wp(pmd))
104 pmd = pmd_swp_clear_uffd_wp(pmd);
105 arch_entry = __pmd_to_swp_entry(pmd);
106
107 /* Temporary until swp_entry_t eliminated. */
108 return swp_entry(__swp_type(arch_entry), __swp_offset(arch_entry));
109}
110
111#else
112
113static inline softleaf_t softleaf_from_pmd(pmd_t pmd)
114{
115 return softleaf_mk_none();
116}
117
118#endif
119
120/**
121 * softleaf_is_none() - Is the leaf entry empty?
122 * @entry: Leaf entry.
123 *
124 * Empty entries are typically the result of a 'none' page table leaf entry
125 * being converted to a leaf entry.
126 *
127 * Returns: true if the entry is empty, false otherwise.
128 */
129static inline bool softleaf_is_none(softleaf_t entry)
130{
131 return entry.val == 0;
132}
133
134/**
135 * softleaf_type() - Identify the type of leaf entry.
136 * @entry: Leaf entry.
137 *
138 * Returns: the leaf entry type associated with @entry.
139 */
140static inline enum softleaf_type softleaf_type(softleaf_t entry)
141{
142 unsigned int type_num;
143
144 if (softleaf_is_none(entry))
145 return SOFTLEAF_NONE;
146
147 type_num = entry.val >> LEAF_TYPE_SHIFT;
148
149 if (type_num < MAX_SWAPFILES)
150 return SOFTLEAF_SWAP;
151
152 switch (type_num) {
153#ifdef CONFIG_MIGRATION
154 case SWP_MIGRATION_READ:
155 return SOFTLEAF_MIGRATION_READ;
156 case SWP_MIGRATION_READ_EXCLUSIVE:
157 return SOFTLEAF_MIGRATION_READ_EXCLUSIVE;
158 case SWP_MIGRATION_WRITE:
159 return SOFTLEAF_MIGRATION_WRITE;
160#endif
161#ifdef CONFIG_DEVICE_PRIVATE
162 case SWP_DEVICE_WRITE:
163 return SOFTLEAF_DEVICE_PRIVATE_WRITE;
164 case SWP_DEVICE_READ:
165 return SOFTLEAF_DEVICE_PRIVATE_READ;
166 case SWP_DEVICE_EXCLUSIVE:
167 return SOFTLEAF_DEVICE_EXCLUSIVE;
168#endif
169#ifdef CONFIG_MEMORY_FAILURE
170 case SWP_HWPOISON:
171 return SOFTLEAF_HWPOISON;
172#endif
173 case SWP_PTE_MARKER:
174 return SOFTLEAF_MARKER;
175 }
176
177 /* Unknown entry type. */
178 VM_WARN_ON_ONCE(1);
179 return SOFTLEAF_NONE;
180}
181
182/**
183 * softleaf_is_swap() - Is this leaf entry a swap entry?
184 * @entry: Leaf entry.
185 *
186 * Returns: true if the leaf entry is a swap entry, otherwise false.
187 */
188static inline bool softleaf_is_swap(softleaf_t entry)
189{
190 return softleaf_type(entry) == SOFTLEAF_SWAP;
191}
192
193/**
194 * softleaf_is_migration_write() - Is this leaf entry a writable migration entry?
195 * @entry: Leaf entry.
196 *
197 * Returns: true if the leaf entry is a writable migration entry, otherwise
198 * false.
199 */
200static inline bool softleaf_is_migration_write(softleaf_t entry)
201{
202 return softleaf_type(entry) == SOFTLEAF_MIGRATION_WRITE;
203}
204
205/**
206 * softleaf_is_migration_read() - Is this leaf entry a readable migration entry?
207 * @entry: Leaf entry.
208 *
209 * Returns: true if the leaf entry is a readable migration entry, otherwise
210 * false.
211 */
212static inline bool softleaf_is_migration_read(softleaf_t entry)
213{
214 return softleaf_type(entry) == SOFTLEAF_MIGRATION_READ;
215}
216
217/**
218 * softleaf_is_migration_read_exclusive() - Is this leaf entry an exclusive
219 * readable migration entry?
220 * @entry: Leaf entry.
221 *
222 * Returns: true if the leaf entry is an exclusive readable migration entry,
223 * otherwise false.
224 */
225static inline bool softleaf_is_migration_read_exclusive(softleaf_t entry)
226{
227 return softleaf_type(entry) == SOFTLEAF_MIGRATION_READ_EXCLUSIVE;
228}
229
230/**
231 * softleaf_is_migration() - Is this leaf entry a migration entry?
232 * @entry: Leaf entry.
233 *
234 * Returns: true if the leaf entry is a migration entry, otherwise false.
235 */
236static inline bool softleaf_is_migration(softleaf_t entry)
237{
238 switch (softleaf_type(entry)) {
239 case SOFTLEAF_MIGRATION_READ:
240 case SOFTLEAF_MIGRATION_READ_EXCLUSIVE:
241 case SOFTLEAF_MIGRATION_WRITE:
242 return true;
243 default:
244 return false;
245 }
246}
247
248/**
249 * softleaf_is_device_private_write() - Is this leaf entry a device private
250 * writable entry?
251 * @entry: Leaf entry.
252 *
253 * Returns: true if the leaf entry is a device private writable entry, otherwise
254 * false.
255 */
256static inline bool softleaf_is_device_private_write(softleaf_t entry)
257{
258 return softleaf_type(entry) == SOFTLEAF_DEVICE_PRIVATE_WRITE;
259}
260
261/**
262 * softleaf_is_device_private() - Is this leaf entry a device private entry?
263 * @entry: Leaf entry.
264 *
265 * Returns: true if the leaf entry is a device private entry, otherwise false.
266 */
267static inline bool softleaf_is_device_private(softleaf_t entry)
268{
269 switch (softleaf_type(entry)) {
270 case SOFTLEAF_DEVICE_PRIVATE_WRITE:
271 case SOFTLEAF_DEVICE_PRIVATE_READ:
272 return true;
273 default:
274 return false;
275 }
276}
277
278/**
279 * softleaf_is_device_exclusive() - Is this leaf entry a device-exclusive entry?
280 * @entry: Leaf entry.
281 *
282 * Returns: true if the leaf entry is a device-exclusive entry, otherwise false.
283 */
284static inline bool softleaf_is_device_exclusive(softleaf_t entry)
285{
286 return softleaf_type(entry) == SOFTLEAF_DEVICE_EXCLUSIVE;
287}
288
289/**
290 * softleaf_is_hwpoison() - Is this leaf entry a hardware poison entry?
291 * @entry: Leaf entry.
292 *
293 * Returns: true if the leaf entry is a hardware poison entry, otherwise false.
294 */
295static inline bool softleaf_is_hwpoison(softleaf_t entry)
296{
297 return softleaf_type(entry) == SOFTLEAF_HWPOISON;
298}
299
300/**
301 * softleaf_is_marker() - Is this leaf entry a marker?
302 * @entry: Leaf entry.
303 *
304 * Returns: true if the leaf entry is a marker entry, otherwise false.
305 */
306static inline bool softleaf_is_marker(softleaf_t entry)
307{
308 return softleaf_type(entry) == SOFTLEAF_MARKER;
309}
310
311/**
312 * softleaf_to_marker() - Obtain marker associated with leaf entry.
313 * @entry: Leaf entry, softleaf_is_marker(@entry) must return true.
314 *
315 * Returns: Marker associated with the leaf entry.
316 */
317static inline pte_marker softleaf_to_marker(softleaf_t entry)
318{
319 VM_WARN_ON_ONCE(!softleaf_is_marker(entry));
320
321 return swp_offset(entry) & PTE_MARKER_MASK;
322}
323
324/**
325 * softleaf_has_pfn() - Does this leaf entry encode a valid PFN number?
326 * @entry: Leaf entry.
327 *
328 * A pfn swap entry is a special type of swap entry that always has a pfn stored
329 * in the swap offset. They can either be used to represent unaddressable device
330 * memory, to restrict access to a page undergoing migration or to represent a
331 * pfn which has been hwpoisoned and unmapped.
332 *
333 * Returns: true if the leaf entry encodes a PFN, otherwise false.
334 */
335static inline bool softleaf_has_pfn(softleaf_t entry)
336{
337 /* Make sure the swp offset can always store the needed fields. */
338 BUILD_BUG_ON(SWP_TYPE_SHIFT < SWP_PFN_BITS);
339
340 if (softleaf_is_migration(entry))
341 return true;
342 if (softleaf_is_device_private(entry))
343 return true;
344 if (softleaf_is_device_exclusive(entry))
345 return true;
346 if (softleaf_is_hwpoison(entry))
347 return true;
348
349 return false;
350}
351
352/**
353 * softleaf_to_pfn() - Obtain PFN encoded within leaf entry.
354 * @entry: Leaf entry, softleaf_has_pfn(@entry) must return true.
355 *
356 * Returns: The PFN associated with the leaf entry.
357 */
358static inline unsigned long softleaf_to_pfn(softleaf_t entry)
359{
360 VM_WARN_ON_ONCE(!softleaf_has_pfn(entry));
361
362 /* Temporary until swp_entry_t eliminated. */
363 return swp_offset(entry) & SWP_PFN_MASK;
364}
365
366static inline void softleaf_migration_sync(softleaf_t entry,
367 struct folio *folio)
368{
369 /*
370 * Ensure we do not race with split, which might alter tail pages into new
371 * folios and thus result in observing an unlocked folio.
372 * This matches the write barrier in __split_folio_to_order().
373 */
374 smp_rmb();
375
376 /*
377 * Any use of migration entries may only occur while the
378 * corresponding page is locked
379 */
380 VM_WARN_ON_ONCE(!folio_test_locked(folio));
381}
382
383/**
384 * softleaf_to_page() - Obtains struct page for PFN encoded within leaf entry.
385 * @entry: Leaf entry, softleaf_has_pfn(@entry) must return true.
386 *
387 * Returns: Pointer to the struct page associated with the leaf entry's PFN.
388 */
389static inline struct page *softleaf_to_page(softleaf_t entry)
390{
391 struct page *page = pfn_to_page(softleaf_to_pfn(entry));
392
393 VM_WARN_ON_ONCE(!softleaf_has_pfn(entry));
394 if (softleaf_is_migration(entry))
395 softleaf_migration_sync(entry, page_folio(page));
396
397 return page;
398}
399
400/**
401 * softleaf_to_folio() - Obtains struct folio for PFN encoded within leaf entry.
402 * @entry: Leaf entry, softleaf_has_pfn(@entry) must return true.
403 *
404 * Returns: Pointer to the struct folio associated with the leaf entry's PFN.
405 */
406static inline struct folio *softleaf_to_folio(softleaf_t entry)
407{
408 struct folio *folio = pfn_folio(softleaf_to_pfn(entry));
409
410 VM_WARN_ON_ONCE(!softleaf_has_pfn(entry));
411 if (softleaf_is_migration(entry))
412 softleaf_migration_sync(entry, folio);
413
414 return folio;
415}
416
417/**
418 * softleaf_is_poison_marker() - Is this leaf entry a poison marker?
419 * @entry: Leaf entry.
420 *
421 * The poison marker is set via UFFDIO_POISON. Userfaultfd-specific.
422 *
423 * Returns: true if the leaf entry is a poison marker, otherwise false.
424 */
425static inline bool softleaf_is_poison_marker(softleaf_t entry)
426{
427 if (!softleaf_is_marker(entry))
428 return false;
429
430 return softleaf_to_marker(entry) & PTE_MARKER_POISONED;
431}
432
433/**
434 * softleaf_is_guard_marker() - Is this leaf entry a guard region marker?
435 * @entry: Leaf entry.
436 *
437 * Returns: true if the leaf entry is a guard marker, otherwise false.
438 */
439static inline bool softleaf_is_guard_marker(softleaf_t entry)
440{
441 if (!softleaf_is_marker(entry))
442 return false;
443
444 return softleaf_to_marker(entry) & PTE_MARKER_GUARD;
445}
446
447/**
448 * softleaf_is_uffd_wp_marker() - Is this leaf entry a userfautlfd write protect
449 * marker?
450 * @entry: Leaf entry.
451 *
452 * Userfaultfd-specific.
453 *
454 * Returns: true if the leaf entry is a UFFD WP marker, otherwise false.
455 */
456static inline bool softleaf_is_uffd_wp_marker(softleaf_t entry)
457{
458 if (!softleaf_is_marker(entry))
459 return false;
460
461 return softleaf_to_marker(entry) & PTE_MARKER_UFFD_WP;
462}
463
464#ifdef CONFIG_MIGRATION
465
466/**
467 * softleaf_is_migration_young() - Does this migration entry contain an accessed
468 * bit?
469 * @entry: Leaf entry.
470 *
471 * If the architecture can support storing A/D bits in migration entries, this
472 * determines whether the accessed (or 'young') bit was set on the migrated page
473 * table entry.
474 *
475 * Returns: true if the entry contains an accessed bit, otherwise false.
476 */
477static inline bool softleaf_is_migration_young(softleaf_t entry)
478{
479 VM_WARN_ON_ONCE(!softleaf_is_migration(entry));
480
481 if (migration_entry_supports_ad())
482 return swp_offset(entry) & SWP_MIG_YOUNG;
483 /* Keep the old behavior of aging page after migration */
484 return false;
485}
486
487/**
488 * softleaf_is_migration_dirty() - Does this migration entry contain a dirty bit?
489 * @entry: Leaf entry.
490 *
491 * If the architecture can support storing A/D bits in migration entries, this
492 * determines whether the dirty bit was set on the migrated page table entry.
493 *
494 * Returns: true if the entry contains a dirty bit, otherwise false.
495 */
496static inline bool softleaf_is_migration_dirty(softleaf_t entry)
497{
498 VM_WARN_ON_ONCE(!softleaf_is_migration(entry));
499
500 if (migration_entry_supports_ad())
501 return swp_offset(entry) & SWP_MIG_DIRTY;
502 /* Keep the old behavior of clean page after migration */
503 return false;
504}
505
506#else /* CONFIG_MIGRATION */
507
508static inline bool softleaf_is_migration_young(softleaf_t entry)
509{
510 return false;
511}
512
513static inline bool softleaf_is_migration_dirty(softleaf_t entry)
514{
515 return false;
516}
517#endif /* CONFIG_MIGRATION */
518
519/**
520 * pte_is_marker() - Does the PTE entry encode a marker leaf entry?
521 * @pte: PTE entry.
522 *
523 * Returns: true if this PTE is a marker leaf entry, otherwise false.
524 */
525static inline bool pte_is_marker(pte_t pte)
526{
527 return softleaf_is_marker(softleaf_from_pte(pte));
528}
529
530/**
531 * pte_is_uffd_wp_marker() - Does this PTE entry encode a userfaultfd write
532 * protect marker leaf entry?
533 * @pte: PTE entry.
534 *
535 * Returns: true if this PTE is a UFFD WP marker leaf entry, otherwise false.
536 */
537static inline bool pte_is_uffd_wp_marker(pte_t pte)
538{
539 const softleaf_t entry = softleaf_from_pte(pte);
540
541 return softleaf_is_uffd_wp_marker(entry);
542}
543
544/**
545 * pte_is_uffd_marker() - Does this PTE entry encode a userfault-specific marker
546 * leaf entry?
547 * @pte: PTE entry.
548 *
549 * It's useful to be able to determine which leaf entries encode UFFD-specific
550 * markers so we can handle these correctly.
551 *
552 * Returns: true if this PTE entry is a UFFD-specific marker, otherwise false.
553 */
554static inline bool pte_is_uffd_marker(pte_t pte)
555{
556 const softleaf_t entry = softleaf_from_pte(pte);
557
558 if (!softleaf_is_marker(entry))
559 return false;
560
561 /* UFFD WP, poisoned swap entries are UFFD-handled. */
562 if (softleaf_is_uffd_wp_marker(entry))
563 return true;
564 if (softleaf_is_poison_marker(entry))
565 return true;
566
567 return false;
568}
569
570#if defined(CONFIG_ZONE_DEVICE) && defined(CONFIG_ARCH_ENABLE_THP_MIGRATION)
571
572/**
573 * pmd_is_device_private_entry() - Check if PMD contains a device private swap
574 * entry.
575 * @pmd: The PMD to check.
576 *
577 * Returns true if the PMD contains a swap entry that represents a device private
578 * page mapping. This is used for zone device private pages that have been
579 * swapped out but still need special handling during various memory management
580 * operations.
581 *
582 * Return: true if PMD contains device private entry, false otherwise
583 */
584static inline bool pmd_is_device_private_entry(pmd_t pmd)
585{
586 return softleaf_is_device_private(softleaf_from_pmd(pmd));
587}
588
589#else /* CONFIG_ZONE_DEVICE && CONFIG_ARCH_ENABLE_THP_MIGRATION */
590
591static inline bool pmd_is_device_private_entry(pmd_t pmd)
592{
593 return false;
594}
595
596#endif /* CONFIG_ZONE_DEVICE && CONFIG_ARCH_ENABLE_THP_MIGRATION */
597
598/**
599 * pmd_is_migration_entry() - Does this PMD entry encode a migration entry?
600 * @pmd: PMD entry.
601 *
602 * Returns: true if the PMD encodes a migration entry, otherwise false.
603 */
604static inline bool pmd_is_migration_entry(pmd_t pmd)
605{
606 return softleaf_is_migration(softleaf_from_pmd(pmd));
607}
608
609/**
610 * softleaf_is_valid_pmd_entry() - Is the specified softleaf entry obtained from
611 * a PMD one that we support at PMD level?
612 * @entry: Entry to check.
613 * Returns: true if the softleaf entry is valid at PMD, otherwise false.
614 */
615static inline bool softleaf_is_valid_pmd_entry(softleaf_t entry)
616{
617 /* Only device private, migration entries valid for PMD. */
618 return softleaf_is_device_private(entry) ||
619 softleaf_is_migration(entry);
620}
621
622/**
623 * pmd_is_valid_softleaf() - Is this PMD entry a valid softleaf entry?
624 * @pmd: PMD entry.
625 *
626 * PMD leaf entries are valid only if they are device private or migration
627 * entries. This function asserts that a PMD leaf entry is valid in this
628 * respect.
629 *
630 * Returns: true if the PMD entry is a valid leaf entry, otherwise false.
631 */
632static inline bool pmd_is_valid_softleaf(pmd_t pmd)
633{
634 const softleaf_t entry = softleaf_from_pmd(pmd);
635
636 return softleaf_is_valid_pmd_entry(entry);
637}
638
639/**
640 * pmd_to_softleaf_folio() - Convert the PMD entry to a folio.
641 * @pmd: PMD entry.
642 *
643 * The PMD entry is expected to be a valid PMD softleaf entry.
644 *
645 * Returns: the folio the softleaf entry references if this is a valid softleaf
646 * entry, otherwise NULL.
647 */
648static inline struct folio *pmd_to_softleaf_folio(pmd_t pmd)
649{
650 const softleaf_t entry = softleaf_from_pmd(pmd);
651
652 if (!softleaf_is_valid_pmd_entry(entry)) {
653 VM_WARN_ON_ONCE(true);
654 return NULL;
655 }
656 return softleaf_to_folio(entry);
657}
658
659#endif /* CONFIG_MMU */
660#endif /* _LINUX_LEAFOPS_H */