userfaultfd: introduce vm_uffd_ops

Current userfaultfd implementation works only with memory managed by core
MM: anonymous, shmem and hugetlb.

First, there is no fundamental reason to limit userfaultfd support only to
the core memory types and userfaults can be handled similarly to regular
page faults provided a VMA owner implements appropriate callbacks.

Second, historically various code paths were conditioned on
vma_is_anonymous(), vma_is_shmem() and is_vm_hugetlb_page() and some of
these conditions can be expressed as operations implemented by a
particular memory type.

Introduce vm_uffd_ops extension to vm_operations_struct that will delegate
memory type specific operations to a VMA owner.

Operations for anonymous memory are handled internally in userfaultfd
using anon_uffd_ops that implicitly assigned to anonymous VMAs.

Start with a single operation, ->can_userfault() that will verify that a
VMA meets requirements for userfaultfd support at registration time.

Implement that method for anonymous, shmem and hugetlb and move relevant
parts of vma_can_userfault() into the new callbacks.

[rppt@kernel.org: relocate VM_DROPPABLE test, per Tal]
Link: https://lore.kernel.org/adffgfM5ANxtPIEF@kernel.org
Link: https://lore.kernel.org/20260402041156.1377214-8-rppt@kernel.org
Signed-off-by: Mike Rapoport (Microsoft) <rppt@kernel.org>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Andrei Vagin <avagin@google.com>
Cc: Axel Rasmussen <axelrasmussen@google.com>
Cc: Baolin Wang <baolin.wang@linux.alibaba.com>
Cc: David Hildenbrand (Arm) <david@kernel.org>
Cc: Harry Yoo <harry.yoo@oracle.com>
Cc: Harry Yoo (Oracle) <harry@kernel.org>
Cc: Hugh Dickins <hughd@google.com>
Cc: James Houghton <jthoughton@google.com>
Cc: Liam Howlett <liam.howlett@oracle.com>
Cc: Lorenzo Stoakes (Oracle) <ljs@kernel.org>
Cc: Matthew Wilcox (Oracle) <willy@infradead.org>
Cc: Michal Hocko <mhocko@suse.com>
Cc: Muchun Song <muchun.song@linux.dev>
Cc: Nikita Kalyazin <kalyazin@amazon.com>
Cc: Oscar Salvador <osalvador@suse.de>
Cc: Paolo Bonzini <pbonzini@redhat.com>
Cc: Peter Xu <peterx@redhat.com>
Cc: Sean Christopherson <seanjc@google.com>
Cc: Shuah Khan <shuah@kernel.org>
Cc: Suren Baghdasaryan <surenb@google.com>
Cc: Vlastimil Babka <vbabka@suse.cz>
Cc: David Carlier <devnexen@gmail.com>
Cc: Tal Zussman <tz2294@columbia.edu>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

authored by

Mike Rapoport (Microsoft) and committed by

Andrew Morton 1 month ago 0f48947c a5bb8669

+69 -10

5 changed files

expand all

include

linux

mm.h

userfaultfd_k.h

hugetlb.c

shmem.c

userfaultfd.c

include/linux/mm.h

··· 758 758 */ 759 759 }; 760 760 761 + struct vm_uffd_ops; 762 + 761 763 /* 762 764 * These are the virtual MM functions - opening of an area, closing and 763 765 * unmapping it (needed to keep files on disk up-to-date etc), pointer ··· 867 865 struct page *(*find_normal_page)(struct vm_area_struct *vma, 868 866 unsigned long addr); 869 867 #endif /* CONFIG_FIND_NORMAL_PAGE */ 868 + #ifdef CONFIG_USERFAULTFD 869 + const struct vm_uffd_ops *uffd_ops; 870 + #endif 870 871 }; 871 872 872 873 #ifdef CONFIG_NUMA_BALANCING

include/linux/userfaultfd_k.h

··· 83 83 84 84 extern vm_fault_t handle_userfault(struct vm_fault *vmf, unsigned long reason); 85 85 86 + /* VMA userfaultfd operations */ 87 + struct vm_uffd_ops { 88 + /* Checks if a VMA can support userfaultfd */ 89 + bool (*can_userfault)(struct vm_area_struct *vma, vm_flags_t vm_flags); 90 + }; 91 + 86 92 /* A combined operation mode + behavior flags. */ 87 93 typedef unsigned int __bitwise uffd_flags_t; 88 94

+15

mm/hugetlb.c

··· 4792 4792 return 0; 4793 4793 } 4794 4794 4795 + #ifdef CONFIG_USERFAULTFD 4796 + static bool hugetlb_can_userfault(struct vm_area_struct *vma, 4797 + vm_flags_t vm_flags) 4798 + { 4799 + return true; 4800 + } 4801 + 4802 + static const struct vm_uffd_ops hugetlb_uffd_ops = { 4803 + .can_userfault = hugetlb_can_userfault, 4804 + }; 4805 + #endif 4806 + 4795 4807 /* 4796 4808 * When a new function is introduced to vm_operations_struct and added 4797 4809 * to hugetlb_vm_ops, please consider adding the function to shm_vm_ops. ··· 4817 4805 .close = hugetlb_vm_op_close, 4818 4806 .may_split = hugetlb_vm_op_split, 4819 4807 .pagesize = hugetlb_vm_op_pagesize, 4808 + #ifdef CONFIG_USERFAULTFD 4809 + .uffd_ops = &hugetlb_uffd_ops, 4810 + #endif 4820 4811 }; 4821 4812 4822 4813 static pte_t make_huge_pte(struct vm_area_struct *vma, struct folio *folio,

+15

mm/shmem.c

··· 3288 3288 shmem_inode_unacct_blocks(inode, 1); 3289 3289 return ret; 3290 3290 } 3291 + 3292 + static bool shmem_can_userfault(struct vm_area_struct *vma, vm_flags_t vm_flags) 3293 + { 3294 + return true; 3295 + } 3296 + 3297 + static const struct vm_uffd_ops shmem_uffd_ops = { 3298 + .can_userfault = shmem_can_userfault, 3299 + }; 3291 3300 #endif /* CONFIG_USERFAULTFD */ 3292 3301 3293 3302 #ifdef CONFIG_TMPFS ··· 5316 5307 .set_policy = shmem_set_policy, 5317 5308 .get_policy = shmem_get_policy, 5318 5309 #endif 5310 + #ifdef CONFIG_USERFAULTFD 5311 + .uffd_ops = &shmem_uffd_ops, 5312 + #endif 5319 5313 }; 5320 5314 5321 5315 static const struct vm_operations_struct shmem_anon_vm_ops = { ··· 5327 5315 #ifdef CONFIG_NUMA 5328 5316 .set_policy = shmem_set_policy, 5329 5317 .get_policy = shmem_get_policy, 5318 + #endif 5319 + #ifdef CONFIG_USERFAULTFD 5320 + .uffd_ops = &shmem_uffd_ops, 5330 5321 #endif 5331 5322 }; 5332 5323

+28 -10

mm/userfaultfd.c

··· 34 34 pmd_t *pmd; 35 35 }; 36 36 37 + static bool anon_can_userfault(struct vm_area_struct *vma, vm_flags_t vm_flags) 38 + { 39 + /* anonymous memory does not support MINOR mode */ 40 + if (vm_flags & VM_UFFD_MINOR) 41 + return false; 42 + return true; 43 + } 44 + 45 + static const struct vm_uffd_ops anon_uffd_ops = { 46 + .can_userfault = anon_can_userfault, 47 + }; 48 + 49 + static const struct vm_uffd_ops *vma_uffd_ops(struct vm_area_struct *vma) 50 + { 51 + if (vma_is_anonymous(vma)) 52 + return &anon_uffd_ops; 53 + return vma->vm_ops ? vma->vm_ops->uffd_ops : NULL; 54 + } 55 + 37 56 static __always_inline 38 57 bool validate_dst_vma(struct vm_area_struct *dst_vma, unsigned long dst_end) 39 58 { ··· 2040 2021 bool vma_can_userfault(struct vm_area_struct *vma, vm_flags_t vm_flags, 2041 2022 bool wp_async) 2042 2023 { 2043 - vm_flags &= __VM_UFFD_FLAGS; 2024 + const struct vm_uffd_ops *ops = vma_uffd_ops(vma); 2044 2025 2045 2026 if (vma->vm_flags & VM_DROPPABLE) 2046 2027 return false; 2047 2028 2048 - if ((vm_flags & VM_UFFD_MINOR) && 2049 - (!is_vm_hugetlb_page(vma) && !vma_is_shmem(vma))) 2050 - return false; 2029 + vm_flags &= __VM_UFFD_FLAGS; 2051 2030 2052 2031 /* 2053 - * If wp async enabled, and WP is the only mode enabled, allow any 2032 + * If WP is the only mode enabled and context is wp async, allow any 2054 2033 * memory type. 2055 2034 */ 2056 2035 if (wp_async && (vm_flags == VM_UFFD_WP)) 2057 2036 return true; 2058 2037 2038 + /* For any other mode reject VMAs that don't implement vm_uffd_ops */ 2039 + if (!ops) 2040 + return false; 2041 + 2059 2042 /* 2060 2043 * If user requested uffd-wp but not enabled pte markers for 2061 - * uffd-wp, then shmem & hugetlbfs are not supported but only 2062 - * anonymous. 2044 + * uffd-wp, then only anonymous memory is supported 2063 2045 */ 2064 2046 if (!uffd_supports_wp_marker() && (vm_flags & VM_UFFD_WP) && 2065 2047 !vma_is_anonymous(vma)) 2066 2048 return false; 2067 2049 2068 - /* By default, allow any of anon|shmem|hugetlb */ 2069 - return vma_is_anonymous(vma) || is_vm_hugetlb_page(vma) || 2070 - vma_is_shmem(vma); 2050 + return ops->can_userfault(vma, vm_flags); 2071 2051 } 2072 2052 2073 2053 static void userfaultfd_set_vm_flags(struct vm_area_struct *vma,

Configure Feed

Configure Feed