vdso/datastore: Allocate data pages dynamically

Allocating the data pages as part of the kernel image does not work on
SPARC. The MMU will raise a fault when userspace tries to access them.

Allocate the data pages through the page allocator instead.

Unused pages in the vDSO VMA are still allocated to keep the virtual
addresses aligned. Switch the mapping from PFNs to 'struct page' as that is
required for dynamically allocated pages. This also aligns the allocation
of the datapages with the code pages and is a prerequisite for mlockall()
support.

VM_MIXEDMAP is necessary for the call to vmf_insert_page() in the timens
prefault path to work.

The data pages need to be order-0, non-compound pages so that the mapping
to userspace and the different orderings work.

These pages are also used by the timekeeping, random pool and architecture
initialization code. Some of these are running before the page allocator is
available. To keep these subsytems working without changes, introduce
early, statically data storage which will then replaced by the real one as
soon as that is available.

Signed-off-by: Thomas Weißschuh <thomas.weissschuh@linutronix.de>
Signed-off-by: Thomas Gleixner <tglx@kernel.org>
Reviewed-by: Christophe Leroy (CS GROUP) <chleroy@kernel.org>
Link: https://patch.msgid.link/20260304-vdso-sparc64-generic-2-v6-3-d8eb3b0e1410@linutronix.de

authored by

Thomas Weißschuh and committed by

Thomas Gleixner 3 months ago 05988dba c0c9439b

+68 -32

3 changed files

expand all

include

linux

vdso_datastore.h

init

main.c

lib

vdso

datastore.c

include/linux/vdso_datastore.h

··· 2 2 #ifndef _LINUX_VDSO_DATASTORE_H 3 3 #define _LINUX_VDSO_DATASTORE_H 4 4 5 + #ifdef CONFIG_HAVE_GENERIC_VDSO 5 6 #include <linux/mm_types.h> 6 7 7 8 extern const struct vm_special_mapping vdso_vvar_mapping; 8 9 struct vm_area_struct *vdso_install_vvar_mapping(struct mm_struct *mm, unsigned long addr); 10 + 11 + void __init vdso_setup_data_pages(void); 12 + #else /* !CONFIG_HAVE_GENERIC_VDSO */ 13 + static inline void vdso_setup_data_pages(void) { } 14 + #endif /* CONFIG_HAVE_GENERIC_VDSO */ 9 15 10 16 #endif /* _LINUX_VDSO_DATASTORE_H */

init/main.c

··· 105 105 #include <linux/ptdump.h> 106 106 #include <linux/time_namespace.h> 107 107 #include <linux/unaligned.h> 108 + #include <linux/vdso_datastore.h> 108 109 #include <net/net_namespace.h> 109 110 110 111 #include <asm/io.h> ··· 1120 1119 srcu_init(); 1121 1120 hrtimers_init(); 1122 1121 softirq_init(); 1122 + vdso_setup_data_pages(); 1123 1123 timekeeping_init(); 1124 1124 time_init(); 1125 1125

+60 -32

lib/vdso/datastore.c

··· 1 1 // SPDX-License-Identifier: GPL-2.0-only 2 2 3 - #include <linux/linkage.h> 3 + #include <linux/gfp.h> 4 + #include <linux/init.h> 4 5 #include <linux/mm.h> 5 6 #include <linux/time_namespace.h> 6 7 #include <linux/types.h> 7 8 #include <linux/vdso_datastore.h> 8 9 #include <vdso/datapage.h> 9 10 10 - /* 11 - * The vDSO data page. 12 - */ 11 + static u8 vdso_initdata[VDSO_NR_PAGES * PAGE_SIZE] __aligned(PAGE_SIZE) __initdata = {}; 12 + 13 13 #ifdef CONFIG_GENERIC_GETTIMEOFDAY 14 - static union { 15 - struct vdso_time_data data; 16 - u8 page[PAGE_SIZE]; 17 - } vdso_time_data_store __page_aligned_data; 18 - struct vdso_time_data *vdso_k_time_data = &vdso_time_data_store.data; 19 - static_assert(sizeof(vdso_time_data_store) == PAGE_SIZE); 14 + struct vdso_time_data *vdso_k_time_data __refdata = 15 + (void *)&vdso_initdata[VDSO_TIME_PAGE_OFFSET * PAGE_SIZE]; 16 + 17 + static_assert(sizeof(struct vdso_time_data) <= PAGE_SIZE); 20 18 #endif /* CONFIG_GENERIC_GETTIMEOFDAY */ 21 19 22 20 #ifdef CONFIG_VDSO_GETRANDOM 23 - static union { 24 - struct vdso_rng_data data; 25 - u8 page[PAGE_SIZE]; 26 - } vdso_rng_data_store __page_aligned_data; 27 - struct vdso_rng_data *vdso_k_rng_data = &vdso_rng_data_store.data; 28 - static_assert(sizeof(vdso_rng_data_store) == PAGE_SIZE); 21 + struct vdso_rng_data *vdso_k_rng_data __refdata = 22 + (void *)&vdso_initdata[VDSO_RNG_PAGE_OFFSET * PAGE_SIZE]; 23 + 24 + static_assert(sizeof(struct vdso_rng_data) <= PAGE_SIZE); 29 25 #endif /* CONFIG_VDSO_GETRANDOM */ 30 26 31 27 #ifdef CONFIG_ARCH_HAS_VDSO_ARCH_DATA 32 - static union { 33 - struct vdso_arch_data data; 34 - u8 page[VDSO_ARCH_DATA_SIZE]; 35 - } vdso_arch_data_store __page_aligned_data; 36 - struct vdso_arch_data *vdso_k_arch_data = &vdso_arch_data_store.data; 28 + struct vdso_arch_data *vdso_k_arch_data __refdata = 29 + (void *)&vdso_initdata[VDSO_ARCH_PAGES_START * PAGE_SIZE]; 37 30 #endif /* CONFIG_ARCH_HAS_VDSO_ARCH_DATA */ 31 + 32 + void __init vdso_setup_data_pages(void) 33 + { 34 + unsigned int order = get_order(VDSO_NR_PAGES * PAGE_SIZE); 35 + struct page *pages; 36 + 37 + /* 38 + * Allocate the data pages dynamically. SPARC does not support mapping 39 + * static pages to be mapped into userspace. 40 + * It is also a requirement for mlockall() support. 41 + * 42 + * Do not use folios. In time namespaces the pages are mapped in a different order 43 + * to userspace, which is not handled by the folio optimizations in finish_fault(). 44 + */ 45 + pages = alloc_pages(GFP_KERNEL, order); 46 + if (!pages) 47 + panic("Unable to allocate VDSO storage pages"); 48 + 49 + /* The pages are mapped one-by-one into userspace and each one needs to be refcounted. */ 50 + split_page(pages, order); 51 + 52 + /* Move the data already written by other subsystems to the new pages */ 53 + memcpy(page_address(pages), vdso_initdata, VDSO_NR_PAGES * PAGE_SIZE); 54 + 55 + if (IS_ENABLED(CONFIG_GENERIC_GETTIMEOFDAY)) 56 + vdso_k_time_data = page_address(pages + VDSO_TIME_PAGE_OFFSET); 57 + 58 + if (IS_ENABLED(CONFIG_VDSO_GETRANDOM)) 59 + vdso_k_rng_data = page_address(pages + VDSO_RNG_PAGE_OFFSET); 60 + 61 + if (IS_ENABLED(CONFIG_ARCH_HAS_VDSO_ARCH_DATA)) 62 + vdso_k_arch_data = page_address(pages + VDSO_ARCH_PAGES_START); 63 + } 38 64 39 65 static vm_fault_t vvar_fault(const struct vm_special_mapping *sm, 40 66 struct vm_area_struct *vma, struct vm_fault *vmf) 41 67 { 42 - struct page *timens_page = find_timens_vvar_page(vma); 43 - unsigned long pfn; 68 + struct page *page, *timens_page; 69 + 70 + timens_page = find_timens_vvar_page(vma); 44 71 45 72 switch (vmf->pgoff) { 46 73 case VDSO_TIME_PAGE_OFFSET: 47 74 if (!IS_ENABLED(CONFIG_GENERIC_GETTIMEOFDAY)) 48 75 return VM_FAULT_SIGBUS; 49 - pfn = __phys_to_pfn(__pa_symbol(vdso_k_time_data)); 76 + page = virt_to_page(vdso_k_time_data); 50 77 if (timens_page) { 51 78 /* 52 79 * Fault in VVAR page too, since it will be accessed ··· 83 56 vm_fault_t err; 84 57 85 58 addr = vmf->address + VDSO_TIMENS_PAGE_OFFSET * PAGE_SIZE; 86 - err = vmf_insert_pfn(vma, addr, pfn); 59 + err = vmf_insert_page(vma, addr, page); 87 60 if (unlikely(err & VM_FAULT_ERROR)) 88 61 return err; 89 - pfn = page_to_pfn(timens_page); 62 + page = timens_page; 90 63 } 91 64 break; 92 65 case VDSO_TIMENS_PAGE_OFFSET: ··· 99 72 */ 100 73 if (!IS_ENABLED(CONFIG_TIME_NS) || !timens_page) 101 74 return VM_FAULT_SIGBUS; 102 - pfn = __phys_to_pfn(__pa_symbol(vdso_k_time_data)); 75 + page = virt_to_page(vdso_k_time_data); 103 76 break; 104 77 case VDSO_RNG_PAGE_OFFSET: 105 78 if (!IS_ENABLED(CONFIG_VDSO_GETRANDOM)) 106 79 return VM_FAULT_SIGBUS; 107 - pfn = __phys_to_pfn(__pa_symbol(vdso_k_rng_data)); 80 + page = virt_to_page(vdso_k_rng_data); 108 81 break; 109 82 case VDSO_ARCH_PAGES_START ... VDSO_ARCH_PAGES_END: 110 83 if (!IS_ENABLED(CONFIG_ARCH_HAS_VDSO_ARCH_DATA)) 111 84 return VM_FAULT_SIGBUS; 112 - pfn = __phys_to_pfn(__pa_symbol(vdso_k_arch_data)) + 113 - vmf->pgoff - VDSO_ARCH_PAGES_START; 85 + page = virt_to_page(vdso_k_arch_data) + vmf->pgoff - VDSO_ARCH_PAGES_START; 114 86 break; 115 87 default: 116 88 return VM_FAULT_SIGBUS; 117 89 } 118 90 119 - return vmf_insert_pfn(vma, vmf->address, pfn); 91 + get_page(page); 92 + vmf->page = page; 93 + return 0; 120 94 } 121 95 122 96 const struct vm_special_mapping vdso_vvar_mapping = { ··· 129 101 { 130 102 return _install_special_mapping(mm, addr, VDSO_NR_PAGES * PAGE_SIZE, 131 103 VM_READ | VM_MAYREAD | VM_IO | VM_DONTDUMP | 132 - VM_PFNMAP | VM_SEALED_SYSMAP, 104 + VM_MIXEDMAP | VM_SEALED_SYSMAP, 133 105 &vdso_vvar_mapping); 134 106 } 135 107

Configure Feed

Configure Feed