Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1// SPDX-License-Identifier: GPL-2.0-only
2
3#include <linux/gfp.h>
4#include <linux/init.h>
5#include <linux/mm.h>
6#include <linux/time_namespace.h>
7#include <linux/types.h>
8#include <linux/vdso_datastore.h>
9#include <vdso/datapage.h>
10
11static u8 vdso_initdata[VDSO_NR_PAGES * PAGE_SIZE] __aligned(PAGE_SIZE) __initdata = {};
12
13#ifdef CONFIG_GENERIC_GETTIMEOFDAY
14struct vdso_time_data *vdso_k_time_data __refdata =
15 (void *)&vdso_initdata[VDSO_TIME_PAGE_OFFSET * PAGE_SIZE];
16
17static_assert(sizeof(struct vdso_time_data) <= PAGE_SIZE);
18#endif /* CONFIG_GENERIC_GETTIMEOFDAY */
19
20#ifdef CONFIG_VDSO_GETRANDOM
21struct vdso_rng_data *vdso_k_rng_data __refdata =
22 (void *)&vdso_initdata[VDSO_RNG_PAGE_OFFSET * PAGE_SIZE];
23
24static_assert(sizeof(struct vdso_rng_data) <= PAGE_SIZE);
25#endif /* CONFIG_VDSO_GETRANDOM */
26
27#ifdef CONFIG_ARCH_HAS_VDSO_ARCH_DATA
28struct vdso_arch_data *vdso_k_arch_data __refdata =
29 (void *)&vdso_initdata[VDSO_ARCH_PAGES_START * PAGE_SIZE];
30#endif /* CONFIG_ARCH_HAS_VDSO_ARCH_DATA */
31
32void __init vdso_setup_data_pages(void)
33{
34 unsigned int order = get_order(VDSO_NR_PAGES * PAGE_SIZE);
35 struct page *pages;
36
37 /*
38 * Allocate the data pages dynamically. SPARC does not support mapping
39 * static pages to be mapped into userspace.
40 * It is also a requirement for mlockall() support.
41 *
42 * Do not use folios. In time namespaces the pages are mapped in a different order
43 * to userspace, which is not handled by the folio optimizations in finish_fault().
44 */
45 pages = alloc_pages(GFP_KERNEL, order);
46 if (!pages)
47 panic("Unable to allocate VDSO storage pages");
48
49 /* The pages are mapped one-by-one into userspace and each one needs to be refcounted. */
50 split_page(pages, order);
51
52 /* Move the data already written by other subsystems to the new pages */
53 memcpy(page_address(pages), vdso_initdata, VDSO_NR_PAGES * PAGE_SIZE);
54
55 if (IS_ENABLED(CONFIG_GENERIC_GETTIMEOFDAY))
56 vdso_k_time_data = page_address(pages + VDSO_TIME_PAGE_OFFSET);
57
58 if (IS_ENABLED(CONFIG_VDSO_GETRANDOM))
59 vdso_k_rng_data = page_address(pages + VDSO_RNG_PAGE_OFFSET);
60
61 if (IS_ENABLED(CONFIG_ARCH_HAS_VDSO_ARCH_DATA))
62 vdso_k_arch_data = page_address(pages + VDSO_ARCH_PAGES_START);
63}
64
65static vm_fault_t vvar_fault(const struct vm_special_mapping *sm,
66 struct vm_area_struct *vma, struct vm_fault *vmf)
67{
68 struct page *page, *timens_page;
69
70 timens_page = find_timens_vvar_page(vma);
71
72 switch (vmf->pgoff) {
73 case VDSO_TIME_PAGE_OFFSET:
74 if (!IS_ENABLED(CONFIG_GENERIC_GETTIMEOFDAY))
75 return VM_FAULT_SIGBUS;
76 page = virt_to_page(vdso_k_time_data);
77 if (timens_page) {
78 /*
79 * Fault in VVAR page too, since it will be accessed
80 * to get clock data anyway.
81 */
82 unsigned long addr;
83 vm_fault_t err;
84
85 addr = vmf->address + VDSO_TIMENS_PAGE_OFFSET * PAGE_SIZE;
86 err = vmf_insert_page(vma, addr, page);
87 if (unlikely(err & VM_FAULT_ERROR))
88 return err;
89 page = timens_page;
90 }
91 break;
92 case VDSO_TIMENS_PAGE_OFFSET:
93 /*
94 * If a task belongs to a time namespace then a namespace
95 * specific VVAR is mapped with the VVAR_DATA_PAGE_OFFSET and
96 * the real VVAR page is mapped with the VVAR_TIMENS_PAGE_OFFSET
97 * offset.
98 * See also the comment near timens_setup_vdso_data().
99 */
100 if (!IS_ENABLED(CONFIG_TIME_NS) || !timens_page)
101 return VM_FAULT_SIGBUS;
102 page = virt_to_page(vdso_k_time_data);
103 break;
104 case VDSO_RNG_PAGE_OFFSET:
105 if (!IS_ENABLED(CONFIG_VDSO_GETRANDOM))
106 return VM_FAULT_SIGBUS;
107 page = virt_to_page(vdso_k_rng_data);
108 break;
109 case VDSO_ARCH_PAGES_START ... VDSO_ARCH_PAGES_END:
110 if (!IS_ENABLED(CONFIG_ARCH_HAS_VDSO_ARCH_DATA))
111 return VM_FAULT_SIGBUS;
112 page = virt_to_page(vdso_k_arch_data) + vmf->pgoff - VDSO_ARCH_PAGES_START;
113 break;
114 default:
115 return VM_FAULT_SIGBUS;
116 }
117
118 get_page(page);
119 vmf->page = page;
120 return 0;
121}
122
123const struct vm_special_mapping vdso_vvar_mapping = {
124 .name = "[vvar]",
125 .fault = vvar_fault,
126};
127
128struct vm_area_struct *vdso_install_vvar_mapping(struct mm_struct *mm, unsigned long addr)
129{
130 return _install_special_mapping(mm, addr, VDSO_NR_PAGES * PAGE_SIZE,
131 VM_READ | VM_MAYREAD | VM_IO | VM_DONTDUMP |
132 VM_MIXEDMAP | VM_SEALED_SYSMAP,
133 &vdso_vvar_mapping);
134}