Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1// SPDX-License-Identifier: GPL-2.0
2
3#define _GNU_SOURCE
4#include <stdio.h>
5#include <fcntl.h>
6#include <string.h>
7#include <sys/mman.h>
8#include <errno.h>
9#include <malloc.h>
10#include "vm_util.h"
11#include "kselftest.h"
12#include <linux/types.h>
13#include <linux/memfd.h>
14#include <linux/userfaultfd.h>
15#include <linux/fs.h>
16#include <sys/ioctl.h>
17#include <sys/stat.h>
18#include <math.h>
19#include <asm/unistd.h>
20#include <pthread.h>
21#include <sys/resource.h>
22#include <assert.h>
23#include <sys/ipc.h>
24#include <sys/shm.h>
25
26#define PAGEMAP_BITS_ALL (PAGE_IS_WPALLOWED | PAGE_IS_WRITTEN | \
27 PAGE_IS_FILE | PAGE_IS_PRESENT | \
28 PAGE_IS_SWAPPED | PAGE_IS_PFNZERO | \
29 PAGE_IS_HUGE)
30#define PAGEMAP_NON_WRITTEN_BITS (PAGE_IS_WPALLOWED | PAGE_IS_FILE | \
31 PAGE_IS_PRESENT | PAGE_IS_SWAPPED | \
32 PAGE_IS_PFNZERO | PAGE_IS_HUGE)
33
34#define TEST_ITERATIONS 100
35#define PAGEMAP "/proc/self/pagemap"
36int pagemap_fd;
37int uffd;
38size_t page_size;
39size_t hpage_size;
40const char *progname;
41
42#define LEN(region) ((region.end - region.start)/page_size)
43
44static long pagemap_ioctl(void *start, int len, void *vec, int vec_len, int flag,
45 int max_pages, long required_mask, long anyof_mask, long excluded_mask,
46 long return_mask)
47{
48 struct pm_scan_arg arg;
49
50 arg.start = (uintptr_t)start;
51 arg.end = (uintptr_t)(start + len);
52 arg.vec = (uintptr_t)vec;
53 arg.vec_len = vec_len;
54 arg.flags = flag;
55 arg.size = sizeof(struct pm_scan_arg);
56 arg.max_pages = max_pages;
57 arg.category_mask = required_mask;
58 arg.category_anyof_mask = anyof_mask;
59 arg.category_inverted = excluded_mask;
60 arg.return_mask = return_mask;
61
62 return ioctl(pagemap_fd, PAGEMAP_SCAN, &arg);
63}
64
65static long pagemap_ioc(void *start, int len, void *vec, int vec_len, int flag,
66 int max_pages, long required_mask, long anyof_mask, long excluded_mask,
67 long return_mask, long *walk_end)
68{
69 struct pm_scan_arg arg;
70 int ret;
71
72 arg.start = (uintptr_t)start;
73 arg.end = (uintptr_t)(start + len);
74 arg.vec = (uintptr_t)vec;
75 arg.vec_len = vec_len;
76 arg.flags = flag;
77 arg.size = sizeof(struct pm_scan_arg);
78 arg.max_pages = max_pages;
79 arg.category_mask = required_mask;
80 arg.category_anyof_mask = anyof_mask;
81 arg.category_inverted = excluded_mask;
82 arg.return_mask = return_mask;
83
84 ret = ioctl(pagemap_fd, PAGEMAP_SCAN, &arg);
85
86 if (walk_end)
87 *walk_end = arg.walk_end;
88
89 return ret;
90}
91
92
93int init_uffd(void)
94{
95 struct uffdio_api uffdio_api;
96
97 uffd = syscall(__NR_userfaultfd, O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY);
98 if (uffd == -1)
99 return uffd;
100
101 uffdio_api.api = UFFD_API;
102 uffdio_api.features = UFFD_FEATURE_WP_UNPOPULATED | UFFD_FEATURE_WP_ASYNC |
103 UFFD_FEATURE_WP_HUGETLBFS_SHMEM;
104 if (ioctl(uffd, UFFDIO_API, &uffdio_api))
105 return -1;
106
107 if (!(uffdio_api.api & UFFDIO_REGISTER_MODE_WP) ||
108 !(uffdio_api.features & UFFD_FEATURE_WP_UNPOPULATED) ||
109 !(uffdio_api.features & UFFD_FEATURE_WP_ASYNC) ||
110 !(uffdio_api.features & UFFD_FEATURE_WP_HUGETLBFS_SHMEM))
111 return -1;
112
113 return 0;
114}
115
116int wp_init(void *addr, long size)
117{
118 struct uffdio_register uffdio_register;
119 struct uffdio_writeprotect wp;
120
121 uffdio_register.range.start = (unsigned long)addr;
122 uffdio_register.range.len = size;
123 uffdio_register.mode = UFFDIO_REGISTER_MODE_WP;
124 if (ioctl(uffd, UFFDIO_REGISTER, &uffdio_register))
125 ksft_exit_fail_msg("ioctl(UFFDIO_REGISTER) %d %s\n", errno, strerror(errno));
126
127 if (!(uffdio_register.ioctls & UFFDIO_WRITEPROTECT))
128 ksft_exit_fail_msg("ioctl set is incorrect\n");
129
130 wp.range.start = (unsigned long)addr;
131 wp.range.len = size;
132 wp.mode = UFFDIO_WRITEPROTECT_MODE_WP;
133
134 if (ioctl(uffd, UFFDIO_WRITEPROTECT, &wp))
135 ksft_exit_fail_msg("ioctl(UFFDIO_WRITEPROTECT)\n");
136
137 return 0;
138}
139
140int wp_free(void *addr, long size)
141{
142 struct uffdio_register uffdio_register;
143
144 uffdio_register.range.start = (unsigned long)addr;
145 uffdio_register.range.len = size;
146 uffdio_register.mode = UFFDIO_REGISTER_MODE_WP;
147 if (ioctl(uffd, UFFDIO_UNREGISTER, &uffdio_register.range))
148 ksft_exit_fail_msg("ioctl unregister failure\n");
149 return 0;
150}
151
152int wp_addr_range(void *addr, int size)
153{
154 if (pagemap_ioctl(addr, size, NULL, 0,
155 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC,
156 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN) < 0)
157 ksft_exit_fail_msg("error %d %d %s\n", 1, errno, strerror(errno));
158
159 return 0;
160}
161
162void *gethugetlb_mem(int size, int *shmid)
163{
164 char *mem;
165
166 if (shmid) {
167 *shmid = shmget(2, size, SHM_HUGETLB | IPC_CREAT | SHM_R | SHM_W);
168 if (*shmid < 0)
169 return NULL;
170
171 mem = shmat(*shmid, 0, 0);
172 if (mem == (char *)-1) {
173 shmctl(*shmid, IPC_RMID, NULL);
174 ksft_exit_fail_msg("Shared memory attach failure\n");
175 }
176 } else {
177 mem = mmap(NULL, size, PROT_READ | PROT_WRITE,
178 MAP_ANONYMOUS | MAP_HUGETLB | MAP_PRIVATE, -1, 0);
179 if (mem == MAP_FAILED)
180 return NULL;
181 }
182
183 return mem;
184}
185
186int userfaultfd_tests(void)
187{
188 long mem_size, vec_size, written, num_pages = 16;
189 char *mem, *vec;
190
191 mem_size = num_pages * page_size;
192 mem = mmap(NULL, mem_size, PROT_NONE, MAP_PRIVATE | MAP_ANON, -1, 0);
193 if (mem == MAP_FAILED)
194 ksft_exit_fail_msg("error nomem\n");
195
196 wp_init(mem, mem_size);
197
198 /* Change protection of pages differently */
199 mprotect(mem, mem_size/8, PROT_READ|PROT_WRITE);
200 mprotect(mem + 1 * mem_size/8, mem_size/8, PROT_READ);
201 mprotect(mem + 2 * mem_size/8, mem_size/8, PROT_READ|PROT_WRITE);
202 mprotect(mem + 3 * mem_size/8, mem_size/8, PROT_READ);
203 mprotect(mem + 4 * mem_size/8, mem_size/8, PROT_READ|PROT_WRITE);
204 mprotect(mem + 5 * mem_size/8, mem_size/8, PROT_NONE);
205 mprotect(mem + 6 * mem_size/8, mem_size/8, PROT_READ|PROT_WRITE);
206 mprotect(mem + 7 * mem_size/8, mem_size/8, PROT_READ);
207
208 wp_addr_range(mem + (mem_size/16), mem_size - 2 * (mem_size/8));
209 wp_addr_range(mem, mem_size);
210
211 vec_size = mem_size/page_size;
212 vec = calloc(vec_size, sizeof(struct page_region));
213
214 written = pagemap_ioctl(mem, mem_size, vec, 1, PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC,
215 vec_size - 2, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN);
216 if (written < 0)
217 ksft_exit_fail_msg("error %ld %d %s\n", written, errno, strerror(errno));
218
219 ksft_test_result(written == 0, "%s all new pages must not be written (dirty)\n", __func__);
220
221 wp_free(mem, mem_size);
222 munmap(mem, mem_size);
223 free(vec);
224 return 0;
225}
226
227int get_reads(struct page_region *vec, int vec_size)
228{
229 int i, sum = 0;
230
231 for (i = 0; i < vec_size; i++)
232 sum += LEN(vec[i]);
233
234 return sum;
235}
236
237int sanity_tests_sd(void)
238{
239 unsigned long long mem_size, vec_size, i, total_pages = 0;
240 long ret, ret2, ret3;
241 int num_pages = 1000;
242 int total_writes, total_reads, reads, count;
243 struct page_region *vec, *vec2;
244 char *mem, *m[2];
245 long walk_end;
246
247 vec_size = num_pages/2;
248 mem_size = num_pages * page_size;
249
250 vec = calloc(vec_size, sizeof(struct page_region));
251 if (!vec)
252 ksft_exit_fail_msg("error nomem\n");
253
254 vec2 = calloc(vec_size, sizeof(struct page_region));
255 if (!vec2)
256 ksft_exit_fail_msg("error nomem\n");
257
258 mem = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0);
259 if (mem == MAP_FAILED)
260 ksft_exit_fail_msg("error nomem\n");
261
262 wp_init(mem, mem_size);
263 wp_addr_range(mem, mem_size);
264
265 /* 1. wrong operation */
266 ksft_test_result(pagemap_ioctl(mem, 0, vec, vec_size, 0,
267 0, PAGEMAP_BITS_ALL, 0, 0, PAGEMAP_BITS_ALL) == 0,
268 "%s Zero range size is valid\n", __func__);
269
270 ksft_test_result(pagemap_ioctl(mem, mem_size, NULL, vec_size, 0,
271 0, PAGEMAP_BITS_ALL, 0, 0, PAGEMAP_BITS_ALL) < 0,
272 "%s output buffer must be specified with size\n", __func__);
273
274 ksft_test_result(pagemap_ioctl(mem, mem_size, vec, 0, 0,
275 0, PAGEMAP_BITS_ALL, 0, 0, PAGEMAP_BITS_ALL) == 0,
276 "%s output buffer can be 0\n", __func__);
277
278 ksft_test_result(pagemap_ioctl(mem, mem_size, 0, 0, 0,
279 0, PAGEMAP_BITS_ALL, 0, 0, PAGEMAP_BITS_ALL) == 0,
280 "%s output buffer can be 0\n", __func__);
281
282 ksft_test_result(pagemap_ioctl(mem, mem_size, vec, vec_size, -1,
283 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN) < 0,
284 "%s wrong flag specified\n", __func__);
285
286 ksft_test_result(pagemap_ioctl(mem, mem_size, vec, vec_size,
287 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC | 0xFF,
288 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN) < 0,
289 "%s flag has extra bits specified\n", __func__);
290
291 ksft_test_result(pagemap_ioctl(mem, mem_size, vec, vec_size, 0,
292 0, 0, 0, 0, PAGE_IS_WRITTEN) >= 0,
293 "%s no selection mask is specified\n", __func__);
294
295 ksft_test_result(pagemap_ioctl(mem, mem_size, vec, vec_size, 0,
296 0, PAGE_IS_WRITTEN, PAGE_IS_WRITTEN, 0, 0) == 0,
297 "%s no return mask is specified\n", __func__);
298
299 ksft_test_result(pagemap_ioctl(mem, mem_size, vec, vec_size, 0,
300 0, PAGE_IS_WRITTEN, 0, 0, 0x1000) < 0,
301 "%s wrong return mask specified\n", __func__);
302
303 ksft_test_result(pagemap_ioctl(mem, mem_size, vec, vec_size,
304 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC,
305 0, 0xFFF, PAGE_IS_WRITTEN, 0, PAGE_IS_WRITTEN) < 0,
306 "%s mixture of correct and wrong flag\n", __func__);
307
308 ksft_test_result(pagemap_ioctl(mem, mem_size, vec, vec_size,
309 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC,
310 0, 0, 0, PAGEMAP_BITS_ALL, PAGE_IS_WRITTEN) >= 0,
311 "%s PAGEMAP_BITS_ALL can be specified with PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC\n",
312 __func__);
313
314 /* 2. Clear area with larger vec size */
315 ret = pagemap_ioctl(mem, mem_size, vec, vec_size,
316 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 0,
317 PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN);
318 ksft_test_result(ret >= 0, "%s Clear area with larger vec size\n", __func__);
319
320 /* 3. Repeated pattern of written and non-written pages */
321 for (i = 0; i < mem_size; i += 2 * page_size)
322 mem[i]++;
323
324 ret = pagemap_ioctl(mem, mem_size, vec, vec_size, 0, 0, PAGE_IS_WRITTEN, 0,
325 0, PAGE_IS_WRITTEN);
326 if (ret < 0)
327 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno));
328
329 ksft_test_result((unsigned long long)ret == mem_size/(page_size * 2),
330 "%s Repeated pattern of written and non-written pages\n", __func__);
331
332 /* 4. Repeated pattern of written and non-written pages in parts */
333 ret = pagemap_ioctl(mem, mem_size, vec, vec_size,
334 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC,
335 num_pages/2 - 2, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN);
336 if (ret < 0)
337 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno));
338
339 ret2 = pagemap_ioctl(mem, mem_size, vec, 2, 0, 0, PAGE_IS_WRITTEN, 0, 0,
340 PAGE_IS_WRITTEN);
341 if (ret2 < 0)
342 ksft_exit_fail_msg("error %ld %d %s\n", ret2, errno, strerror(errno));
343
344 ret3 = pagemap_ioctl(mem, mem_size, vec, vec_size,
345 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC,
346 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN);
347 if (ret3 < 0)
348 ksft_exit_fail_msg("error %ld %d %s\n", ret3, errno, strerror(errno));
349
350 ksft_test_result((ret + ret3) == num_pages/2 && ret2 == 2,
351 "%s Repeated pattern of written and non-written pages in parts %ld %ld %ld\n",
352 __func__, ret, ret3, ret2);
353
354 /* 5. Repeated pattern of written and non-written pages max_pages */
355 for (i = 0; i < mem_size; i += 2 * page_size)
356 mem[i]++;
357 mem[(mem_size/page_size - 1) * page_size]++;
358
359 ret = pagemap_ioctl(mem, mem_size, vec, vec_size,
360 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC,
361 num_pages/2, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN);
362 if (ret < 0)
363 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno));
364
365 ret2 = pagemap_ioctl(mem, mem_size, vec, vec_size,
366 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC,
367 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN);
368 if (ret2 < 0)
369 ksft_exit_fail_msg("error %ld %d %s\n", ret2, errno, strerror(errno));
370
371 ksft_test_result(ret == num_pages/2 && ret2 == 1,
372 "%s Repeated pattern of written and non-written pages max_pages\n",
373 __func__);
374
375 /* 6. only get 2 dirty pages and clear them as well */
376 vec_size = mem_size/page_size;
377 memset(mem, -1, mem_size);
378
379 /* get and clear second and third pages */
380 ret = pagemap_ioctl(mem + page_size, 2 * page_size, vec, 1,
381 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC,
382 2, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN);
383 if (ret < 0)
384 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno));
385
386 ret2 = pagemap_ioctl(mem, mem_size, vec2, vec_size, 0, 0,
387 PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN);
388 if (ret2 < 0)
389 ksft_exit_fail_msg("error %ld %d %s\n", ret2, errno, strerror(errno));
390
391 ksft_test_result(ret == 1 && LEN(vec[0]) == 2 &&
392 vec[0].start == (uintptr_t)(mem + page_size) &&
393 ret2 == 2 && LEN(vec2[0]) == 1 && vec2[0].start == (uintptr_t)mem &&
394 LEN(vec2[1]) == vec_size - 3 &&
395 vec2[1].start == (uintptr_t)(mem + 3 * page_size),
396 "%s only get 2 written pages and clear them as well\n", __func__);
397
398 wp_free(mem, mem_size);
399 munmap(mem, mem_size);
400
401 /* 7. Two regions */
402 m[0] = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0);
403 if (m[0] == MAP_FAILED)
404 ksft_exit_fail_msg("error nomem\n");
405 m[1] = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0);
406 if (m[1] == MAP_FAILED)
407 ksft_exit_fail_msg("error nomem\n");
408
409 wp_init(m[0], mem_size);
410 wp_init(m[1], mem_size);
411 wp_addr_range(m[0], mem_size);
412 wp_addr_range(m[1], mem_size);
413
414 memset(m[0], 'a', mem_size);
415 memset(m[1], 'b', mem_size);
416
417 wp_addr_range(m[0], mem_size);
418
419 ret = pagemap_ioctl(m[1], mem_size, vec, 1, 0, 0, PAGE_IS_WRITTEN, 0, 0,
420 PAGE_IS_WRITTEN);
421 if (ret < 0)
422 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno));
423
424 ksft_test_result(ret == 1 && LEN(vec[0]) == mem_size/page_size,
425 "%s Two regions\n", __func__);
426
427 wp_free(m[0], mem_size);
428 wp_free(m[1], mem_size);
429 munmap(m[0], mem_size);
430 munmap(m[1], mem_size);
431
432 free(vec);
433 free(vec2);
434
435 /* 8. Smaller vec */
436 mem_size = 1050 * page_size;
437 vec_size = mem_size/(page_size*2);
438
439 vec = calloc(vec_size, sizeof(struct page_region));
440 if (!vec)
441 ksft_exit_fail_msg("error nomem\n");
442
443 mem = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0);
444 if (mem == MAP_FAILED)
445 ksft_exit_fail_msg("error nomem\n");
446
447 wp_init(mem, mem_size);
448 wp_addr_range(mem, mem_size);
449
450 ret = pagemap_ioctl(mem, mem_size, vec, vec_size,
451 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 0,
452 PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN);
453 if (ret < 0)
454 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno));
455
456 for (i = 0; i < mem_size/page_size; i += 2)
457 mem[i * page_size]++;
458
459 ret = pagemap_ioctl(mem, mem_size, vec, vec_size,
460 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC,
461 mem_size/(page_size*5), PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN);
462 if (ret < 0)
463 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno));
464
465 total_pages += ret;
466
467 ret = pagemap_ioctl(mem, mem_size, vec, vec_size,
468 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC,
469 mem_size/(page_size*5), PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN);
470 if (ret < 0)
471 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno));
472
473 total_pages += ret;
474
475 ret = pagemap_ioctl(mem, mem_size, vec, vec_size,
476 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC,
477 mem_size/(page_size*5), PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN);
478 if (ret < 0)
479 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno));
480
481 total_pages += ret;
482
483 ksft_test_result(total_pages == mem_size/(page_size*2), "%s Smaller max_pages\n", __func__);
484
485 free(vec);
486 wp_free(mem, mem_size);
487 munmap(mem, mem_size);
488 total_pages = 0;
489
490 /* 9. Smaller vec */
491 mem_size = 10000 * page_size;
492 vec_size = 50;
493
494 vec = calloc(vec_size, sizeof(struct page_region));
495 if (!vec)
496 ksft_exit_fail_msg("error nomem\n");
497
498 mem = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0);
499 if (mem == MAP_FAILED)
500 ksft_exit_fail_msg("error nomem\n");
501
502 wp_init(mem, mem_size);
503 wp_addr_range(mem, mem_size);
504
505 for (count = 0; count < TEST_ITERATIONS; count++) {
506 total_writes = total_reads = 0;
507 walk_end = (long)mem;
508
509 for (i = 0; i < mem_size; i += page_size) {
510 if (rand() % 2) {
511 mem[i]++;
512 total_writes++;
513 }
514 }
515
516 while (total_reads < total_writes) {
517 ret = pagemap_ioc((void *)walk_end, mem_size-(walk_end - (long)mem), vec,
518 vec_size, PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC,
519 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end);
520 if (ret < 0)
521 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno));
522
523 if ((unsigned long)ret > vec_size)
524 break;
525
526 reads = get_reads(vec, ret);
527 total_reads += reads;
528 }
529
530 if (total_reads != total_writes)
531 break;
532 }
533
534 ksft_test_result(count == TEST_ITERATIONS, "Smaller vec\n");
535
536 free(vec);
537 wp_free(mem, mem_size);
538 munmap(mem, mem_size);
539
540 /* 10. Walk_end tester */
541 vec_size = 1000;
542 mem_size = vec_size * page_size;
543
544 vec = calloc(vec_size, sizeof(struct page_region));
545 if (!vec)
546 ksft_exit_fail_msg("error nomem\n");
547
548 mem = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0);
549 if (mem == MAP_FAILED)
550 ksft_exit_fail_msg("error nomem\n");
551
552 wp_init(mem, mem_size);
553 wp_addr_range(mem, mem_size);
554
555 memset(mem, 0, mem_size);
556
557 ret = pagemap_ioc(mem, 0, vec, vec_size, 0,
558 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end);
559 if (ret < 0)
560 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno));
561 ksft_test_result(ret == 0 && walk_end == (long)mem,
562 "Walk_end: Same start and end address\n");
563
564 ret = pagemap_ioc(mem, 0, vec, vec_size, PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC,
565 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end);
566 if (ret < 0)
567 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno));
568 ksft_test_result(ret == 0 && walk_end == (long)mem,
569 "Walk_end: Same start and end with WP\n");
570
571 ret = pagemap_ioc(mem, 0, vec, 0, PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC,
572 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end);
573 if (ret < 0)
574 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno));
575 ksft_test_result(ret == 0 && walk_end == (long)mem,
576 "Walk_end: Same start and end with 0 output buffer\n");
577
578 ret = pagemap_ioc(mem, mem_size, vec, vec_size, 0,
579 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end);
580 if (ret < 0)
581 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno));
582 ksft_test_result(ret == 1 && walk_end == (long)(mem + mem_size),
583 "Walk_end: Big vec\n");
584
585 ret = pagemap_ioc(mem, mem_size, vec, 1, 0,
586 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end);
587 if (ret < 0)
588 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno));
589 ksft_test_result(ret == 1 && walk_end == (long)(mem + mem_size),
590 "Walk_end: vec of minimum length\n");
591
592 ret = pagemap_ioc(mem, mem_size, vec, 1, 0,
593 vec_size, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end);
594 if (ret < 0)
595 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno));
596 ksft_test_result(ret == 1 && walk_end == (long)(mem + mem_size),
597 "Walk_end: Max pages specified\n");
598
599 ret = pagemap_ioc(mem, mem_size, vec, vec_size, 0,
600 vec_size/2, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end);
601 if (ret < 0)
602 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno));
603 ksft_test_result(ret == 1 && walk_end == (long)(mem + mem_size/2),
604 "Walk_end: Half max pages\n");
605
606 ret = pagemap_ioc(mem, mem_size, vec, vec_size, 0,
607 1, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end);
608 if (ret < 0)
609 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno));
610 ksft_test_result(ret == 1 && walk_end == (long)(mem + page_size),
611 "Walk_end: 1 max page\n");
612
613 ret = pagemap_ioc(mem, mem_size, vec, vec_size, 0,
614 -1, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end);
615 if (ret < 0)
616 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno));
617 ksft_test_result(ret == 1 && walk_end == (long)(mem + mem_size),
618 "Walk_end: max pages\n");
619
620 wp_addr_range(mem, mem_size);
621 for (i = 0; i < mem_size; i += 2 * page_size)
622 mem[i]++;
623
624 ret = pagemap_ioc(mem, mem_size, vec, vec_size, 0,
625 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end);
626 if (ret < 0)
627 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno));
628 ksft_test_result((unsigned long)ret == vec_size/2 && walk_end == (long)(mem + mem_size),
629 "Walk_end sparse: Big vec\n");
630
631 ret = pagemap_ioc(mem, mem_size, vec, 1, 0,
632 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end);
633 if (ret < 0)
634 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno));
635 ksft_test_result(ret == 1 && walk_end == (long)(mem + page_size * 2),
636 "Walk_end sparse: vec of minimum length\n");
637
638 ret = pagemap_ioc(mem, mem_size, vec, 1, 0,
639 vec_size, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end);
640 if (ret < 0)
641 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno));
642 ksft_test_result(ret == 1 && walk_end == (long)(mem + page_size * 2),
643 "Walk_end sparse: Max pages specified\n");
644
645 ret = pagemap_ioc(mem, mem_size, vec, vec_size/2, 0,
646 vec_size, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end);
647 if (ret < 0)
648 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno));
649 ksft_test_result((unsigned long)ret == vec_size/2 && walk_end == (long)(mem + mem_size),
650 "Walk_end sparse: Max pages specified\n");
651
652 ret = pagemap_ioc(mem, mem_size, vec, vec_size, 0,
653 vec_size, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end);
654 if (ret < 0)
655 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno));
656 ksft_test_result((unsigned long)ret == vec_size/2 && walk_end == (long)(mem + mem_size),
657 "Walk_end sparse: Max pages specified\n");
658
659 ret = pagemap_ioc(mem, mem_size, vec, vec_size, 0,
660 vec_size/2, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end);
661 if (ret < 0)
662 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno));
663 ksft_test_result((unsigned long)ret == vec_size/2 && walk_end == (long)(mem + mem_size),
664 "Walk_endsparse : Half max pages\n");
665
666 ret = pagemap_ioc(mem, mem_size, vec, vec_size, 0,
667 1, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN, &walk_end);
668 if (ret < 0)
669 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno));
670 ksft_test_result(ret == 1 && walk_end == (long)(mem + page_size * 2),
671 "Walk_end: 1 max page\n");
672
673 free(vec);
674 wp_free(mem, mem_size);
675 munmap(mem, mem_size);
676
677 return 0;
678}
679
680int base_tests(char *prefix, char *mem, unsigned long long mem_size, int skip)
681{
682 unsigned long long vec_size;
683 int written;
684 struct page_region *vec, *vec2;
685
686 if (skip) {
687 ksft_test_result_skip("%s all new pages must not be written (dirty)\n", prefix);
688 ksft_test_result_skip("%s all pages must be written (dirty)\n", prefix);
689 ksft_test_result_skip("%s all pages dirty other than first and the last one\n",
690 prefix);
691 ksft_test_result_skip("%s PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC\n", prefix);
692 ksft_test_result_skip("%s only middle page dirty\n", prefix);
693 ksft_test_result_skip("%s only two middle pages dirty\n", prefix);
694 return 0;
695 }
696
697 vec_size = mem_size/page_size;
698 vec = calloc(vec_size, sizeof(struct page_region));
699 vec2 = calloc(vec_size, sizeof(struct page_region));
700
701 /* 1. all new pages must be not be written (dirty) */
702 written = pagemap_ioctl(mem, mem_size, vec, 1, PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC,
703 vec_size - 2, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN);
704 if (written < 0)
705 ksft_exit_fail_msg("error %d %d %s\n", written, errno, strerror(errno));
706
707 ksft_test_result(written == 0, "%s all new pages must not be written (dirty)\n", prefix);
708
709 /* 2. all pages must be written */
710 memset(mem, -1, mem_size);
711
712 written = pagemap_ioctl(mem, mem_size, vec, 1, 0, 0, PAGE_IS_WRITTEN, 0, 0,
713 PAGE_IS_WRITTEN);
714 if (written < 0)
715 ksft_exit_fail_msg("error %d %d %s\n", written, errno, strerror(errno));
716
717 ksft_test_result(written == 1 && LEN(vec[0]) == mem_size/page_size,
718 "%s all pages must be written (dirty)\n", prefix);
719
720 /* 3. all pages dirty other than first and the last one */
721 written = pagemap_ioctl(mem, mem_size, vec, 1, PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC,
722 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN);
723 if (written < 0)
724 ksft_exit_fail_msg("error %d %d %s\n", written, errno, strerror(errno));
725
726 memset(mem + page_size, 0, mem_size - (2 * page_size));
727
728 written = pagemap_ioctl(mem, mem_size, vec, 1, PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC,
729 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN);
730 if (written < 0)
731 ksft_exit_fail_msg("error %d %d %s\n", written, errno, strerror(errno));
732
733 ksft_test_result(written == 1 && LEN(vec[0]) >= vec_size - 2 && LEN(vec[0]) <= vec_size,
734 "%s all pages dirty other than first and the last one\n", prefix);
735
736 written = pagemap_ioctl(mem, mem_size, vec, 1, 0, 0,
737 PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN);
738 if (written < 0)
739 ksft_exit_fail_msg("error %d %d %s\n", written, errno, strerror(errno));
740
741 ksft_test_result(written == 0,
742 "%s PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC\n", prefix);
743
744 /* 4. only middle page dirty */
745 written = pagemap_ioctl(mem, mem_size, vec, 1, PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC,
746 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN);
747 if (written < 0)
748 ksft_exit_fail_msg("error %d %d %s\n", written, errno, strerror(errno));
749
750 mem[vec_size/2 * page_size]++;
751
752 written = pagemap_ioctl(mem, mem_size, vec, vec_size, 0, 0, PAGE_IS_WRITTEN,
753 0, 0, PAGE_IS_WRITTEN);
754 if (written < 0)
755 ksft_exit_fail_msg("error %d %d %s\n", written, errno, strerror(errno));
756
757 ksft_test_result(written == 1 && LEN(vec[0]) >= 1,
758 "%s only middle page dirty\n", prefix);
759
760 /* 5. only two middle pages dirty and walk over only middle pages */
761 written = pagemap_ioctl(mem, mem_size, vec, 1, PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC,
762 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN | PAGE_IS_HUGE);
763 if (written < 0)
764 ksft_exit_fail_msg("error %d %d %s\n", written, errno, strerror(errno));
765
766 mem[vec_size/2 * page_size]++;
767 mem[(vec_size/2 + 1) * page_size]++;
768
769 written = pagemap_ioctl(&mem[vec_size/2 * page_size], 2 * page_size, vec, 1, 0,
770 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN | PAGE_IS_HUGE);
771 if (written < 0)
772 ksft_exit_fail_msg("error %d %d %s\n", written, errno, strerror(errno));
773
774 ksft_test_result(written == 1 && vec[0].start == (uintptr_t)(&mem[vec_size/2 * page_size])
775 && LEN(vec[0]) == 2,
776 "%s only two middle pages dirty\n", prefix);
777
778 free(vec);
779 free(vec2);
780 return 0;
781}
782
783void *gethugepage(int map_size)
784{
785 int ret;
786 char *map;
787
788 map = memalign(hpage_size, map_size);
789 if (!map)
790 ksft_exit_fail_msg("memalign failed %d %s\n", errno, strerror(errno));
791
792 ret = madvise(map, map_size, MADV_HUGEPAGE);
793 if (ret)
794 return NULL;
795
796 memset(map, 0, map_size);
797
798 return map;
799}
800
801int hpage_unit_tests(void)
802{
803 char *map;
804 int ret, ret2;
805 size_t num_pages = 10;
806 unsigned long long map_size = hpage_size * num_pages;
807 unsigned long long vec_size = map_size/page_size;
808 struct page_region *vec, *vec2;
809
810 vec = calloc(vec_size, sizeof(struct page_region));
811 vec2 = calloc(vec_size, sizeof(struct page_region));
812 if (!vec || !vec2)
813 ksft_exit_fail_msg("malloc failed\n");
814
815 map = gethugepage(map_size);
816 if (map) {
817 wp_init(map, map_size);
818 wp_addr_range(map, map_size);
819
820 /* 1. all new huge page must not be written (dirty) */
821 ret = pagemap_ioctl(map, map_size, vec, vec_size,
822 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 0,
823 PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN);
824 if (ret < 0)
825 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno));
826
827 ksft_test_result(ret == 0, "%s all new huge page must not be written (dirty)\n",
828 __func__);
829
830 /* 2. all the huge page must not be written */
831 ret = pagemap_ioctl(map, map_size, vec, vec_size, 0, 0,
832 PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN);
833 if (ret < 0)
834 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno));
835
836 ksft_test_result(ret == 0, "%s all the huge page must not be written\n", __func__);
837
838 /* 3. all the huge page must be written and clear dirty as well */
839 memset(map, -1, map_size);
840 ret = pagemap_ioctl(map, map_size, vec, vec_size,
841 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC,
842 0, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN);
843 if (ret < 0)
844 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno));
845
846 ksft_test_result(ret == 1 && vec[0].start == (uintptr_t)map &&
847 LEN(vec[0]) == vec_size && vec[0].categories == PAGE_IS_WRITTEN,
848 "%s all the huge page must be written and clear\n", __func__);
849
850 /* 4. only middle page written */
851 wp_free(map, map_size);
852 free(map);
853 map = gethugepage(map_size);
854 wp_init(map, map_size);
855 wp_addr_range(map, map_size);
856 map[vec_size/2 * page_size]++;
857
858 ret = pagemap_ioctl(map, map_size, vec, vec_size, 0, 0,
859 PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN);
860 if (ret < 0)
861 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno));
862
863 ksft_test_result(ret == 1 && LEN(vec[0]) > 0,
864 "%s only middle page written\n", __func__);
865
866 wp_free(map, map_size);
867 free(map);
868 } else {
869 ksft_test_result_skip("%s all new huge page must be written\n", __func__);
870 ksft_test_result_skip("%s all the huge page must not be written\n", __func__);
871 ksft_test_result_skip("%s all the huge page must be written and clear\n", __func__);
872 ksft_test_result_skip("%s only middle page written\n", __func__);
873 }
874
875 /* 5. clear first half of huge page */
876 map = gethugepage(map_size);
877 if (map) {
878 wp_init(map, map_size);
879 wp_addr_range(map, map_size);
880
881 memset(map, 0, map_size);
882
883 wp_addr_range(map, map_size/2);
884
885 ret = pagemap_ioctl(map, map_size, vec, vec_size, 0, 0,
886 PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN);
887 if (ret < 0)
888 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno));
889
890 ksft_test_result(ret == 1 && LEN(vec[0]) == vec_size/2 &&
891 vec[0].start == (uintptr_t)(map + map_size/2),
892 "%s clear first half of huge page\n", __func__);
893 wp_free(map, map_size);
894 free(map);
895 } else {
896 ksft_test_result_skip("%s clear first half of huge page\n", __func__);
897 }
898
899 /* 6. clear first half of huge page with limited buffer */
900 map = gethugepage(map_size);
901 if (map) {
902 wp_init(map, map_size);
903 wp_addr_range(map, map_size);
904
905 memset(map, 0, map_size);
906
907 ret = pagemap_ioctl(map, map_size, vec, vec_size,
908 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC,
909 vec_size/2, PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN);
910 if (ret < 0)
911 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno));
912
913 ret = pagemap_ioctl(map, map_size, vec, vec_size, 0, 0,
914 PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN);
915 if (ret < 0)
916 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno));
917
918 ksft_test_result(ret == 1 && LEN(vec[0]) == vec_size/2 &&
919 vec[0].start == (uintptr_t)(map + map_size/2),
920 "%s clear first half of huge page with limited buffer\n",
921 __func__);
922 wp_free(map, map_size);
923 free(map);
924 } else {
925 ksft_test_result_skip("%s clear first half of huge page with limited buffer\n",
926 __func__);
927 }
928
929 /* 7. clear second half of huge page */
930 map = gethugepage(map_size);
931 if (map) {
932 wp_init(map, map_size);
933 wp_addr_range(map, map_size);
934
935 memset(map, -1, map_size);
936
937 ret = pagemap_ioctl(map + map_size/2, map_size/2, vec, vec_size,
938 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, vec_size/2,
939 PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN);
940 if (ret < 0)
941 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno));
942
943 ret = pagemap_ioctl(map, map_size, vec, vec_size, 0, 0,
944 PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN);
945 if (ret < 0)
946 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno));
947
948 ksft_test_result(ret == 1 && LEN(vec[0]) == vec_size/2,
949 "%s clear second half huge page\n", __func__);
950 wp_free(map, map_size);
951 free(map);
952 } else {
953 ksft_test_result_skip("%s clear second half huge page\n", __func__);
954 }
955
956 /* 8. get half huge page */
957 map = gethugepage(map_size);
958 if (map) {
959 wp_init(map, map_size);
960 wp_addr_range(map, map_size);
961
962 memset(map, -1, map_size);
963 usleep(100);
964
965 ret = pagemap_ioctl(map, map_size, vec, 1,
966 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC,
967 hpage_size/(2*page_size), PAGE_IS_WRITTEN, 0, 0,
968 PAGE_IS_WRITTEN);
969 if (ret < 0)
970 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno));
971
972 ksft_test_result(ret == 1 && LEN(vec[0]) == hpage_size/(2*page_size),
973 "%s get half huge page\n", __func__);
974
975 ret2 = pagemap_ioctl(map, map_size, vec, vec_size, 0, 0,
976 PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN);
977 if (ret2 < 0)
978 ksft_exit_fail_msg("error %d %d %s\n", ret2, errno, strerror(errno));
979
980 ksft_test_result(ret2 == 1 && LEN(vec[0]) == (map_size - hpage_size/2)/page_size,
981 "%s get half huge page\n", __func__);
982
983 wp_free(map, map_size);
984 free(map);
985 } else {
986 ksft_test_result_skip("%s get half huge page\n", __func__);
987 ksft_test_result_skip("%s get half huge page\n", __func__);
988 }
989
990 free(vec);
991 free(vec2);
992 return 0;
993}
994
995int unmapped_region_tests(void)
996{
997 void *start = (void *)0x10000000;
998 int written, len = 0x00040000;
999 long vec_size = len / page_size;
1000 struct page_region *vec = calloc(vec_size, sizeof(struct page_region));
1001
1002 /* 1. Get written pages */
1003 written = pagemap_ioctl(start, len, vec, vec_size, 0, 0,
1004 PAGEMAP_NON_WRITTEN_BITS, 0, 0, PAGEMAP_NON_WRITTEN_BITS);
1005 if (written < 0)
1006 ksft_exit_fail_msg("error %d %d %s\n", written, errno, strerror(errno));
1007
1008 ksft_test_result(written >= 0, "%s Get status of pages\n", __func__);
1009
1010 free(vec);
1011 return 0;
1012}
1013
1014static void test_simple(void)
1015{
1016 int i;
1017 char *map;
1018 struct page_region vec;
1019
1020 map = aligned_alloc(page_size, page_size);
1021 if (!map)
1022 ksft_exit_fail_msg("aligned_alloc failed\n");
1023
1024 wp_init(map, page_size);
1025 wp_addr_range(map, page_size);
1026
1027 for (i = 0 ; i < TEST_ITERATIONS; i++) {
1028 if (pagemap_ioctl(map, page_size, &vec, 1, 0, 0,
1029 PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN) == 1) {
1030 ksft_print_msg("written bit was 1, but should be 0 (i=%d)\n", i);
1031 break;
1032 }
1033
1034 wp_addr_range(map, page_size);
1035 /* Write something to the page to get the written bit enabled on the page */
1036 map[0]++;
1037
1038 if (pagemap_ioctl(map, page_size, &vec, 1, 0, 0,
1039 PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN) == 0) {
1040 ksft_print_msg("written bit was 0, but should be 1 (i=%d)\n", i);
1041 break;
1042 }
1043
1044 wp_addr_range(map, page_size);
1045 }
1046 wp_free(map, page_size);
1047 free(map);
1048
1049 ksft_test_result(i == TEST_ITERATIONS, "Test %s\n", __func__);
1050}
1051
1052int sanity_tests(void)
1053{
1054 unsigned long long mem_size, vec_size;
1055 long ret, fd, i, buf_size, nr_pages;
1056 struct page_region *vec;
1057 char *mem, *fmem;
1058 struct stat sbuf;
1059
1060 /* 1. wrong operation */
1061 mem_size = 10 * page_size;
1062 vec_size = mem_size / page_size;
1063
1064 vec = calloc(vec_size, sizeof(struct page_region));
1065 mem = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0);
1066 if (mem == MAP_FAILED || vec == MAP_FAILED)
1067 ksft_exit_fail_msg("error nomem\n");
1068
1069 wp_init(mem, mem_size);
1070 wp_addr_range(mem, mem_size);
1071
1072 ksft_test_result(pagemap_ioctl(mem, mem_size, vec, vec_size,
1073 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC,
1074 0, PAGEMAP_BITS_ALL, 0, 0, PAGEMAP_BITS_ALL) >= 0,
1075 "%s WP op can be specified with !PAGE_IS_WRITTEN\n", __func__);
1076 ksft_test_result(pagemap_ioctl(mem, mem_size, vec, vec_size, 0, 0,
1077 PAGEMAP_BITS_ALL, 0, 0, PAGEMAP_BITS_ALL) >= 0,
1078 "%s required_mask specified\n", __func__);
1079 ksft_test_result(pagemap_ioctl(mem, mem_size, vec, vec_size, 0, 0,
1080 0, PAGEMAP_BITS_ALL, 0, PAGEMAP_BITS_ALL) >= 0,
1081 "%s anyof_mask specified\n", __func__);
1082 ksft_test_result(pagemap_ioctl(mem, mem_size, vec, vec_size, 0, 0,
1083 0, 0, PAGEMAP_BITS_ALL, PAGEMAP_BITS_ALL) >= 0,
1084 "%s excluded_mask specified\n", __func__);
1085 ksft_test_result(pagemap_ioctl(mem, mem_size, vec, vec_size, 0, 0,
1086 PAGEMAP_BITS_ALL, PAGEMAP_BITS_ALL, 0,
1087 PAGEMAP_BITS_ALL) >= 0,
1088 "%s required_mask and anyof_mask specified\n", __func__);
1089 wp_free(mem, mem_size);
1090 munmap(mem, mem_size);
1091
1092 /* 2. Get sd and present pages with anyof_mask */
1093 mem = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0);
1094 if (mem == MAP_FAILED)
1095 ksft_exit_fail_msg("error nomem\n");
1096 wp_init(mem, mem_size);
1097 wp_addr_range(mem, mem_size);
1098
1099 memset(mem, 0, mem_size);
1100
1101 ret = pagemap_ioctl(mem, mem_size, vec, vec_size, 0, 0,
1102 0, PAGEMAP_BITS_ALL, 0, PAGEMAP_BITS_ALL);
1103 ksft_test_result(ret >= 0 && vec[0].start == (uintptr_t)mem && LEN(vec[0]) == vec_size &&
1104 (vec[0].categories & (PAGE_IS_WRITTEN | PAGE_IS_PRESENT)) ==
1105 (PAGE_IS_WRITTEN | PAGE_IS_PRESENT),
1106 "%s Get sd and present pages with anyof_mask\n", __func__);
1107
1108 /* 3. Get sd and present pages with required_mask */
1109 ret = pagemap_ioctl(mem, mem_size, vec, vec_size, 0, 0,
1110 PAGEMAP_BITS_ALL, 0, 0, PAGEMAP_BITS_ALL);
1111 ksft_test_result(ret >= 0 && vec[0].start == (uintptr_t)mem && LEN(vec[0]) == vec_size &&
1112 (vec[0].categories & (PAGE_IS_WRITTEN | PAGE_IS_PRESENT)) ==
1113 (PAGE_IS_WRITTEN | PAGE_IS_PRESENT),
1114 "%s Get all the pages with required_mask\n", __func__);
1115
1116 /* 4. Get sd and present pages with required_mask and anyof_mask */
1117 ret = pagemap_ioctl(mem, mem_size, vec, vec_size, 0, 0,
1118 PAGE_IS_WRITTEN, PAGE_IS_PRESENT, 0, PAGEMAP_BITS_ALL);
1119 ksft_test_result(ret >= 0 && vec[0].start == (uintptr_t)mem && LEN(vec[0]) == vec_size &&
1120 (vec[0].categories & (PAGE_IS_WRITTEN | PAGE_IS_PRESENT)) ==
1121 (PAGE_IS_WRITTEN | PAGE_IS_PRESENT),
1122 "%s Get sd and present pages with required_mask and anyof_mask\n",
1123 __func__);
1124
1125 /* 5. Don't get sd pages */
1126 ret = pagemap_ioctl(mem, mem_size, vec, vec_size, 0, 0,
1127 PAGE_IS_WRITTEN, 0, PAGE_IS_WRITTEN, PAGEMAP_BITS_ALL);
1128 ksft_test_result(ret == 0, "%s Don't get sd pages\n", __func__);
1129
1130 /* 6. Don't get present pages */
1131 ret = pagemap_ioctl(mem, mem_size, vec, vec_size, 0, 0,
1132 PAGE_IS_PRESENT, 0, PAGE_IS_PRESENT, PAGEMAP_BITS_ALL);
1133 ksft_test_result(ret == 0, "%s Don't get present pages\n", __func__);
1134
1135 wp_free(mem, mem_size);
1136 munmap(mem, mem_size);
1137
1138 /* 8. Find written present pages with return mask */
1139 mem = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0);
1140 if (mem == MAP_FAILED)
1141 ksft_exit_fail_msg("error nomem\n");
1142 wp_init(mem, mem_size);
1143 wp_addr_range(mem, mem_size);
1144
1145 memset(mem, 0, mem_size);
1146
1147 ret = pagemap_ioctl(mem, mem_size, vec, vec_size,
1148 PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC, 0,
1149 0, PAGEMAP_BITS_ALL, 0, PAGE_IS_WRITTEN);
1150 ksft_test_result(ret >= 0 && vec[0].start == (uintptr_t)mem && LEN(vec[0]) == vec_size &&
1151 vec[0].categories == PAGE_IS_WRITTEN,
1152 "%s Find written present pages with return mask\n", __func__);
1153 wp_free(mem, mem_size);
1154 munmap(mem, mem_size);
1155
1156 /* 9. Memory mapped file */
1157 fd = open(progname, O_RDONLY);
1158 if (fd < 0)
1159 ksft_exit_fail_msg("%s Memory mapped file\n", __func__);
1160
1161 ret = stat(progname, &sbuf);
1162 if (ret < 0)
1163 ksft_exit_fail_msg("error %ld %d %s\n", ret, errno, strerror(errno));
1164
1165 fmem = mmap(NULL, sbuf.st_size, PROT_READ, MAP_PRIVATE, fd, 0);
1166 if (fmem == MAP_FAILED)
1167 ksft_exit_fail_msg("error nomem %d %s\n", errno, strerror(errno));
1168
1169 nr_pages = (sbuf.st_size + page_size - 1) / page_size;
1170 force_read_pages(fmem, nr_pages, page_size);
1171
1172 ret = pagemap_ioctl(fmem, sbuf.st_size, vec, vec_size, 0, 0,
1173 0, PAGEMAP_NON_WRITTEN_BITS, 0, PAGEMAP_NON_WRITTEN_BITS);
1174
1175 ksft_test_result(ret >= 0 && vec[0].start == (uintptr_t)fmem &&
1176 LEN(vec[0]) == nr_pages &&
1177 (vec[0].categories & PAGE_IS_FILE),
1178 "%s Memory mapped file\n", __func__);
1179
1180 munmap(fmem, sbuf.st_size);
1181 close(fd);
1182
1183 /* 10. Create and read/write to a memory mapped file */
1184 buf_size = page_size * 10;
1185
1186 fd = open(__FILE__".tmp2", O_RDWR | O_CREAT, 0666);
1187 if (fd < 0)
1188 ksft_exit_fail_msg("Read/write to memory: %s\n",
1189 strerror(errno));
1190
1191 for (i = 0; i < buf_size; i++)
1192 if (write(fd, "c", 1) < 0)
1193 ksft_exit_fail_msg("Create and read/write to a memory mapped file\n");
1194
1195 fmem = mmap(NULL, buf_size, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0);
1196 if (fmem == MAP_FAILED)
1197 ksft_exit_fail_msg("error nomem %d %s\n", errno, strerror(errno));
1198
1199 wp_init(fmem, buf_size);
1200 wp_addr_range(fmem, buf_size);
1201
1202 for (i = 0; i < buf_size; i++)
1203 fmem[i] = 'z';
1204
1205 msync(fmem, buf_size, MS_SYNC);
1206
1207 ret = pagemap_ioctl(fmem, buf_size, vec, vec_size, 0, 0,
1208 PAGE_IS_WRITTEN, PAGE_IS_PRESENT | PAGE_IS_SWAPPED | PAGE_IS_FILE, 0,
1209 PAGEMAP_BITS_ALL);
1210
1211 ksft_test_result(ret >= 0 && vec[0].start == (uintptr_t)fmem &&
1212 LEN(vec[0]) == (buf_size/page_size) &&
1213 (vec[0].categories & PAGE_IS_WRITTEN),
1214 "%s Read/write to memory\n", __func__);
1215
1216 wp_free(fmem, buf_size);
1217 munmap(fmem, buf_size);
1218 close(fd);
1219
1220 free(vec);
1221 return 0;
1222}
1223
1224int mprotect_tests(void)
1225{
1226 int ret;
1227 char *mem, *mem2;
1228 struct page_region vec;
1229 int pagemap_fd = open("/proc/self/pagemap", O_RDONLY);
1230
1231 if (pagemap_fd < 0) {
1232 fprintf(stderr, "open() failed\n");
1233 exit(1);
1234 }
1235
1236 /* 1. Map two pages */
1237 mem = mmap(0, 2 * page_size, PROT_READ|PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0);
1238 if (mem == MAP_FAILED)
1239 ksft_exit_fail_msg("error nomem\n");
1240 wp_init(mem, 2 * page_size);
1241 wp_addr_range(mem, 2 * page_size);
1242
1243 /* Populate both pages. */
1244 memset(mem, 1, 2 * page_size);
1245
1246 ret = pagemap_ioctl(mem, 2 * page_size, &vec, 1, 0, 0, PAGE_IS_WRITTEN,
1247 0, 0, PAGE_IS_WRITTEN);
1248 if (ret < 0)
1249 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno));
1250
1251 ksft_test_result(ret == 1 && LEN(vec) == 2, "%s Both pages written\n", __func__);
1252
1253 /* 2. Start tracking */
1254 wp_addr_range(mem, 2 * page_size);
1255
1256 ksft_test_result(pagemap_ioctl(mem, 2 * page_size, &vec, 1, 0, 0,
1257 PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN) == 0,
1258 "%s Both pages are not written (dirty)\n", __func__);
1259
1260 /* 3. Remap the second page */
1261 mem2 = mmap(mem + page_size, page_size, PROT_READ|PROT_WRITE,
1262 MAP_PRIVATE|MAP_ANON|MAP_FIXED, -1, 0);
1263 if (mem2 == MAP_FAILED)
1264 ksft_exit_fail_msg("error nomem\n");
1265 wp_init(mem2, page_size);
1266 wp_addr_range(mem2, page_size);
1267
1268 /* Protect + unprotect. */
1269 mprotect(mem, page_size, PROT_NONE);
1270 mprotect(mem, 2 * page_size, PROT_READ);
1271 mprotect(mem, 2 * page_size, PROT_READ|PROT_WRITE);
1272
1273 /* Modify both pages. */
1274 memset(mem, 2, 2 * page_size);
1275
1276 /* Protect + unprotect. */
1277 mprotect(mem, page_size, PROT_NONE);
1278 mprotect(mem, page_size, PROT_READ);
1279 mprotect(mem, page_size, PROT_READ|PROT_WRITE);
1280
1281 ret = pagemap_ioctl(mem, 2 * page_size, &vec, 1, 0, 0, PAGE_IS_WRITTEN,
1282 0, 0, PAGE_IS_WRITTEN);
1283 if (ret < 0)
1284 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno));
1285
1286 ksft_test_result(ret == 1 && LEN(vec) == 2,
1287 "%s Both pages written after remap and mprotect\n", __func__);
1288
1289 /* 4. Clear and make the pages written */
1290 wp_addr_range(mem, 2 * page_size);
1291
1292 memset(mem, 'A', 2 * page_size);
1293
1294 ret = pagemap_ioctl(mem, 2 * page_size, &vec, 1, 0, 0, PAGE_IS_WRITTEN,
1295 0, 0, PAGE_IS_WRITTEN);
1296 if (ret < 0)
1297 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno));
1298
1299 ksft_test_result(ret == 1 && LEN(vec) == 2,
1300 "%s Clear and make the pages written\n", __func__);
1301
1302 wp_free(mem, 2 * page_size);
1303 munmap(mem, 2 * page_size);
1304 return 0;
1305}
1306
1307/* transact test */
1308static const unsigned int nthreads = 6, pages_per_thread = 32, access_per_thread = 8;
1309static pthread_barrier_t start_barrier, end_barrier;
1310static unsigned int extra_thread_faults;
1311static unsigned int iter_count = 1000;
1312static volatile int finish;
1313
1314static ssize_t get_dirty_pages_reset(char *mem, unsigned int count,
1315 int reset, int page_size)
1316{
1317 struct pm_scan_arg arg = {0};
1318 struct page_region rgns[256];
1319 unsigned long long i, j;
1320 long ret;
1321 int cnt;
1322
1323 arg.size = sizeof(struct pm_scan_arg);
1324 arg.start = (uintptr_t)mem;
1325 arg.max_pages = count;
1326 arg.end = (uintptr_t)(mem + count * page_size);
1327 arg.vec = (uintptr_t)rgns;
1328 arg.vec_len = sizeof(rgns) / sizeof(*rgns);
1329 if (reset)
1330 arg.flags |= PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC;
1331 arg.category_mask = PAGE_IS_WRITTEN;
1332 arg.return_mask = PAGE_IS_WRITTEN;
1333
1334 ret = ioctl(pagemap_fd, PAGEMAP_SCAN, &arg);
1335 if (ret < 0)
1336 ksft_exit_fail_msg("ioctl failed\n");
1337
1338 cnt = 0;
1339 for (i = 0; i < (unsigned long)ret; ++i) {
1340 if (rgns[i].categories != PAGE_IS_WRITTEN)
1341 ksft_exit_fail_msg("wrong flags\n");
1342
1343 for (j = 0; j < LEN(rgns[i]); ++j)
1344 cnt++;
1345 }
1346
1347 return cnt;
1348}
1349
1350void *thread_proc(void *mem)
1351{
1352 int *m = mem;
1353 long curr_faults, faults;
1354 struct rusage r;
1355 unsigned int i;
1356 int ret;
1357
1358 if (getrusage(RUSAGE_THREAD, &r))
1359 ksft_exit_fail_msg("getrusage\n");
1360
1361 curr_faults = r.ru_minflt;
1362
1363 while (!finish) {
1364 ret = pthread_barrier_wait(&start_barrier);
1365 if (ret && ret != PTHREAD_BARRIER_SERIAL_THREAD)
1366 ksft_exit_fail_msg("pthread_barrier_wait\n");
1367
1368 for (i = 0; i < access_per_thread; ++i)
1369 __atomic_add_fetch(m + i * (0x1000 / sizeof(*m)), 1, __ATOMIC_SEQ_CST);
1370
1371 ret = pthread_barrier_wait(&end_barrier);
1372 if (ret && ret != PTHREAD_BARRIER_SERIAL_THREAD)
1373 ksft_exit_fail_msg("pthread_barrier_wait\n");
1374
1375 if (getrusage(RUSAGE_THREAD, &r))
1376 ksft_exit_fail_msg("getrusage\n");
1377
1378 faults = r.ru_minflt - curr_faults;
1379 if (faults < access_per_thread)
1380 ksft_exit_fail_msg("faults < access_per_thread");
1381
1382 __atomic_add_fetch(&extra_thread_faults, faults - access_per_thread,
1383 __ATOMIC_SEQ_CST);
1384 curr_faults = r.ru_minflt;
1385 }
1386
1387 return NULL;
1388}
1389
1390static void transact_test(int page_size)
1391{
1392 unsigned int i, count, extra_pages;
1393 unsigned int c;
1394 pthread_t th;
1395 char *mem;
1396 int ret;
1397
1398 if (pthread_barrier_init(&start_barrier, NULL, nthreads + 1))
1399 ksft_exit_fail_msg("pthread_barrier_init\n");
1400
1401 if (pthread_barrier_init(&end_barrier, NULL, nthreads + 1))
1402 ksft_exit_fail_msg("pthread_barrier_init\n");
1403
1404 mem = mmap(NULL, 0x1000 * nthreads * pages_per_thread, PROT_READ | PROT_WRITE,
1405 MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
1406 if (mem == MAP_FAILED)
1407 ksft_exit_fail_msg("Error mmap %s.\n", strerror(errno));
1408
1409 wp_init(mem, 0x1000 * nthreads * pages_per_thread);
1410 wp_addr_range(mem, 0x1000 * nthreads * pages_per_thread);
1411
1412 memset(mem, 0, 0x1000 * nthreads * pages_per_thread);
1413
1414 count = get_dirty_pages_reset(mem, nthreads * pages_per_thread, 1, page_size);
1415 ksft_test_result(count > 0, "%s count %u\n", __func__, count);
1416 count = get_dirty_pages_reset(mem, nthreads * pages_per_thread, 1, page_size);
1417 ksft_test_result(count == 0, "%s count %u\n", __func__, count);
1418
1419 finish = 0;
1420 for (i = 0; i < nthreads; ++i)
1421 pthread_create(&th, NULL, thread_proc, mem + 0x1000 * i * pages_per_thread);
1422
1423 extra_pages = 0;
1424 for (i = 0; i < iter_count; ++i) {
1425 count = 0;
1426
1427 ret = pthread_barrier_wait(&start_barrier);
1428 if (ret && ret != PTHREAD_BARRIER_SERIAL_THREAD)
1429 ksft_exit_fail_msg("pthread_barrier_wait\n");
1430
1431 count = get_dirty_pages_reset(mem, nthreads * pages_per_thread, 1,
1432 page_size);
1433
1434 ret = pthread_barrier_wait(&end_barrier);
1435 if (ret && ret != PTHREAD_BARRIER_SERIAL_THREAD)
1436 ksft_exit_fail_msg("pthread_barrier_wait\n");
1437
1438 if (count > nthreads * access_per_thread)
1439 ksft_exit_fail_msg("Too big count %u expected %u, iter %u\n",
1440 count, nthreads * access_per_thread, i);
1441
1442 c = get_dirty_pages_reset(mem, nthreads * pages_per_thread, 1, page_size);
1443 count += c;
1444
1445 if (c > nthreads * access_per_thread) {
1446 ksft_test_result_fail(" %s count > nthreads\n", __func__);
1447 return;
1448 }
1449
1450 if (count != nthreads * access_per_thread) {
1451 /*
1452 * The purpose of the test is to make sure that no page updates are lost
1453 * when the page updates and read-resetting soft dirty flags are performed
1454 * in parallel. However, it is possible that the application will get the
1455 * soft dirty flags twice on the two consecutive read-resets. This seems
1456 * unavoidable as soft dirty flag is handled in software through page faults
1457 * in kernel. While the updating the flags is supposed to be synchronized
1458 * between page fault handling and read-reset, it is possible that
1459 * read-reset happens after page fault PTE update but before the application
1460 * re-executes write instruction. So read-reset gets the flag, clears write
1461 * access and application gets page fault again for the same write.
1462 */
1463 if (count < nthreads * access_per_thread) {
1464 ksft_test_result_fail("Lost update, iter %u, %u vs %u.\n", i, count,
1465 nthreads * access_per_thread);
1466 return;
1467 }
1468
1469 extra_pages += count - nthreads * access_per_thread;
1470 }
1471 }
1472
1473 pthread_barrier_wait(&start_barrier);
1474 finish = 1;
1475 pthread_barrier_wait(&end_barrier);
1476
1477 ksft_test_result_pass("%s Extra pages %u (%.1lf%%), extra thread faults %u.\n", __func__,
1478 extra_pages,
1479 100.0 * extra_pages / (iter_count * nthreads * access_per_thread),
1480 extra_thread_faults);
1481}
1482
1483void zeropfn_tests(void)
1484{
1485 unsigned long long mem_size;
1486 struct page_region vec;
1487 int i, ret;
1488 char *mmap_mem, *mem;
1489
1490 /* Test with normal memory */
1491 mem_size = 10 * page_size;
1492 mem = mmap(NULL, mem_size, PROT_READ, MAP_PRIVATE | MAP_ANON, -1, 0);
1493 if (mem == MAP_FAILED)
1494 ksft_exit_fail_msg("error nomem\n");
1495
1496 /* Touch each page to ensure it's mapped */
1497 for (i = 0; i < mem_size; i += page_size)
1498 (void)((volatile char *)mem)[i];
1499
1500 ret = pagemap_ioctl(mem, mem_size, &vec, 1, 0,
1501 (mem_size / page_size), PAGE_IS_PFNZERO, 0, 0, PAGE_IS_PFNZERO);
1502 if (ret < 0)
1503 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno));
1504
1505 ksft_test_result(ret == 1 && LEN(vec) == (mem_size / page_size),
1506 "%s all pages must have PFNZERO set\n", __func__);
1507
1508 munmap(mem, mem_size);
1509
1510 /* Test with huge page if user_zero_page is set to 1 */
1511 if (!detect_huge_zeropage()) {
1512 ksft_test_result_skip("%s use_zero_page not supported or set to 1\n", __func__);
1513 return;
1514 }
1515
1516 mem_size = 2 * hpage_size;
1517 mmap_mem = mmap(NULL, mem_size, PROT_READ | PROT_WRITE,
1518 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
1519 if (mmap_mem == MAP_FAILED)
1520 ksft_exit_fail_msg("error nomem\n");
1521
1522 /* We need a THP-aligned memory area. */
1523 mem = (char *)(((uintptr_t)mmap_mem + hpage_size) & ~(hpage_size - 1));
1524
1525 ret = madvise(mem, hpage_size, MADV_HUGEPAGE);
1526 if (!ret) {
1527 FORCE_READ(*mem);
1528
1529 ret = pagemap_ioctl(mem, hpage_size, &vec, 1, 0,
1530 0, PAGE_IS_PFNZERO, 0, 0, PAGE_IS_PFNZERO);
1531 if (ret < 0)
1532 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno));
1533
1534 ksft_test_result(ret == 1 && LEN(vec) == (hpage_size / page_size),
1535 "%s all huge pages must have PFNZERO set\n", __func__);
1536 } else {
1537 ksft_test_result_skip("%s huge page not supported\n", __func__);
1538 }
1539
1540 munmap(mmap_mem, mem_size);
1541}
1542
1543int main(int __attribute__((unused)) argc, char *argv[])
1544{
1545 int shmid, buf_size, fd, i, ret;
1546 unsigned long long mem_size;
1547 char *mem, *map, *fmem;
1548 struct stat sbuf;
1549
1550 progname = argv[0];
1551
1552 ksft_print_header();
1553
1554 if (init_uffd())
1555 ksft_exit_skip("Failed to initialize userfaultfd\n");
1556
1557 ksft_set_plan(117);
1558
1559 page_size = getpagesize();
1560 hpage_size = read_pmd_pagesize();
1561
1562 pagemap_fd = open(PAGEMAP, O_RDONLY);
1563 if (pagemap_fd < 0)
1564 ksft_exit_fail_msg("Failed to open " PAGEMAP "\n");
1565
1566 /* 1. Sanity testing */
1567 sanity_tests_sd();
1568
1569 /* 2. Normal page testing */
1570 mem_size = 10 * page_size;
1571 mem = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0);
1572 if (mem == MAP_FAILED)
1573 ksft_exit_fail_msg("error nomem\n");
1574 wp_init(mem, mem_size);
1575 wp_addr_range(mem, mem_size);
1576
1577 base_tests("Page testing:", mem, mem_size, 0);
1578
1579 wp_free(mem, mem_size);
1580 munmap(mem, mem_size);
1581
1582 /* 3. Large page testing */
1583 mem_size = 512 * 10 * page_size;
1584 mem = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0);
1585 if (mem == MAP_FAILED)
1586 ksft_exit_fail_msg("error nomem\n");
1587 wp_init(mem, mem_size);
1588 wp_addr_range(mem, mem_size);
1589
1590 base_tests("Large Page testing:", mem, mem_size, 0);
1591
1592 wp_free(mem, mem_size);
1593 munmap(mem, mem_size);
1594
1595 /* 4. Huge page testing */
1596 map = gethugepage(hpage_size);
1597 if (map) {
1598 wp_init(map, hpage_size);
1599 wp_addr_range(map, hpage_size);
1600 base_tests("Huge page testing:", map, hpage_size, 0);
1601 wp_free(map, hpage_size);
1602 free(map);
1603 } else {
1604 base_tests("Huge page testing:", NULL, 0, 1);
1605 }
1606
1607 /* 5. SHM Hugetlb page testing */
1608 mem_size = 2*1024*1024;
1609 mem = gethugetlb_mem(mem_size, &shmid);
1610 if (mem) {
1611 wp_init(mem, mem_size);
1612 wp_addr_range(mem, mem_size);
1613
1614 base_tests("Hugetlb shmem testing:", mem, mem_size, 0);
1615
1616 wp_free(mem, mem_size);
1617 shmctl(shmid, IPC_RMID, NULL);
1618 } else {
1619 base_tests("Hugetlb shmem testing:", NULL, 0, 1);
1620 }
1621
1622 /* 6. Hugetlb page testing */
1623 mem = gethugetlb_mem(mem_size, NULL);
1624 if (mem) {
1625 wp_init(mem, mem_size);
1626 wp_addr_range(mem, mem_size);
1627
1628 base_tests("Hugetlb mem testing:", mem, mem_size, 0);
1629
1630 wp_free(mem, mem_size);
1631 } else {
1632 base_tests("Hugetlb mem testing:", NULL, 0, 1);
1633 }
1634
1635 /* 7. File Hugetlb testing */
1636 mem_size = 2*1024*1024;
1637 fd = memfd_create("uffd-test", MFD_HUGETLB | MFD_NOEXEC_SEAL);
1638 if (fd < 0)
1639 ksft_exit_fail_msg("uffd-test creation failed %d %s\n", errno, strerror(errno));
1640 mem = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
1641 if (mem != MAP_FAILED) {
1642 wp_init(mem, mem_size);
1643 wp_addr_range(mem, mem_size);
1644
1645 base_tests("Hugetlb shmem testing:", mem, mem_size, 0);
1646
1647 wp_free(mem, mem_size);
1648 shmctl(shmid, IPC_RMID, NULL);
1649 } else {
1650 base_tests("Hugetlb shmem testing:", NULL, 0, 1);
1651 }
1652 close(fd);
1653
1654 /* 8. File memory testing */
1655 buf_size = page_size * 10;
1656
1657 fd = open(__FILE__".tmp0", O_RDWR | O_CREAT, 0777);
1658 if (fd < 0)
1659 ksft_exit_fail_msg("Create and read/write to a memory mapped file: %s\n",
1660 strerror(errno));
1661
1662 for (i = 0; i < buf_size; i++)
1663 if (write(fd, "c", 1) < 0)
1664 ksft_exit_fail_msg("Create and read/write to a memory mapped file\n");
1665
1666 ret = stat(__FILE__".tmp0", &sbuf);
1667 if (ret < 0)
1668 ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno));
1669
1670 fmem = mmap(NULL, sbuf.st_size, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0);
1671 if (fmem == MAP_FAILED)
1672 ksft_exit_fail_msg("error nomem %d %s\n", errno, strerror(errno));
1673
1674 wp_init(fmem, sbuf.st_size);
1675 wp_addr_range(fmem, sbuf.st_size);
1676
1677 base_tests("File memory testing:", fmem, sbuf.st_size, 0);
1678
1679 wp_free(fmem, sbuf.st_size);
1680 munmap(fmem, sbuf.st_size);
1681 close(fd);
1682
1683 /* 9. File memory testing */
1684 buf_size = page_size * 10;
1685
1686 fd = memfd_create(__FILE__".tmp00", MFD_NOEXEC_SEAL);
1687 if (fd < 0)
1688 ksft_exit_fail_msg("Create and read/write to a memory mapped file: %s\n",
1689 strerror(errno));
1690
1691 if (ftruncate(fd, buf_size))
1692 ksft_exit_fail_msg("Error ftruncate\n");
1693
1694 for (i = 0; i < buf_size; i++)
1695 if (write(fd, "c", 1) < 0)
1696 ksft_exit_fail_msg("Create and read/write to a memory mapped file\n");
1697
1698 fmem = mmap(NULL, buf_size, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0);
1699 if (fmem == MAP_FAILED)
1700 ksft_exit_fail_msg("error nomem %d %s\n", errno, strerror(errno));
1701
1702 wp_init(fmem, buf_size);
1703 wp_addr_range(fmem, buf_size);
1704
1705 base_tests("File anonymous memory testing:", fmem, buf_size, 0);
1706
1707 wp_free(fmem, buf_size);
1708 munmap(fmem, buf_size);
1709 close(fd);
1710
1711 /* 10. Huge page tests */
1712 hpage_unit_tests();
1713
1714 /* 11. Iterative test */
1715 test_simple();
1716
1717 /* 12. Mprotect test */
1718 mprotect_tests();
1719
1720 /* 13. Transact test */
1721 transact_test(page_size);
1722
1723 /* 14. Sanity testing */
1724 sanity_tests();
1725
1726 /*15. Unmapped address test */
1727 unmapped_region_tests();
1728
1729 /* 16. Userfaultfd tests */
1730 userfaultfd_tests();
1731
1732 /* 17. ZEROPFN tests */
1733 zeropfn_tests();
1734
1735 close(pagemap_fd);
1736 ksft_finished();
1737}