selftests/mm/uffd: refactor non-composite global vars into struct

+146 -125

tools/testing/selftests/mm/uffd-common.c

··· 7 7 8 8 #include "uffd-common.h" 9 9 10 - #define BASE_PMD_ADDR ((void *)(1UL << 30)) 11 - 12 - volatile bool test_uffdio_copy_eexist = true; 13 - unsigned long nr_parallel, nr_pages, nr_pages_per_cpu, page_size; 14 - char *area_src, *area_src_alias, *area_dst, *area_dst_alias, *area_remap; 15 - int uffd = -1, uffd_flags, finished, *pipefd, test_type; 16 - bool map_shared; 17 - bool test_uffdio_wp = true; 18 - unsigned long long *count_verify; 19 10 uffd_test_ops_t *uffd_test_ops; 20 11 uffd_test_case_ops_t *uffd_test_case_ops; 21 - atomic_bool ready_for_fork; 12 + 13 + #define BASE_PMD_ADDR ((void *)(1UL << 30)) 14 + 15 + /* pthread_mutex_t starts at page offset 0 */ 16 + pthread_mutex_t *area_mutex(char *area, unsigned long nr, uffd_global_test_opts_t *gopts) 17 + { 18 + return (pthread_mutex_t *) (area + nr * gopts->page_size); 19 + } 20 + 21 + /* 22 + * count is placed in the page after pthread_mutex_t naturally aligned 23 + * to avoid non alignment faults on non-x86 archs. 24 + */ 25 + volatile unsigned long long *area_count(char *area, unsigned long nr, 26 + uffd_global_test_opts_t *gopts) 27 + { 28 + return (volatile unsigned long long *) 29 + ((unsigned long)(area + nr * gopts->page_size + 30 + sizeof(pthread_mutex_t) + sizeof(unsigned long long) - 1) & 31 + ~(unsigned long)(sizeof(unsigned long long) - 1)); 32 + } 22 33 23 34 static int uffd_mem_fd_create(off_t mem_size, bool hugetlb) 24 35 { ··· 51 40 return mem_fd; 52 41 } 53 42 54 - static void anon_release_pages(char *rel_area) 43 + static void anon_release_pages(uffd_global_test_opts_t *gopts, char *rel_area) 55 44 { 56 - if (madvise(rel_area, nr_pages * page_size, MADV_DONTNEED)) 45 + if (madvise(rel_area, gopts->nr_pages * gopts->page_size, MADV_DONTNEED)) 57 46 err("madvise(MADV_DONTNEED) failed"); 58 47 } 59 48 60 - static int anon_allocate_area(void **alloc_area, bool is_src) 49 + static int anon_allocate_area(uffd_global_test_opts_t *gopts, void **alloc_area, bool is_src) 61 50 { 62 - *alloc_area = mmap(NULL, nr_pages * page_size, PROT_READ | PROT_WRITE, 51 + *alloc_area = mmap(NULL, gopts->nr_pages * gopts->page_size, PROT_READ | PROT_WRITE, 63 52 MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); 64 53 if (*alloc_area == MAP_FAILED) { 65 54 *alloc_area = NULL; ··· 68 57 return 0; 69 58 } 70 59 71 - static void noop_alias_mapping(__u64 *start, size_t len, unsigned long offset) 60 + static void noop_alias_mapping(uffd_global_test_opts_t *gopts, __u64 *start, 61 + size_t len, unsigned long offset) 72 62 { 73 63 } 74 64 75 - static void hugetlb_release_pages(char *rel_area) 65 + static void hugetlb_release_pages(uffd_global_test_opts_t *gopts, char *rel_area) 76 66 { 77 - if (!map_shared) { 78 - if (madvise(rel_area, nr_pages * page_size, MADV_DONTNEED)) 67 + if (!gopts->map_shared) { 68 + if (madvise(rel_area, gopts->nr_pages * gopts->page_size, MADV_DONTNEED)) 79 69 err("madvise(MADV_DONTNEED) failed"); 80 70 } else { 81 - if (madvise(rel_area, nr_pages * page_size, MADV_REMOVE)) 71 + if (madvise(rel_area, gopts->nr_pages * gopts->page_size, MADV_REMOVE)) 82 72 err("madvise(MADV_REMOVE) failed"); 83 73 } 84 74 } 85 75 86 - static int hugetlb_allocate_area(void **alloc_area, bool is_src) 76 + static int hugetlb_allocate_area(uffd_global_test_opts_t *gopts, void **alloc_area, bool is_src) 87 77 { 88 - off_t size = nr_pages * page_size; 78 + off_t size = gopts->nr_pages * gopts->page_size; 89 79 off_t offset = is_src ? 0 : size; 90 80 void *area_alias = NULL; 91 81 char **alloc_area_alias; 92 82 int mem_fd = uffd_mem_fd_create(size * 2, true); 93 83 94 84 *alloc_area = mmap(NULL, size, PROT_READ | PROT_WRITE, 95 - (map_shared ? MAP_SHARED : MAP_PRIVATE) | 85 + (gopts->map_shared ? MAP_SHARED : MAP_PRIVATE) | 96 86 (is_src ? 0 : MAP_NORESERVE), 97 87 mem_fd, offset); 98 88 if (*alloc_area == MAP_FAILED) { ··· 101 89 return -errno; 102 90 } 103 91 104 - if (map_shared) { 92 + if (gopts->map_shared) { 105 93 area_alias = mmap(NULL, size, PROT_READ | PROT_WRITE, 106 94 MAP_SHARED, mem_fd, offset); 107 95 if (area_alias == MAP_FAILED) ··· 109 97 } 110 98 111 99 if (is_src) { 112 - alloc_area_alias = &area_src_alias; 100 + alloc_area_alias = &gopts->area_src_alias; 113 101 } else { 114 - alloc_area_alias = &area_dst_alias; 102 + alloc_area_alias = &gopts->area_dst_alias; 115 103 } 116 104 if (area_alias) 117 105 *alloc_area_alias = area_alias; ··· 120 108 return 0; 121 109 } 122 110 123 - static void hugetlb_alias_mapping(__u64 *start, size_t len, unsigned long offset) 111 + static void hugetlb_alias_mapping(uffd_global_test_opts_t *gopts, __u64 *start, 112 + size_t len, unsigned long offset) 124 113 { 125 - if (!map_shared) 114 + if (!gopts->map_shared) 126 115 return; 127 116 128 - *start = (unsigned long) area_dst_alias + offset; 117 + *start = (unsigned long) gopts->area_dst_alias + offset; 129 118 } 130 119 131 - static void shmem_release_pages(char *rel_area) 120 + static void shmem_release_pages(uffd_global_test_opts_t *gopts, char *rel_area) 132 121 { 133 - if (madvise(rel_area, nr_pages * page_size, MADV_REMOVE)) 122 + if (madvise(rel_area, gopts->nr_pages * gopts->page_size, MADV_REMOVE)) 134 123 err("madvise(MADV_REMOVE) failed"); 135 124 } 136 125 137 - static int shmem_allocate_area(void **alloc_area, bool is_src) 126 + static int shmem_allocate_area(uffd_global_test_opts_t *gopts, void **alloc_area, bool is_src) 138 127 { 139 128 void *area_alias = NULL; 140 - size_t bytes = nr_pages * page_size, hpage_size = read_pmd_pagesize(); 129 + size_t bytes = gopts->nr_pages * gopts->page_size, hpage_size = read_pmd_pagesize(); 141 130 unsigned long offset = is_src ? 0 : bytes; 142 131 char *p = NULL, *p_alias = NULL; 143 132 int mem_fd = uffd_mem_fd_create(bytes * 2, false); ··· 172 159 err("mmap of anonymous memory failed at %p", p_alias); 173 160 174 161 if (is_src) 175 - area_src_alias = area_alias; 162 + gopts->area_src_alias = area_alias; 176 163 else 177 - area_dst_alias = area_alias; 164 + gopts->area_dst_alias = area_alias; 178 165 179 166 close(mem_fd); 180 167 return 0; 181 168 } 182 169 183 - static void shmem_alias_mapping(__u64 *start, size_t len, unsigned long offset) 170 + static void shmem_alias_mapping(uffd_global_test_opts_t *gopts, __u64 *start, 171 + size_t len, unsigned long offset) 184 172 { 185 - *start = (unsigned long)area_dst_alias + offset; 173 + *start = (unsigned long)gopts->area_dst_alias + offset; 186 174 } 187 175 188 - static void shmem_check_pmd_mapping(void *p, int expect_nr_hpages) 176 + static void shmem_check_pmd_mapping(uffd_global_test_opts_t *gopts, void *p, int expect_nr_hpages) 189 177 { 190 - if (!check_huge_shmem(area_dst_alias, expect_nr_hpages, 178 + if (!check_huge_shmem(gopts->area_dst_alias, expect_nr_hpages, 191 179 read_pmd_pagesize())) 192 180 err("Did not find expected %d number of hugepages", 193 181 expect_nr_hpages); ··· 248 234 printf("\n"); 249 235 } 250 236 251 - int userfaultfd_open(uint64_t *features) 237 + int userfaultfd_open(uffd_global_test_opts_t *gopts, uint64_t *features) 252 238 { 253 239 struct uffdio_api uffdio_api; 254 240 255 - uffd = uffd_open(UFFD_FLAGS); 256 - if (uffd < 0) 241 + gopts->uffd = uffd_open(UFFD_FLAGS); 242 + if (gopts->uffd < 0) 257 243 return -1; 258 - uffd_flags = fcntl(uffd, F_GETFD, NULL); 244 + gopts->uffd_flags = fcntl(gopts->uffd, F_GETFD, NULL); 259 245 260 246 uffdio_api.api = UFFD_API; 261 247 uffdio_api.features = *features; 262 - if (ioctl(uffd, UFFDIO_API, &uffdio_api)) 248 + if (ioctl(gopts->uffd, UFFDIO_API, &uffdio_api)) 263 249 /* Probably lack of CAP_PTRACE? */ 264 250 return -1; 265 251 if (uffdio_api.api != UFFD_API) ··· 269 255 return 0; 270 256 } 271 257 272 - static inline void munmap_area(void **area) 258 + static inline void munmap_area(uffd_global_test_opts_t *gopts, void **area) 273 259 { 274 260 if (*area) 275 - if (munmap(*area, nr_pages * page_size)) 261 + if (munmap(*area, gopts->nr_pages * gopts->page_size)) 276 262 err("munmap"); 277 263 278 264 *area = NULL; 279 265 } 280 266 281 - void uffd_test_ctx_clear(void) 267 + void uffd_test_ctx_clear(uffd_global_test_opts_t *gopts) 282 268 { 283 269 size_t i; 284 270 285 - if (pipefd) { 286 - for (i = 0; i < nr_parallel * 2; ++i) { 287 - if (close(pipefd[i])) 271 + if (gopts->pipefd) { 272 + for (i = 0; i < gopts->nr_parallel * 2; ++i) { 273 + if (close(gopts->pipefd[i])) 288 274 err("close pipefd"); 289 275 } 290 - free(pipefd); 291 - pipefd = NULL; 276 + free(gopts->pipefd); 277 + gopts->pipefd = NULL; 292 278 } 293 279 294 - if (count_verify) { 295 - free(count_verify); 296 - count_verify = NULL; 280 + if (gopts->count_verify) { 281 + free(gopts->count_verify); 282 + gopts->count_verify = NULL; 297 283 } 298 284 299 - if (uffd != -1) { 300 - if (close(uffd)) 285 + if (gopts->uffd != -1) { 286 + if (close(gopts->uffd)) 301 287 err("close uffd"); 302 - uffd = -1; 288 + gopts->uffd = -1; 303 289 } 304 290 305 - munmap_area((void **)&area_src); 306 - munmap_area((void **)&area_src_alias); 307 - munmap_area((void **)&area_dst); 308 - munmap_area((void **)&area_dst_alias); 309 - munmap_area((void **)&area_remap); 291 + munmap_area(gopts, (void **)&gopts->area_src); 292 + munmap_area(gopts, (void **)&gopts->area_src_alias); 293 + munmap_area(gopts, (void **)&gopts->area_dst); 294 + munmap_area(gopts, (void **)&gopts->area_dst_alias); 295 + munmap_area(gopts, (void **)&gopts->area_remap); 310 296 } 311 297 312 - int uffd_test_ctx_init(uint64_t features, const char **errmsg) 298 + int uffd_test_ctx_init(uffd_global_test_opts_t *gopts, uint64_t features, const char **errmsg) 313 299 { 314 300 unsigned long nr, cpu; 315 301 int ret; 316 302 303 + gopts->area_src_alias = NULL; 304 + gopts->area_dst_alias = NULL; 305 + gopts->area_remap = NULL; 306 + 317 307 if (uffd_test_case_ops && uffd_test_case_ops->pre_alloc) { 318 - ret = uffd_test_case_ops->pre_alloc(errmsg); 308 + ret = uffd_test_case_ops->pre_alloc(gopts, errmsg); 319 309 if (ret) 320 310 return ret; 321 311 } 322 312 323 - ret = uffd_test_ops->allocate_area((void **)&area_src, true); 324 - ret |= uffd_test_ops->allocate_area((void **)&area_dst, false); 313 + ret = uffd_test_ops->allocate_area(gopts, (void **) &gopts->area_src, true); 314 + ret |= uffd_test_ops->allocate_area(gopts, (void **) &gopts->area_dst, false); 325 315 if (ret) { 326 316 if (errmsg) 327 317 *errmsg = "memory allocation failed"; ··· 333 315 } 334 316 335 317 if (uffd_test_case_ops && uffd_test_case_ops->post_alloc) { 336 - ret = uffd_test_case_ops->post_alloc(errmsg); 318 + ret = uffd_test_case_ops->post_alloc(gopts, errmsg); 337 319 if (ret) 338 320 return ret; 339 321 } 340 322 341 - ret = userfaultfd_open(&features); 323 + ret = userfaultfd_open(gopts, &features); 342 324 if (ret) { 343 325 if (errmsg) 344 326 *errmsg = "possible lack of privilege"; 345 327 return ret; 346 328 } 347 329 348 - count_verify = malloc(nr_pages * sizeof(unsigned long long)); 349 - if (!count_verify) 330 + gopts->count_verify = malloc(gopts->nr_pages * sizeof(unsigned long long)); 331 + if (!gopts->count_verify) 350 332 err("count_verify"); 351 333 352 - for (nr = 0; nr < nr_pages; nr++) { 353 - *area_mutex(area_src, nr) = 334 + for (nr = 0; nr < gopts->nr_pages; nr++) { 335 + *area_mutex(gopts->area_src, nr, gopts) = 354 336 (pthread_mutex_t)PTHREAD_MUTEX_INITIALIZER; 355 - count_verify[nr] = *area_count(area_src, nr) = 1; 337 + gopts->count_verify[nr] = *area_count(gopts->area_src, nr, gopts) = 1; 356 338 /* 357 339 * In the transition between 255 to 256, powerpc will 358 340 * read out of order in my_bcmp and see both bytes as ··· 360 342 * after the count, to avoid my_bcmp to trigger false 361 343 * positives. 362 344 */ 363 - *(area_count(area_src, nr) + 1) = 1; 345 + *(area_count(gopts->area_src, nr, gopts) + 1) = 1; 364 346 } 365 347 366 348 /* ··· 381 363 * proactively split the thp and drop any accidentally initialized 382 364 * pages within area_dst. 383 365 */ 384 - uffd_test_ops->release_pages(area_dst); 366 + uffd_test_ops->release_pages(gopts, gopts->area_dst); 385 367 386 - pipefd = malloc(sizeof(int) * nr_parallel * 2); 387 - if (!pipefd) 368 + gopts->pipefd = malloc(sizeof(int) * gopts->nr_parallel * 2); 369 + if (!gopts->pipefd) 388 370 err("pipefd"); 389 - for (cpu = 0; cpu < nr_parallel; cpu++) 390 - if (pipe2(&pipefd[cpu * 2], O_CLOEXEC | O_NONBLOCK)) 371 + for (cpu = 0; cpu < gopts->nr_parallel; cpu++) 372 + if (pipe2(&gopts->pipefd[cpu * 2], O_CLOEXEC | O_NONBLOCK)) 391 373 err("pipe"); 392 374 393 375 return 0; ··· 434 416 ret, (int64_t) req.mapped); 435 417 } 436 418 437 - int uffd_read_msg(int ufd, struct uffd_msg *msg) 419 + int uffd_read_msg(uffd_global_test_opts_t *gopts, struct uffd_msg *msg) 438 420 { 439 - int ret = read(uffd, msg, sizeof(*msg)); 421 + int ret = read(gopts->uffd, msg, sizeof(*msg)); 440 422 441 423 if (ret != sizeof(*msg)) { 442 424 if (ret < 0) { ··· 451 433 return 0; 452 434 } 453 435 454 - void uffd_handle_page_fault(struct uffd_msg *msg, struct uffd_args *args) 436 + void uffd_handle_page_fault(uffd_global_test_opts_t *gopts, struct uffd_msg *msg, 437 + struct uffd_args *args) 455 438 { 456 439 unsigned long offset; 457 440 ··· 461 442 462 443 if (msg->arg.pagefault.flags & UFFD_PAGEFAULT_FLAG_WP) { 463 444 /* Write protect page faults */ 464 - wp_range(uffd, msg->arg.pagefault.address, page_size, false); 445 + wp_range(gopts->uffd, msg->arg.pagefault.address, gopts->page_size, false); 465 446 args->wp_faults++; 466 447 } else if (msg->arg.pagefault.flags & UFFD_PAGEFAULT_FLAG_MINOR) { 467 448 uint8_t *area; ··· 479 460 * (UFFD-registered). 480 461 */ 481 462 482 - area = (uint8_t *)(area_dst + 483 - ((char *)msg->arg.pagefault.address - 484 - area_dst_alias)); 485 - for (b = 0; b < page_size; ++b) 463 + area = (uint8_t *)(gopts->area_dst + 464 + ((char *)msg->arg.pagefault.address - 465 + gopts->area_dst_alias)); 466 + for (b = 0; b < gopts->page_size; ++b) 486 467 area[b] = ~area[b]; 487 - continue_range(uffd, msg->arg.pagefault.address, page_size, 468 + continue_range(gopts->uffd, msg->arg.pagefault.address, gopts->page_size, 488 469 args->apply_wp); 489 470 args->minor_faults++; 490 471 } else { ··· 512 493 if (msg->arg.pagefault.flags & UFFD_PAGEFAULT_FLAG_WRITE) 513 494 err("unexpected write fault"); 514 495 515 - offset = (char *)(unsigned long)msg->arg.pagefault.address - area_dst; 516 - offset &= ~(page_size-1); 496 + offset = (char *)(unsigned long)msg->arg.pagefault.address - gopts->area_dst; 497 + offset &= ~(gopts->page_size-1); 517 498 518 - if (copy_page(uffd, offset, args->apply_wp)) 499 + if (copy_page(gopts, offset, args->apply_wp)) 519 500 args->missing_faults++; 520 501 } 521 502 } ··· 523 504 void *uffd_poll_thread(void *arg) 524 505 { 525 506 struct uffd_args *args = (struct uffd_args *)arg; 507 + uffd_global_test_opts_t *gopts = args->gopts; 526 508 unsigned long cpu = args->cpu; 527 509 struct pollfd pollfd[2]; 528 510 struct uffd_msg msg; ··· 534 514 if (!args->handle_fault) 535 515 args->handle_fault = uffd_handle_page_fault; 536 516 537 - pollfd[0].fd = uffd; 517 + pollfd[0].fd = gopts->uffd; 538 518 pollfd[0].events = POLLIN; 539 - pollfd[1].fd = pipefd[cpu*2]; 519 + pollfd[1].fd = gopts->pipefd[cpu*2]; 540 520 pollfd[1].events = POLLIN; 541 521 542 - ready_for_fork = true; 522 + gopts->ready_for_fork = true; 543 523 544 524 for (;;) { 545 525 ret = poll(pollfd, 2, -1); ··· 557 537 } 558 538 if (!(pollfd[0].revents & POLLIN)) 559 539 err("pollfd[0].revents %d", pollfd[0].revents); 560 - if (uffd_read_msg(uffd, &msg)) 540 + if (uffd_read_msg(gopts, &msg)) 561 541 continue; 562 542 switch (msg.event) { 563 543 default: 564 544 err("unexpected msg event %u\n", msg.event); 565 545 break; 566 546 case UFFD_EVENT_PAGEFAULT: 567 - args->handle_fault(&msg, args); 547 + args->handle_fault(gopts, &msg, args); 568 548 break; 569 549 case UFFD_EVENT_FORK: 570 - close(uffd); 571 - uffd = msg.arg.fork.ufd; 572 - pollfd[0].fd = uffd; 550 + close(gopts->uffd); 551 + gopts->uffd = msg.arg.fork.ufd; 552 + pollfd[0].fd = gopts->uffd; 573 553 break; 574 554 case UFFD_EVENT_REMOVE: 575 555 uffd_reg.range.start = msg.arg.remove.start; 576 556 uffd_reg.range.len = msg.arg.remove.end - 577 557 msg.arg.remove.start; 578 - if (ioctl(uffd, UFFDIO_UNREGISTER, &uffd_reg.range)) 558 + if (ioctl(gopts->uffd, UFFDIO_UNREGISTER, &uffd_reg.range)) 579 559 err("remove failure"); 580 560 break; 581 561 case UFFD_EVENT_REMAP: 582 - area_remap = area_dst; /* save for later unmap */ 583 - area_dst = (char *)(unsigned long)msg.arg.remap.to; 562 + gopts->area_remap = gopts->area_dst; /* save for later unmap */ 563 + gopts->area_dst = (char *)(unsigned long)msg.arg.remap.to; 584 564 break; 585 565 } 586 566 } ··· 588 568 return NULL; 589 569 } 590 570 591 - static void retry_copy_page(int ufd, struct uffdio_copy *uffdio_copy, 571 + static void retry_copy_page(uffd_global_test_opts_t *gopts, struct uffdio_copy *uffdio_copy, 592 572 unsigned long offset) 593 573 { 594 - uffd_test_ops->alias_mapping(&uffdio_copy->dst, 574 + uffd_test_ops->alias_mapping(gopts, 575 + &uffdio_copy->dst, 595 576 uffdio_copy->len, 596 577 offset); 597 - if (ioctl(ufd, UFFDIO_COPY, uffdio_copy)) { 578 + if (ioctl(gopts->uffd, UFFDIO_COPY, uffdio_copy)) { 598 579 /* real retval in ufdio_copy.copy */ 599 580 if (uffdio_copy->copy != -EEXIST) 600 581 err("UFFDIO_COPY retry error: %"PRId64, 601 - (int64_t)uffdio_copy->copy); 582 + (int64_t)uffdio_copy->copy); 602 583 } else { 603 584 err("UFFDIO_COPY retry unexpected: %"PRId64, 604 585 (int64_t)uffdio_copy->copy); ··· 618 597 addr), exit(1); 619 598 } 620 599 621 - int __copy_page(int ufd, unsigned long offset, bool retry, bool wp) 600 + int __copy_page(uffd_global_test_opts_t *gopts, unsigned long offset, bool retry, bool wp) 622 601 { 623 602 struct uffdio_copy uffdio_copy; 624 603 625 - if (offset >= nr_pages * page_size) 604 + if (offset >= gopts->nr_pages * gopts->page_size) 626 605 err("unexpected offset %lu\n", offset); 627 - uffdio_copy.dst = (unsigned long) area_dst + offset; 628 - uffdio_copy.src = (unsigned long) area_src + offset; 629 - uffdio_copy.len = page_size; 606 + uffdio_copy.dst = (unsigned long) gopts->area_dst + offset; 607 + uffdio_copy.src = (unsigned long) gopts->area_src + offset; 608 + uffdio_copy.len = gopts->page_size; 630 609 if (wp) 631 610 uffdio_copy.mode = UFFDIO_COPY_MODE_WP; 632 611 else 633 612 uffdio_copy.mode = 0; 634 613 uffdio_copy.copy = 0; 635 - if (ioctl(ufd, UFFDIO_COPY, &uffdio_copy)) { 614 + if (ioctl(gopts->uffd, UFFDIO_COPY, &uffdio_copy)) { 636 615 /* real retval in ufdio_copy.copy */ 637 616 if (uffdio_copy.copy != -EEXIST) 638 617 err("UFFDIO_COPY error: %"PRId64, 639 618 (int64_t)uffdio_copy.copy); 640 - wake_range(ufd, uffdio_copy.dst, page_size); 641 - } else if (uffdio_copy.copy != page_size) { 619 + wake_range(gopts->uffd, uffdio_copy.dst, gopts->page_size); 620 + } else if (uffdio_copy.copy != gopts->page_size) { 642 621 err("UFFDIO_COPY error: %"PRId64, (int64_t)uffdio_copy.copy); 643 622 } else { 644 - if (test_uffdio_copy_eexist && retry) { 645 - test_uffdio_copy_eexist = false; 646 - retry_copy_page(ufd, &uffdio_copy, offset); 623 + if (gopts->test_uffdio_copy_eexist && retry) { 624 + gopts->test_uffdio_copy_eexist = false; 625 + retry_copy_page(gopts, &uffdio_copy, offset); 647 626 } 648 627 return 1; 649 628 } 650 629 return 0; 651 630 } 652 631 653 - int copy_page(int ufd, unsigned long offset, bool wp) 632 + int copy_page(uffd_global_test_opts_t *gopts, unsigned long offset, bool wp) 654 633 { 655 - return __copy_page(ufd, offset, false, wp); 634 + return __copy_page(gopts, offset, false, wp); 656 635 } 657 636 658 - int move_page(int ufd, unsigned long offset, unsigned long len) 637 + int move_page(uffd_global_test_opts_t *gopts, unsigned long offset, unsigned long len) 659 638 { 660 639 struct uffdio_move uffdio_move; 661 640 662 - if (offset + len > nr_pages * page_size) 641 + if (offset + len > gopts->nr_pages * gopts->page_size) 663 642 err("unexpected offset %lu and length %lu\n", offset, len); 664 - uffdio_move.dst = (unsigned long) area_dst + offset; 665 - uffdio_move.src = (unsigned long) area_src + offset; 643 + uffdio_move.dst = (unsigned long) gopts->area_dst + offset; 644 + uffdio_move.src = (unsigned long) gopts->area_src + offset; 666 645 uffdio_move.len = len; 667 646 uffdio_move.mode = UFFDIO_MOVE_MODE_ALLOW_SRC_HOLES; 668 647 uffdio_move.move = 0; 669 - if (ioctl(ufd, UFFDIO_MOVE, &uffdio_move)) { 648 + if (ioctl(gopts->uffd, UFFDIO_MOVE, &uffdio_move)) { 670 649 /* real retval in uffdio_move.move */ 671 650 if (uffdio_move.move != -EEXIST) 672 651 err("UFFDIO_MOVE error: %"PRId64, 673 652 (int64_t)uffdio_move.move); 674 - wake_range(ufd, uffdio_move.dst, len); 653 + wake_range(gopts->uffd, uffdio_move.dst, len); 675 654 } else if (uffdio_move.move != len) { 676 655 err("UFFDIO_MOVE error: %"PRId64, (int64_t)uffdio_move.move); 677 656 } else

+40 -38

tools/testing/selftests/mm/uffd-common.h

··· 56 56 57 57 #define err(fmt, ...) errexit(1, fmt, ##__VA_ARGS__) 58 58 59 - /* pthread_mutex_t starts at page offset 0 */ 60 - #define area_mutex(___area, ___nr) \ 61 - ((pthread_mutex_t *) ((___area) + (___nr)*page_size)) 62 - /* 63 - * count is placed in the page after pthread_mutex_t naturally aligned 64 - * to avoid non alignment faults on non-x86 archs. 65 - */ 66 - #define area_count(___area, ___nr) \ 67 - ((volatile unsigned long long *) ((unsigned long) \ 68 - ((___area) + (___nr)*page_size + \ 69 - sizeof(pthread_mutex_t) + \ 70 - sizeof(unsigned long long) - 1) & \ 71 - ~(unsigned long)(sizeof(unsigned long long) \ 72 - - 1))) 59 + struct uffd_global_test_opts { 60 + unsigned long nr_parallel, nr_pages, nr_pages_per_cpu, page_size; 61 + char *area_src, *area_src_alias, *area_dst, *area_dst_alias, *area_remap; 62 + int uffd, uffd_flags, finished, *pipefd, test_type; 63 + bool map_shared; 64 + bool test_uffdio_wp; 65 + unsigned long long *count_verify; 66 + volatile bool test_uffdio_copy_eexist; 67 + atomic_bool ready_for_fork; 68 + }; 69 + typedef struct uffd_global_test_opts uffd_global_test_opts_t; 73 70 74 71 /* Userfaultfd test statistics */ 75 72 struct uffd_args { ··· 76 79 unsigned long missing_faults; 77 80 unsigned long wp_faults; 78 81 unsigned long minor_faults; 82 + struct uffd_global_test_opts *gopts; 79 83 80 84 /* A custom fault handler; defaults to uffd_handle_page_fault. */ 81 - void (*handle_fault)(struct uffd_msg *msg, struct uffd_args *args); 85 + void (*handle_fault)(struct uffd_global_test_opts *gopts, 86 + struct uffd_msg *msg, 87 + struct uffd_args *args); 82 88 }; 83 89 84 90 struct uffd_test_ops { 85 - int (*allocate_area)(void **alloc_area, bool is_src); 86 - void (*release_pages)(char *rel_area); 87 - void (*alias_mapping)(__u64 *start, size_t len, unsigned long offset); 88 - void (*check_pmd_mapping)(void *p, int expect_nr_hpages); 91 + int (*allocate_area)(uffd_global_test_opts_t *gopts, void **alloc_area, bool is_src); 92 + void (*release_pages)(uffd_global_test_opts_t *gopts, char *rel_area); 93 + void (*alias_mapping)(uffd_global_test_opts_t *gopts, 94 + __u64 *start, 95 + size_t len, 96 + unsigned long offset); 97 + void (*check_pmd_mapping)(uffd_global_test_opts_t *gopts, void *p, int expect_nr_hpages); 89 98 }; 90 99 typedef struct uffd_test_ops uffd_test_ops_t; 91 100 92 101 struct uffd_test_case_ops { 93 - int (*pre_alloc)(const char **errmsg); 94 - int (*post_alloc)(const char **errmsg); 102 + int (*pre_alloc)(uffd_global_test_opts_t *gopts, const char **errmsg); 103 + int (*post_alloc)(uffd_global_test_opts_t *gopts, const char **errmsg); 95 104 }; 96 105 typedef struct uffd_test_case_ops uffd_test_case_ops_t; 97 106 98 - extern unsigned long nr_parallel, nr_pages, nr_pages_per_cpu, page_size; 99 - extern char *area_src, *area_src_alias, *area_dst, *area_dst_alias, *area_remap; 100 - extern int uffd, uffd_flags, finished, *pipefd, test_type; 101 - extern bool map_shared; 102 - extern bool test_uffdio_wp; 103 - extern unsigned long long *count_verify; 104 - extern volatile bool test_uffdio_copy_eexist; 105 - extern atomic_bool ready_for_fork; 106 - 107 + extern uffd_global_test_opts_t *uffd_gtest_opts; 107 108 extern uffd_test_ops_t anon_uffd_test_ops; 108 109 extern uffd_test_ops_t shmem_uffd_test_ops; 109 110 extern uffd_test_ops_t hugetlb_uffd_test_ops; 110 111 extern uffd_test_ops_t *uffd_test_ops; 111 112 extern uffd_test_case_ops_t *uffd_test_case_ops; 112 113 114 + pthread_mutex_t *area_mutex(char *area, unsigned long nr, uffd_global_test_opts_t *gopts); 115 + volatile unsigned long long *area_count(char *area, 116 + unsigned long nr, 117 + uffd_global_test_opts_t *gopts); 118 + 113 119 void uffd_stats_report(struct uffd_args *args, int n_cpus); 114 - int uffd_test_ctx_init(uint64_t features, const char **errmsg); 115 - void uffd_test_ctx_clear(void); 116 - int userfaultfd_open(uint64_t *features); 117 - int uffd_read_msg(int ufd, struct uffd_msg *msg); 120 + int uffd_test_ctx_init(uffd_global_test_opts_t *gopts, uint64_t features, const char **errmsg); 121 + void uffd_test_ctx_clear(uffd_global_test_opts_t *gopts); 122 + int userfaultfd_open(uffd_global_test_opts_t *gopts, uint64_t *features); 123 + int uffd_read_msg(uffd_global_test_opts_t *gopts, struct uffd_msg *msg); 118 124 void wp_range(int ufd, __u64 start, __u64 len, bool wp); 119 - void uffd_handle_page_fault(struct uffd_msg *msg, struct uffd_args *args); 120 - int __copy_page(int ufd, unsigned long offset, bool retry, bool wp); 121 - int copy_page(int ufd, unsigned long offset, bool wp); 122 - int move_page(int ufd, unsigned long offset, unsigned long len); 125 + void uffd_handle_page_fault(uffd_global_test_opts_t *gopts, 126 + struct uffd_msg *msg, 127 + struct uffd_args *args); 128 + int __copy_page(uffd_global_test_opts_t *gopts, unsigned long offset, bool retry, bool wp); 129 + int copy_page(uffd_global_test_opts_t *gopts, unsigned long offset, bool wp); 130 + int move_page(uffd_global_test_opts_t *gopts, unsigned long offset, unsigned long len); 123 131 void *uffd_poll_thread(void *arg); 124 132 125 133 int uffd_open_dev(unsigned int flags);

+124 -104

tools/testing/selftests/mm/uffd-stress.c

··· 44 44 #define BOUNCE_VERIFY (1<<2) 45 45 #define BOUNCE_POLL (1<<3) 46 46 static int bounces; 47 + /* defined globally for this particular test as the sigalrm handler 48 + * depends on test_uffdio_*_eexist. 49 + * XXX: define gopts in main() when we figure out a way to deal with 50 + * test_uffdio_*_eexist. 51 + */ 52 + static uffd_global_test_opts_t *gopts; 47 53 48 54 /* exercise the test_uffdio_*_eexist every ALARM_INTERVAL_SECS */ 49 55 #define ALARM_INTERVAL_SECS 10 ··· 82 76 exit(1); 83 77 } 84 78 85 - static void uffd_stats_reset(struct uffd_args *args, unsigned long n_cpus) 79 + static void uffd_stats_reset(uffd_global_test_opts_t *gopts, struct uffd_args *args, 80 + unsigned long n_cpus) 86 81 { 87 82 int i; 88 83 89 84 for (i = 0; i < n_cpus; i++) { 90 85 args[i].cpu = i; 91 - args[i].apply_wp = test_uffdio_wp; 86 + args[i].apply_wp = gopts->test_uffdio_wp; 92 87 args[i].missing_faults = 0; 93 88 args[i].wp_faults = 0; 94 89 args[i].minor_faults = 0; 90 + args[i].gopts = gopts; 95 91 } 96 92 } 97 93 98 94 static void *locking_thread(void *arg) 99 95 { 100 - unsigned long cpu = (unsigned long) arg; 96 + struct uffd_args *args = (struct uffd_args *) arg; 97 + uffd_global_test_opts_t *gopts = args->gopts; 98 + unsigned long cpu = (unsigned long) args->cpu; 101 99 unsigned long page_nr; 102 100 unsigned long long count; 103 101 104 102 if (!(bounces & BOUNCE_RANDOM)) { 105 103 page_nr = -bounces; 106 104 if (!(bounces & BOUNCE_RACINGFAULTS)) 107 - page_nr += cpu * nr_pages_per_cpu; 105 + page_nr += cpu * gopts->nr_pages_per_cpu; 108 106 } 109 107 110 - while (!finished) { 108 + while (!gopts->finished) { 111 109 if (bounces & BOUNCE_RANDOM) { 112 110 if (getrandom(&page_nr, sizeof(page_nr), 0) != sizeof(page_nr)) 113 111 err("getrandom failed"); 114 112 } else 115 113 page_nr += 1; 116 - page_nr %= nr_pages; 117 - pthread_mutex_lock(area_mutex(area_dst, page_nr)); 118 - count = *area_count(area_dst, page_nr); 119 - if (count != count_verify[page_nr]) 114 + page_nr %= gopts->nr_pages; 115 + pthread_mutex_lock(area_mutex(gopts->area_dst, page_nr, gopts)); 116 + count = *area_count(gopts->area_dst, page_nr, gopts); 117 + if (count != gopts->count_verify[page_nr]) 120 118 err("page_nr %lu memory corruption %llu %llu", 121 - page_nr, count, count_verify[page_nr]); 119 + page_nr, count, gopts->count_verify[page_nr]); 122 120 count++; 123 - *area_count(area_dst, page_nr) = count_verify[page_nr] = count; 124 - pthread_mutex_unlock(area_mutex(area_dst, page_nr)); 121 + *area_count(gopts->area_dst, page_nr, gopts) = gopts->count_verify[page_nr] = count; 122 + pthread_mutex_unlock(area_mutex(gopts->area_dst, page_nr, gopts)); 125 123 } 126 124 127 125 return NULL; 128 126 } 129 127 130 - static int copy_page_retry(int ufd, unsigned long offset) 128 + static int copy_page_retry(uffd_global_test_opts_t *gopts, unsigned long offset) 131 129 { 132 - return __copy_page(ufd, offset, true, test_uffdio_wp); 130 + return __copy_page(gopts, offset, true, gopts->test_uffdio_wp); 133 131 } 134 132 135 133 pthread_mutex_t uffd_read_mutex = PTHREAD_MUTEX_INITIALIZER; ··· 141 131 static void *uffd_read_thread(void *arg) 142 132 { 143 133 struct uffd_args *args = (struct uffd_args *)arg; 134 + uffd_global_test_opts_t *gopts = args->gopts; 144 135 struct uffd_msg msg; 145 136 146 137 pthread_mutex_unlock(&uffd_read_mutex); 147 138 /* from here cancellation is ok */ 148 139 149 140 for (;;) { 150 - if (uffd_read_msg(uffd, &msg)) 141 + if (uffd_read_msg(gopts, &msg)) 151 142 continue; 152 - uffd_handle_page_fault(&msg, args); 143 + uffd_handle_page_fault(gopts, &msg, args); 153 144 } 154 145 155 146 return NULL; ··· 158 147 159 148 static void *background_thread(void *arg) 160 149 { 161 - unsigned long cpu = (unsigned long) arg; 150 + struct uffd_args *args = (struct uffd_args *) arg; 151 + uffd_global_test_opts_t *gopts = args->gopts; 152 + unsigned long cpu = (unsigned long) args->cpu; 162 153 unsigned long page_nr, start_nr, mid_nr, end_nr; 163 154 164 - start_nr = cpu * nr_pages_per_cpu; 165 - end_nr = (cpu+1) * nr_pages_per_cpu; 155 + start_nr = cpu * gopts->nr_pages_per_cpu; 156 + end_nr = (cpu+1) * gopts->nr_pages_per_cpu; 166 157 mid_nr = (start_nr + end_nr) / 2; 167 158 168 159 /* Copy the first half of the pages */ 169 160 for (page_nr = start_nr; page_nr < mid_nr; page_nr++) 170 - copy_page_retry(uffd, page_nr * page_size); 161 + copy_page_retry(gopts, page_nr * gopts->page_size); 171 162 172 163 /* 173 164 * If we need to test uffd-wp, set it up now. Then we'll have 174 165 * at least the first half of the pages mapped already which 175 166 * can be write-protected for testing 176 167 */ 177 - if (test_uffdio_wp) 178 - wp_range(uffd, (unsigned long)area_dst + start_nr * page_size, 179 - nr_pages_per_cpu * page_size, true); 168 + if (gopts->test_uffdio_wp) 169 + wp_range(gopts->uffd, (unsigned long)gopts->area_dst + start_nr * gopts->page_size, 170 + gopts->nr_pages_per_cpu * gopts->page_size, true); 180 171 181 172 /* 182 173 * Continue the 2nd half of the page copying, handling write 183 174 * protection faults if any 184 175 */ 185 176 for (page_nr = mid_nr; page_nr < end_nr; page_nr++) 186 - copy_page_retry(uffd, page_nr * page_size); 177 + copy_page_retry(gopts, page_nr * gopts->page_size); 187 178 188 179 return NULL; 189 180 } ··· 193 180 static int stress(struct uffd_args *args) 194 181 { 195 182 unsigned long cpu; 196 - pthread_t locking_threads[nr_parallel]; 197 - pthread_t uffd_threads[nr_parallel]; 198 - pthread_t background_threads[nr_parallel]; 183 + uffd_global_test_opts_t *gopts = args->gopts; 184 + pthread_t locking_threads[gopts->nr_parallel]; 185 + pthread_t uffd_threads[gopts->nr_parallel]; 186 + pthread_t background_threads[gopts->nr_parallel]; 199 187 200 - finished = 0; 201 - for (cpu = 0; cpu < nr_parallel; cpu++) { 188 + gopts->finished = 0; 189 + for (cpu = 0; cpu < gopts->nr_parallel; cpu++) { 202 190 if (pthread_create(&locking_threads[cpu], &attr, 203 - locking_thread, (void *)cpu)) 191 + locking_thread, (void *)&args[cpu])) 204 192 return 1; 205 193 if (bounces & BOUNCE_POLL) { 206 - if (pthread_create(&uffd_threads[cpu], &attr, uffd_poll_thread, &args[cpu])) 194 + if (pthread_create(&uffd_threads[cpu], 195 + &attr, 196 + uffd_poll_thread, 197 + (void *) &args[cpu])) 207 198 err("uffd_poll_thread create"); 208 199 } else { 209 200 if (pthread_create(&uffd_threads[cpu], &attr, ··· 217 200 pthread_mutex_lock(&uffd_read_mutex); 218 201 } 219 202 if (pthread_create(&background_threads[cpu], &attr, 220 - background_thread, (void *)cpu)) 203 + background_thread, (void *)&args[cpu])) 221 204 return 1; 222 205 } 223 - for (cpu = 0; cpu < nr_parallel; cpu++) 206 + for (cpu = 0; cpu < gopts->nr_parallel; cpu++) 224 207 if (pthread_join(background_threads[cpu], NULL)) 225 208 return 1; 226 209 ··· 233 216 * UFFDIO_COPY without writing zero pages into area_dst 234 217 * because the background threads already completed). 235 218 */ 236 - uffd_test_ops->release_pages(area_src); 219 + uffd_test_ops->release_pages(gopts, gopts->area_src); 237 220 238 - finished = 1; 239 - for (cpu = 0; cpu < nr_parallel; cpu++) 221 + gopts->finished = 1; 222 + for (cpu = 0; cpu < gopts->nr_parallel; cpu++) 240 223 if (pthread_join(locking_threads[cpu], NULL)) 241 224 return 1; 242 225 243 - for (cpu = 0; cpu < nr_parallel; cpu++) { 226 + for (cpu = 0; cpu < gopts->nr_parallel; cpu++) { 244 227 char c; 245 228 if (bounces & BOUNCE_POLL) { 246 - if (write(pipefd[cpu*2+1], &c, 1) != 1) 229 + if (write(gopts->pipefd[cpu*2+1], &c, 1) != 1) 247 230 err("pipefd write error"); 248 231 if (pthread_join(uffd_threads[cpu], 249 232 (void *)&args[cpu])) ··· 259 242 return 0; 260 243 } 261 244 262 - static int userfaultfd_stress(void) 245 + static int userfaultfd_stress(uffd_global_test_opts_t *gopts) 263 246 { 264 247 void *area; 265 248 unsigned long nr; 266 - struct uffd_args args[nr_parallel]; 267 - uint64_t mem_size = nr_pages * page_size; 249 + struct uffd_args args[gopts->nr_parallel]; 250 + uint64_t mem_size = gopts->nr_pages * gopts->page_size; 268 251 int flags = 0; 269 252 270 - memset(args, 0, sizeof(struct uffd_args) * nr_parallel); 253 + memset(args, 0, sizeof(struct uffd_args) * gopts->nr_parallel); 271 254 272 - if (features & UFFD_FEATURE_WP_UNPOPULATED && test_type == TEST_ANON) 255 + if (features & UFFD_FEATURE_WP_UNPOPULATED && gopts->test_type == TEST_ANON) 273 256 flags = UFFD_FEATURE_WP_UNPOPULATED; 274 257 275 - if (uffd_test_ctx_init(flags, NULL)) 258 + if (uffd_test_ctx_init(gopts, flags, NULL)) 276 259 err("context init failed"); 277 260 278 - if (posix_memalign(&area, page_size, page_size)) 261 + if (posix_memalign(&area, gopts->page_size, gopts->page_size)) 279 262 err("out of memory"); 280 263 zeropage = area; 281 - bzero(zeropage, page_size); 264 + bzero(zeropage, gopts->page_size); 282 265 283 266 pthread_mutex_lock(&uffd_read_mutex); 284 267 ··· 301 284 fflush(stdout); 302 285 303 286 if (bounces & BOUNCE_POLL) 304 - fcntl(uffd, F_SETFL, uffd_flags | O_NONBLOCK); 287 + fcntl(gopts->uffd, F_SETFL, gopts->uffd_flags | O_NONBLOCK); 305 288 else 306 - fcntl(uffd, F_SETFL, uffd_flags & ~O_NONBLOCK); 289 + fcntl(gopts->uffd, F_SETFL, gopts->uffd_flags & ~O_NONBLOCK); 307 290 308 291 /* register */ 309 - if (uffd_register(uffd, area_dst, mem_size, 310 - true, test_uffdio_wp, false)) 292 + if (uffd_register(gopts->uffd, gopts->area_dst, mem_size, 293 + true, gopts->test_uffdio_wp, false)) 311 294 err("register failure"); 312 295 313 - if (area_dst_alias) { 314 - if (uffd_register(uffd, area_dst_alias, mem_size, 315 - true, test_uffdio_wp, false)) 296 + if (gopts->area_dst_alias) { 297 + if (uffd_register(gopts->uffd, gopts->area_dst_alias, mem_size, 298 + true, gopts->test_uffdio_wp, false)) 316 299 err("register failure alias"); 317 300 } 318 301 ··· 340 323 * MADV_DONTNEED only after the UFFDIO_REGISTER, so it's 341 324 * required to MADV_DONTNEED here. 342 325 */ 343 - uffd_test_ops->release_pages(area_dst); 326 + uffd_test_ops->release_pages(gopts, gopts->area_dst); 344 327 345 - uffd_stats_reset(args, nr_parallel); 328 + uffd_stats_reset(gopts, args, gopts->nr_parallel); 346 329 347 330 /* bounce pass */ 348 331 if (stress(args)) { 349 - uffd_test_ctx_clear(); 332 + uffd_test_ctx_clear(gopts); 350 333 return 1; 351 334 } 352 335 353 336 /* Clear all the write protections if there is any */ 354 - if (test_uffdio_wp) 355 - wp_range(uffd, (unsigned long)area_dst, 356 - nr_pages * page_size, false); 337 + if (gopts->test_uffdio_wp) 338 + wp_range(gopts->uffd, (unsigned long)gopts->area_dst, 339 + gopts->nr_pages * gopts->page_size, false); 357 340 358 341 /* unregister */ 359 - if (uffd_unregister(uffd, area_dst, mem_size)) 342 + if (uffd_unregister(gopts->uffd, gopts->area_dst, mem_size)) 360 343 err("unregister failure"); 361 - if (area_dst_alias) { 362 - if (uffd_unregister(uffd, area_dst_alias, mem_size)) 344 + if (gopts->area_dst_alias) { 345 + if (uffd_unregister(gopts->uffd, gopts->area_dst_alias, mem_size)) 363 346 err("unregister failure alias"); 364 347 } 365 348 366 349 /* verification */ 367 350 if (bounces & BOUNCE_VERIFY) 368 - for (nr = 0; nr < nr_pages; nr++) 369 - if (*area_count(area_dst, nr) != count_verify[nr]) 351 + for (nr = 0; nr < gopts->nr_pages; nr++) 352 + if (*area_count(gopts->area_dst, nr, gopts) != 353 + gopts->count_verify[nr]) 370 354 err("error area_count %llu %llu %lu\n", 371 - *area_count(area_src, nr), 372 - count_verify[nr], nr); 355 + *area_count(gopts->area_src, nr, gopts), 356 + gopts->count_verify[nr], nr); 373 357 374 358 /* prepare next bounce */ 375 - swap(area_src, area_dst); 359 + swap(gopts->area_src, gopts->area_dst); 376 360 377 - swap(area_src_alias, area_dst_alias); 361 + swap(gopts->area_src_alias, gopts->area_dst_alias); 378 362 379 - uffd_stats_report(args, nr_parallel); 363 + uffd_stats_report(args, gopts->nr_parallel); 380 364 } 381 - uffd_test_ctx_clear(); 365 + uffd_test_ctx_clear(gopts); 382 366 383 367 return 0; 384 368 } 385 369 386 - static void set_test_type(const char *type) 370 + static void set_test_type(uffd_global_test_opts_t *gopts, const char *type) 387 371 { 388 372 if (!strcmp(type, "anon")) { 389 - test_type = TEST_ANON; 373 + gopts->test_type = TEST_ANON; 390 374 uffd_test_ops = &anon_uffd_test_ops; 391 375 } else if (!strcmp(type, "hugetlb")) { 392 - test_type = TEST_HUGETLB; 376 + gopts->test_type = TEST_HUGETLB; 393 377 uffd_test_ops = &hugetlb_uffd_test_ops; 394 - map_shared = true; 378 + gopts->map_shared = true; 395 379 } else if (!strcmp(type, "hugetlb-private")) { 396 - test_type = TEST_HUGETLB; 380 + gopts->test_type = TEST_HUGETLB; 397 381 uffd_test_ops = &hugetlb_uffd_test_ops; 398 382 } else if (!strcmp(type, "shmem")) { 399 - map_shared = true; 400 - test_type = TEST_SHMEM; 383 + gopts->map_shared = true; 384 + gopts->test_type = TEST_SHMEM; 401 385 uffd_test_ops = &shmem_uffd_test_ops; 402 386 } else if (!strcmp(type, "shmem-private")) { 403 - test_type = TEST_SHMEM; 387 + gopts->test_type = TEST_SHMEM; 404 388 uffd_test_ops = &shmem_uffd_test_ops; 405 389 } 406 390 } 407 391 408 - static void parse_test_type_arg(const char *raw_type) 392 + static void parse_test_type_arg(uffd_global_test_opts_t *gopts, const char *raw_type) 409 393 { 410 - set_test_type(raw_type); 394 + set_test_type(gopts, raw_type); 411 395 412 - if (!test_type) 396 + if (!gopts->test_type) 413 397 err("failed to parse test type argument: '%s'", raw_type); 414 398 415 - if (test_type == TEST_HUGETLB) 416 - page_size = default_huge_page_size(); 399 + if (gopts->test_type == TEST_HUGETLB) 400 + gopts->page_size = default_huge_page_size(); 417 401 else 418 - page_size = sysconf(_SC_PAGE_SIZE); 402 + gopts->page_size = sysconf(_SC_PAGE_SIZE); 419 403 420 - if (!page_size) 404 + if (!gopts->page_size) 421 405 err("Unable to determine page size"); 422 - if ((unsigned long) area_count(NULL, 0) + sizeof(unsigned long long) * 2 423 - > page_size) 406 + if ((unsigned long) area_count(NULL, 0, gopts) + sizeof(unsigned long long) * 2 407 + > gopts->page_size) 424 408 err("Impossible to run this test"); 425 409 426 410 /* ··· 433 415 if (uffd_get_features(&features) && errno == ENOENT) 434 416 ksft_exit_skip("failed to get available features (%d)\n", errno); 435 417 436 - test_uffdio_wp = test_uffdio_wp && 418 + gopts->test_uffdio_wp = gopts->test_uffdio_wp && 437 419 (features & UFFD_FEATURE_PAGEFAULT_FLAG_WP); 438 420 439 - if (test_type != TEST_ANON && !(features & UFFD_FEATURE_WP_HUGETLBFS_SHMEM)) 440 - test_uffdio_wp = false; 421 + if (gopts->test_type != TEST_ANON && !(features & UFFD_FEATURE_WP_HUGETLBFS_SHMEM)) 422 + gopts->test_uffdio_wp = false; 441 423 442 - close(uffd); 443 - uffd = -1; 424 + close(gopts->uffd); 425 + gopts->uffd = -1; 444 426 } 445 427 446 428 static void sigalrm(int sig) 447 429 { 448 430 if (sig != SIGALRM) 449 431 abort(); 450 - test_uffdio_copy_eexist = true; 432 + gopts->test_uffdio_copy_eexist = true; 451 433 alarm(ALARM_INTERVAL_SECS); 452 434 } 453 435 ··· 456 438 unsigned long nr_cpus; 457 439 size_t bytes; 458 440 441 + gopts = (uffd_global_test_opts_t *) malloc(sizeof(uffd_global_test_opts_t)); 442 + 459 443 if (argc < 4) 460 444 usage(); 461 445 ··· 465 445 err("failed to arm SIGALRM"); 466 446 alarm(ALARM_INTERVAL_SECS); 467 447 468 - parse_test_type_arg(argv[1]); 448 + parse_test_type_arg(gopts, argv[1]); 469 449 bytes = atol(argv[2]) * 1024 * 1024; 470 450 471 451 nr_cpus = sysconf(_SC_NPROCESSORS_ONLN); ··· 473 453 /* Don't let calculation below go to zero. */ 474 454 ksft_print_msg("_SC_NPROCESSORS_ONLN (%lu) too large, capping nr_threads to 32\n", 475 455 nr_cpus); 476 - nr_parallel = 32; 456 + gopts->nr_parallel = 32; 477 457 } else { 478 - nr_parallel = nr_cpus; 458 + gopts->nr_parallel = nr_cpus; 479 459 } 480 460 481 461 /* ··· 483 463 * Ensure nr_parallel - 1 hugepages on top of that to account 484 464 * for racy extra reservation of hugepages. 485 465 */ 486 - if (test_type == TEST_HUGETLB && 487 - get_free_hugepages() < 2 * (bytes / page_size) + nr_parallel - 1) { 466 + if (gopts->test_type == TEST_HUGETLB && 467 + get_free_hugepages() < 2 * (bytes / gopts->page_size) + gopts->nr_parallel - 1) { 488 468 printf("skip: Skipping userfaultfd... not enough hugepages\n"); 489 469 return KSFT_SKIP; 490 470 } 491 471 492 - nr_pages_per_cpu = bytes / page_size / nr_parallel; 493 - if (!nr_pages_per_cpu) { 472 + gopts->nr_pages_per_cpu = bytes / gopts->page_size / gopts->nr_parallel; 473 + if (!gopts->nr_pages_per_cpu) { 494 474 _err("pages_per_cpu = 0, cannot test (%lu / %lu / %lu)", 495 - bytes, page_size, nr_parallel); 475 + bytes, gopts->page_size, gopts->nr_parallel); 496 476 usage(); 497 477 } 498 478 ··· 501 481 _err("invalid bounces"); 502 482 usage(); 503 483 } 504 - nr_pages = nr_pages_per_cpu * nr_parallel; 484 + gopts->nr_pages = gopts->nr_pages_per_cpu * gopts->nr_parallel; 505 485 506 486 printf("nr_pages: %lu, nr_pages_per_cpu: %lu\n", 507 - nr_pages, nr_pages_per_cpu); 508 - return userfaultfd_stress(); 487 + gopts->nr_pages, gopts->nr_pages_per_cpu); 488 + return userfaultfd_stress(gopts); 509 489 } 510 490 511 491 #else /* __NR_userfaultfd */

+294 -265

tools/testing/selftests/mm/uffd-unit-tests.c

··· 76 76 typedef struct uffd_test_args uffd_test_args_t; 77 77 78 78 /* Returns: UFFD_TEST_* */ 79 - typedef void (*uffd_test_fn)(uffd_test_args_t *); 79 + typedef void (*uffd_test_fn)(uffd_global_test_opts_t *, uffd_test_args_t *); 80 80 81 81 typedef struct { 82 82 const char *name; ··· 181 181 return 1; 182 182 } 183 183 184 - /* 185 - * This function initializes the global variables. TODO: remove global 186 - * vars and then remove this. 187 - */ 188 - static int 189 - uffd_setup_environment(uffd_test_args_t *args, uffd_test_case_t *test, 190 - mem_type_t *mem_type, const char **errmsg) 191 - { 192 - map_shared = mem_type->shared; 193 - uffd_test_ops = mem_type->mem_ops; 194 - uffd_test_case_ops = test->test_case_ops; 195 - 196 - if (mem_type->mem_flag & (MEM_HUGETLB_PRIVATE | MEM_HUGETLB)) 197 - page_size = default_huge_page_size(); 198 - else 199 - page_size = psize(); 200 - 201 - /* Ensure we have at least 2 pages */ 202 - nr_pages = MAX(UFFD_TEST_MEM_SIZE, page_size * 2) / page_size; 203 - /* TODO: remove this global var.. it's so ugly */ 204 - nr_parallel = 1; 205 - 206 - /* Initialize test arguments */ 207 - args->mem_type = mem_type; 208 - 209 - return uffd_test_ctx_init(test->uffd_feature_required, errmsg); 210 - } 211 184 212 185 static bool uffd_feature_supported(uffd_test_case_t *test) 213 186 { ··· 210 237 } while (0) 211 238 212 239 typedef struct { 213 - int parent_uffd, child_uffd; 240 + uffd_global_test_opts_t *gopts; 241 + int child_uffd; 214 242 } fork_event_args; 215 243 216 244 static void *fork_event_consumer(void *data) ··· 219 245 fork_event_args *args = data; 220 246 struct uffd_msg msg = { 0 }; 221 247 222 - ready_for_fork = true; 248 + args->gopts->ready_for_fork = true; 223 249 224 250 /* Read until a full msg received */ 225 - while (uffd_read_msg(args->parent_uffd, &msg)); 251 + while (uffd_read_msg(args->gopts, &msg)); 226 252 227 253 if (msg.event != UFFD_EVENT_FORK) 228 254 err("wrong message: %u\n", msg.event); ··· 278 304 args->pinned = false; 279 305 } 280 306 281 - static int pagemap_test_fork(int uffd, bool with_event, bool test_pin) 307 + static int pagemap_test_fork(uffd_global_test_opts_t *gopts, bool with_event, bool test_pin) 282 308 { 283 - fork_event_args args = { .parent_uffd = uffd, .child_uffd = -1 }; 309 + fork_event_args args = { .gopts = gopts, .child_uffd = -1 }; 284 310 pthread_t thread; 285 311 pid_t child; 286 312 uint64_t value; ··· 288 314 289 315 /* Prepare a thread to resolve EVENT_FORK */ 290 316 if (with_event) { 291 - ready_for_fork = false; 317 + gopts->ready_for_fork = false; 292 318 if (pthread_create(&thread, NULL, fork_event_consumer, &args)) 293 319 err("pthread_create()"); 294 - while (!ready_for_fork) 320 + while (!gopts->ready_for_fork) 295 321 ; /* Wait for the poll_thread to start executing before forking */ 296 322 } 297 323 ··· 302 328 303 329 fd = pagemap_open(); 304 330 305 - if (test_pin && pin_pages(&args, area_dst, page_size)) 331 + if (test_pin && pin_pages(&args, gopts->area_dst, gopts->page_size)) 306 332 /* 307 333 * Normally when reach here we have pinned in 308 334 * previous tests, so shouldn't fail anymore 309 335 */ 310 336 err("pin page failed in child"); 311 337 312 - value = pagemap_get_entry(fd, area_dst); 338 + value = pagemap_get_entry(fd, gopts->area_dst); 313 339 /* 314 340 * After fork(), we should handle uffd-wp bit differently: 315 341 * ··· 335 361 return result; 336 362 } 337 363 338 - static void uffd_wp_unpopulated_test(uffd_test_args_t *args) 364 + static void uffd_wp_unpopulated_test(uffd_global_test_opts_t *gopts, uffd_test_args_t *args) 339 365 { 340 366 uint64_t value; 341 367 int pagemap_fd; 342 368 343 - if (uffd_register(uffd, area_dst, nr_pages * page_size, 369 + if (uffd_register(gopts->uffd, gopts->area_dst, gopts->nr_pages * gopts->page_size, 344 370 false, true, false)) 345 371 err("register failed"); 346 372 347 373 pagemap_fd = pagemap_open(); 348 374 349 375 /* Test applying pte marker to anon unpopulated */ 350 - wp_range(uffd, (uint64_t)area_dst, page_size, true); 351 - value = pagemap_get_entry(pagemap_fd, area_dst); 376 + wp_range(gopts->uffd, (uint64_t)gopts->area_dst, gopts->page_size, true); 377 + value = pagemap_get_entry(pagemap_fd, gopts->area_dst); 352 378 pagemap_check_wp(value, true); 353 379 354 380 /* Test unprotect on anon pte marker */ 355 - wp_range(uffd, (uint64_t)area_dst, page_size, false); 356 - value = pagemap_get_entry(pagemap_fd, area_dst); 381 + wp_range(gopts->uffd, (uint64_t)gopts->area_dst, gopts->page_size, false); 382 + value = pagemap_get_entry(pagemap_fd, gopts->area_dst); 357 383 pagemap_check_wp(value, false); 358 384 359 385 /* Test zap on anon marker */ 360 - wp_range(uffd, (uint64_t)area_dst, page_size, true); 361 - if (madvise(area_dst, page_size, MADV_DONTNEED)) 386 + wp_range(gopts->uffd, (uint64_t)gopts->area_dst, gopts->page_size, true); 387 + if (madvise(gopts->area_dst, gopts->page_size, MADV_DONTNEED)) 362 388 err("madvise(MADV_DONTNEED) failed"); 363 - value = pagemap_get_entry(pagemap_fd, area_dst); 389 + value = pagemap_get_entry(pagemap_fd, gopts->area_dst); 364 390 pagemap_check_wp(value, false); 365 391 366 392 /* Test fault in after marker removed */ 367 - *area_dst = 1; 368 - value = pagemap_get_entry(pagemap_fd, area_dst); 393 + *gopts->area_dst = 1; 394 + value = pagemap_get_entry(pagemap_fd, gopts->area_dst); 369 395 pagemap_check_wp(value, false); 370 396 /* Drop it to make pte none again */ 371 - if (madvise(area_dst, page_size, MADV_DONTNEED)) 397 + if (madvise(gopts->area_dst, gopts->page_size, MADV_DONTNEED)) 372 398 err("madvise(MADV_DONTNEED) failed"); 373 399 374 400 /* Test read-zero-page upon pte marker */ 375 - wp_range(uffd, (uint64_t)area_dst, page_size, true); 376 - *(volatile char *)area_dst; 401 + wp_range(gopts->uffd, (uint64_t)gopts->area_dst, gopts->page_size, true); 402 + *(volatile char *)gopts->area_dst; 377 403 /* Drop it to make pte none again */ 378 - if (madvise(area_dst, page_size, MADV_DONTNEED)) 404 + if (madvise(gopts->area_dst, gopts->page_size, MADV_DONTNEED)) 379 405 err("madvise(MADV_DONTNEED) failed"); 380 406 381 407 uffd_test_pass(); 382 408 } 383 409 384 - static void uffd_wp_fork_test_common(uffd_test_args_t *args, 410 + static void uffd_wp_fork_test_common(uffd_global_test_opts_t *gopts, uffd_test_args_t *args, 385 411 bool with_event) 386 412 { 387 413 int pagemap_fd; 388 414 uint64_t value; 389 415 390 - if (uffd_register(uffd, area_dst, nr_pages * page_size, 416 + if (uffd_register(gopts->uffd, gopts->area_dst, gopts->nr_pages * gopts->page_size, 391 417 false, true, false)) 392 418 err("register failed"); 393 419 394 420 pagemap_fd = pagemap_open(); 395 421 396 422 /* Touch the page */ 397 - *area_dst = 1; 398 - wp_range(uffd, (uint64_t)area_dst, page_size, true); 399 - value = pagemap_get_entry(pagemap_fd, area_dst); 423 + *gopts->area_dst = 1; 424 + wp_range(gopts->uffd, (uint64_t)gopts->area_dst, gopts->page_size, true); 425 + value = pagemap_get_entry(pagemap_fd, gopts->area_dst); 400 426 pagemap_check_wp(value, true); 401 - if (pagemap_test_fork(uffd, with_event, false)) { 427 + if (pagemap_test_fork(gopts, with_event, false)) { 402 428 uffd_test_fail("Detected %s uffd-wp bit in child in present pte", 403 429 with_event ? "missing" : "stall"); 404 430 goto out; ··· 416 442 * to expose pte markers. 417 443 */ 418 444 if (args->mem_type->shared) { 419 - if (madvise(area_dst, page_size, MADV_DONTNEED)) 445 + if (madvise(gopts->area_dst, gopts->page_size, MADV_DONTNEED)) 420 446 err("MADV_DONTNEED"); 421 447 } else { 422 448 /* 423 449 * NOTE: ignore retval because private-hugetlb doesn't yet 424 450 * support swapping, so it could fail. 425 451 */ 426 - madvise(area_dst, page_size, MADV_PAGEOUT); 452 + madvise(gopts->area_dst, gopts->page_size, MADV_PAGEOUT); 427 453 } 428 454 429 455 /* Uffd-wp should persist even swapped out */ 430 - value = pagemap_get_entry(pagemap_fd, area_dst); 456 + value = pagemap_get_entry(pagemap_fd, gopts->area_dst); 431 457 pagemap_check_wp(value, true); 432 - if (pagemap_test_fork(uffd, with_event, false)) { 458 + if (pagemap_test_fork(gopts, with_event, false)) { 433 459 uffd_test_fail("Detected %s uffd-wp bit in child in zapped pte", 434 460 with_event ? "missing" : "stall"); 435 461 goto out; 436 462 } 437 463 438 464 /* Unprotect; this tests swap pte modifications */ 439 - wp_range(uffd, (uint64_t)area_dst, page_size, false); 440 - value = pagemap_get_entry(pagemap_fd, area_dst); 465 + wp_range(gopts->uffd, (uint64_t)gopts->area_dst, gopts->page_size, false); 466 + value = pagemap_get_entry(pagemap_fd, gopts->area_dst); 441 467 pagemap_check_wp(value, false); 442 468 443 469 /* Fault in the page from disk */ 444 - *area_dst = 2; 445 - value = pagemap_get_entry(pagemap_fd, area_dst); 470 + *gopts->area_dst = 2; 471 + value = pagemap_get_entry(pagemap_fd, gopts->area_dst); 446 472 pagemap_check_wp(value, false); 447 473 uffd_test_pass(); 448 474 out: 449 - if (uffd_unregister(uffd, area_dst, nr_pages * page_size)) 475 + if (uffd_unregister(gopts->uffd, gopts->area_dst, gopts->nr_pages * gopts->page_size)) 450 476 err("unregister failed"); 451 477 close(pagemap_fd); 452 478 } 453 479 454 - static void uffd_wp_fork_test(uffd_test_args_t *args) 480 + static void uffd_wp_fork_test(uffd_global_test_opts_t *gopts, uffd_test_args_t *args) 455 481 { 456 - uffd_wp_fork_test_common(args, false); 482 + uffd_wp_fork_test_common(gopts, args, false); 457 483 } 458 484 459 - static void uffd_wp_fork_with_event_test(uffd_test_args_t *args) 485 + static void uffd_wp_fork_with_event_test(uffd_global_test_opts_t *gopts, uffd_test_args_t *args) 460 486 { 461 - uffd_wp_fork_test_common(args, true); 487 + uffd_wp_fork_test_common(gopts, args, true); 462 488 } 463 489 464 - static void uffd_wp_fork_pin_test_common(uffd_test_args_t *args, 490 + static void uffd_wp_fork_pin_test_common(uffd_global_test_opts_t *gopts, 491 + uffd_test_args_t *args, 465 492 bool with_event) 466 493 { 467 494 int pagemap_fd; 468 495 pin_args pin_args = {}; 469 496 470 - if (uffd_register(uffd, area_dst, page_size, false, true, false)) 497 + if (uffd_register(gopts->uffd, gopts->area_dst, gopts->page_size, false, true, false)) 471 498 err("register failed"); 472 499 473 500 pagemap_fd = pagemap_open(); 474 501 475 502 /* Touch the page */ 476 - *area_dst = 1; 477 - wp_range(uffd, (uint64_t)area_dst, page_size, true); 503 + *gopts->area_dst = 1; 504 + wp_range(gopts->uffd, (uint64_t)gopts->area_dst, gopts->page_size, true); 478 505 479 506 /* 480 507 * 1. First pin, then fork(). This tests fork() special path when 481 508 * doing early CoW if the page is private. 482 509 */ 483 - if (pin_pages(&pin_args, area_dst, page_size)) { 510 + if (pin_pages(&pin_args, gopts->area_dst, gopts->page_size)) { 484 511 uffd_test_skip("Possibly CONFIG_GUP_TEST missing " 485 512 "or unprivileged"); 486 513 close(pagemap_fd); 487 - uffd_unregister(uffd, area_dst, page_size); 514 + uffd_unregister(gopts->uffd, gopts->area_dst, gopts->page_size); 488 515 return; 489 516 } 490 517 491 - if (pagemap_test_fork(uffd, with_event, false)) { 518 + if (pagemap_test_fork(gopts, with_event, false)) { 492 519 uffd_test_fail("Detected %s uffd-wp bit in early CoW of fork()", 493 520 with_event ? "missing" : "stall"); 494 521 unpin_pages(&pin_args); ··· 502 527 * 2. First fork(), then pin (in the child, where test_pin==true). 503 528 * This tests COR, aka, page unsharing on private memories. 504 529 */ 505 - if (pagemap_test_fork(uffd, with_event, true)) { 530 + if (pagemap_test_fork(gopts, with_event, true)) { 506 531 uffd_test_fail("Detected %s uffd-wp bit when RO pin", 507 532 with_event ? "missing" : "stall"); 508 533 goto out; 509 534 } 510 535 uffd_test_pass(); 511 536 out: 512 - if (uffd_unregister(uffd, area_dst, page_size)) 537 + if (uffd_unregister(gopts->uffd, gopts->area_dst, gopts->page_size)) 513 538 err("register failed"); 514 539 close(pagemap_fd); 515 540 } 516 541 517 - static void uffd_wp_fork_pin_test(uffd_test_args_t *args) 542 + static void uffd_wp_fork_pin_test(uffd_global_test_opts_t *gopts, uffd_test_args_t *args) 518 543 { 519 - uffd_wp_fork_pin_test_common(args, false); 544 + uffd_wp_fork_pin_test_common(gopts, args, false); 520 545 } 521 546 522 - static void uffd_wp_fork_pin_with_event_test(uffd_test_args_t *args) 547 + static void uffd_wp_fork_pin_with_event_test(uffd_global_test_opts_t *gopts, uffd_test_args_t *args) 523 548 { 524 - uffd_wp_fork_pin_test_common(args, true); 549 + uffd_wp_fork_pin_test_common(gopts, args, true); 525 550 } 526 551 527 - static void check_memory_contents(char *p) 552 + static void check_memory_contents(uffd_global_test_opts_t *gopts, char *p) 528 553 { 529 554 unsigned long i, j; 530 555 uint8_t expected_byte; 531 556 532 - for (i = 0; i < nr_pages; ++i) { 557 + for (i = 0; i < gopts->nr_pages; ++i) { 533 558 expected_byte = ~((uint8_t)(i % ((uint8_t)-1))); 534 - for (j = 0; j < page_size; j++) { 535 - uint8_t v = *(uint8_t *)(p + (i * page_size) + j); 559 + for (j = 0; j < gopts->page_size; j++) { 560 + uint8_t v = *(uint8_t *)(p + (i * gopts->page_size) + j); 536 561 if (v != expected_byte) 537 562 err("unexpected page contents"); 538 563 } 539 564 } 540 565 } 541 566 542 - static void uffd_minor_test_common(bool test_collapse, bool test_wp) 567 + static void uffd_minor_test_common(uffd_global_test_opts_t *gopts, bool test_collapse, bool test_wp) 543 568 { 544 569 unsigned long p; 545 570 pthread_t uffd_mon; 546 571 char c; 547 572 struct uffd_args args = { 0 }; 573 + args.gopts = gopts; 548 574 549 575 /* 550 576 * NOTE: MADV_COLLAPSE is not yet compatible with WP, so testing ··· 553 577 */ 554 578 assert(!(test_collapse && test_wp)); 555 579 556 - if (uffd_register(uffd, area_dst_alias, nr_pages * page_size, 580 + if (uffd_register(gopts->uffd, gopts->area_dst_alias, gopts->nr_pages * gopts->page_size, 557 581 /* NOTE! MADV_COLLAPSE may not work with uffd-wp */ 558 582 false, test_wp, true)) 559 583 err("register failure"); ··· 562 586 * After registering with UFFD, populate the non-UFFD-registered side of 563 587 * the shared mapping. This should *not* trigger any UFFD minor faults. 564 588 */ 565 - for (p = 0; p < nr_pages; ++p) 566 - memset(area_dst + (p * page_size), p % ((uint8_t)-1), 567 - page_size); 589 + for (p = 0; p < gopts->nr_pages; ++p) 590 + memset(gopts->area_dst + (p * gopts->page_size), p % ((uint8_t)-1), 591 + gopts->page_size); 568 592 569 593 args.apply_wp = test_wp; 570 594 if (pthread_create(&uffd_mon, NULL, uffd_poll_thread, &args)) ··· 576 600 * fault. uffd_poll_thread will resolve the fault by bit-flipping the 577 601 * page's contents, and then issuing a CONTINUE ioctl. 578 602 */ 579 - check_memory_contents(area_dst_alias); 603 + check_memory_contents(gopts, gopts->area_dst_alias); 580 604 581 - if (write(pipefd[1], &c, sizeof(c)) != sizeof(c)) 605 + if (write(gopts->pipefd[1], &c, sizeof(c)) != sizeof(c)) 582 606 err("pipe write"); 583 607 if (pthread_join(uffd_mon, NULL)) 584 608 err("join() failed"); 585 609 586 610 if (test_collapse) { 587 - if (madvise(area_dst_alias, nr_pages * page_size, 611 + if (madvise(gopts->area_dst_alias, gopts->nr_pages * gopts->page_size, 588 612 MADV_COLLAPSE)) { 589 613 /* It's fine to fail for this one... */ 590 614 uffd_test_skip("MADV_COLLAPSE failed"); 591 615 return; 592 616 } 593 617 594 - uffd_test_ops->check_pmd_mapping(area_dst, 595 - nr_pages * page_size / 618 + uffd_test_ops->check_pmd_mapping(gopts, 619 + gopts->area_dst, 620 + gopts->nr_pages * gopts->page_size / 596 621 read_pmd_pagesize()); 597 622 /* 598 623 * This won't cause uffd-fault - it purely just makes sure there 599 624 * was no corruption. 600 625 */ 601 - check_memory_contents(area_dst_alias); 626 + check_memory_contents(gopts, gopts->area_dst_alias); 602 627 } 603 628 604 - if (args.missing_faults != 0 || args.minor_faults != nr_pages) 629 + if (args.missing_faults != 0 || args.minor_faults != gopts->nr_pages) 605 630 uffd_test_fail("stats check error"); 606 631 else 607 632 uffd_test_pass(); 608 633 } 609 634 610 - void uffd_minor_test(uffd_test_args_t *args) 635 + void uffd_minor_test(uffd_global_test_opts_t *gopts, uffd_test_args_t *args) 611 636 { 612 - uffd_minor_test_common(false, false); 637 + uffd_minor_test_common(gopts, false, false); 613 638 } 614 639 615 - void uffd_minor_wp_test(uffd_test_args_t *args) 640 + void uffd_minor_wp_test(uffd_global_test_opts_t *gopts, uffd_test_args_t *args) 616 641 { 617 - uffd_minor_test_common(false, true); 642 + uffd_minor_test_common(gopts, false, true); 618 643 } 619 644 620 - void uffd_minor_collapse_test(uffd_test_args_t *args) 645 + void uffd_minor_collapse_test(uffd_global_test_opts_t *gopts, uffd_test_args_t *args) 621 646 { 622 - uffd_minor_test_common(true, false); 647 + uffd_minor_test_common(gopts, true, false); 623 648 } 624 649 625 650 static sigjmp_buf jbuf, *sigbuf; ··· 655 678 * This also tests UFFD_FEATURE_EVENT_FORK event along with the signal 656 679 * feature. Using monitor thread, verify no userfault events are generated. 657 680 */ 658 - static int faulting_process(int signal_test, bool wp) 681 + static int faulting_process(uffd_global_test_opts_t *gopts, int signal_test, bool wp) 659 682 { 660 683 unsigned long nr, i; 661 684 unsigned long long count; ··· 664 687 struct sigaction act; 665 688 volatile unsigned long signalled = 0; 666 689 667 - split_nr_pages = (nr_pages + 1) / 2; 690 + split_nr_pages = (gopts->nr_pages + 1) / 2; 668 691 669 692 if (signal_test) { 670 693 sigbuf = &jbuf; ··· 678 701 679 702 for (nr = 0; nr < split_nr_pages; nr++) { 680 703 volatile int steps = 1; 681 - unsigned long offset = nr * page_size; 704 + unsigned long offset = nr * gopts->page_size; 682 705 683 706 if (signal_test) { 684 707 if (sigsetjmp(*sigbuf, 1) != 0) { ··· 690 713 if (steps == 1) { 691 714 /* This is a MISSING request */ 692 715 steps++; 693 - if (copy_page(uffd, offset, wp)) 716 + if (copy_page(gopts, offset, wp)) 694 717 signalled++; 695 718 } else { 696 719 /* This is a WP request */ 697 720 assert(steps == 2); 698 - wp_range(uffd, 699 - (__u64)area_dst + 721 + wp_range(gopts->uffd, 722 + (__u64)gopts->area_dst + 700 723 offset, 701 - page_size, false); 724 + gopts->page_size, false); 702 725 } 703 726 } else { 704 727 signalled++; ··· 707 730 } 708 731 } 709 732 710 - count = *area_count(area_dst, nr); 711 - if (count != count_verify[nr]) 733 + count = *area_count(gopts->area_dst, nr, gopts); 734 + if (count != gopts->count_verify[nr]) 712 735 err("nr %lu memory corruption %llu %llu\n", 713 - nr, count, count_verify[nr]); 736 + nr, count, gopts->count_verify[nr]); 714 737 /* 715 738 * Trigger write protection if there is by writing 716 739 * the same value back. 717 740 */ 718 - *area_count(area_dst, nr) = count; 741 + *area_count(gopts->area_dst, nr, gopts) = count; 719 742 } 720 743 721 744 if (signal_test) 722 745 return signalled != split_nr_pages; 723 746 724 - area_dst = mremap(area_dst, nr_pages * page_size, nr_pages * page_size, 725 - MREMAP_MAYMOVE | MREMAP_FIXED, area_src); 726 - if (area_dst == MAP_FAILED) 747 + gopts->area_dst = mremap(gopts->area_dst, gopts->nr_pages * gopts->page_size, 748 + gopts->nr_pages * gopts->page_size, 749 + MREMAP_MAYMOVE | MREMAP_FIXED, 750 + gopts->area_src); 751 + if (gopts->area_dst == MAP_FAILED) 727 752 err("mremap"); 728 753 /* Reset area_src since we just clobbered it */ 729 - area_src = NULL; 754 + gopts->area_src = NULL; 730 755 731 - for (; nr < nr_pages; nr++) { 732 - count = *area_count(area_dst, nr); 733 - if (count != count_verify[nr]) { 756 + for (; nr < gopts->nr_pages; nr++) { 757 + count = *area_count(gopts->area_dst, nr, gopts); 758 + if (count != gopts->count_verify[nr]) { 734 759 err("nr %lu memory corruption %llu %llu\n", 735 - nr, count, count_verify[nr]); 760 + nr, count, gopts->count_verify[nr]); 736 761 } 737 762 /* 738 763 * Trigger write protection if there is by writing 739 764 * the same value back. 740 765 */ 741 - *area_count(area_dst, nr) = count; 766 + *area_count(gopts->area_dst, nr, gopts) = count; 742 767 } 743 768 744 - uffd_test_ops->release_pages(area_dst); 769 + uffd_test_ops->release_pages(gopts, gopts->area_dst); 745 770 746 - for (nr = 0; nr < nr_pages; nr++) 747 - for (i = 0; i < page_size; i++) 748 - if (*(area_dst + nr * page_size + i) != 0) 771 + for (nr = 0; nr < gopts->nr_pages; nr++) 772 + for (i = 0; i < gopts->page_size; i++) 773 + if (*(gopts->area_dst + nr * gopts->page_size + i) != 0) 749 774 err("page %lu offset %lu is not zero", nr, i); 750 775 751 776 return 0; 752 777 } 753 778 754 - static void uffd_sigbus_test_common(bool wp) 779 + static void uffd_sigbus_test_common(uffd_global_test_opts_t *gopts, bool wp) 755 780 { 756 781 unsigned long userfaults; 757 782 pthread_t uffd_mon; ··· 761 782 int err; 762 783 char c; 763 784 struct uffd_args args = { 0 }; 785 + args.gopts = gopts; 764 786 765 - ready_for_fork = false; 787 + gopts->ready_for_fork = false; 766 788 767 - fcntl(uffd, F_SETFL, uffd_flags | O_NONBLOCK); 789 + fcntl(gopts->uffd, F_SETFL, gopts->uffd_flags | O_NONBLOCK); 768 790 769 - if (uffd_register(uffd, area_dst, nr_pages * page_size, 791 + if (uffd_register(gopts->uffd, gopts->area_dst, gopts->nr_pages * gopts->page_size, 770 792 true, wp, false)) 771 793 err("register failure"); 772 794 773 - if (faulting_process(1, wp)) 795 + if (faulting_process(gopts, 1, wp)) 774 796 err("faulting process failed"); 775 797 776 - uffd_test_ops->release_pages(area_dst); 798 + uffd_test_ops->release_pages(gopts, gopts->area_dst); 777 799 778 800 args.apply_wp = wp; 779 801 if (pthread_create(&uffd_mon, NULL, uffd_poll_thread, &args)) 780 802 err("uffd_poll_thread create"); 781 803 782 - while (!ready_for_fork) 804 + while (!gopts->ready_for_fork) 783 805 ; /* Wait for the poll_thread to start executing before forking */ 784 806 785 807 pid = fork(); ··· 788 808 err("fork"); 789 809 790 810 if (!pid) 791 - exit(faulting_process(2, wp)); 811 + exit(faulting_process(gopts, 2, wp)); 792 812 793 813 waitpid(pid, &err, 0); 794 814 if (err) 795 815 err("faulting process failed"); 796 - if (write(pipefd[1], &c, sizeof(c)) != sizeof(c)) 816 + if (write(gopts->pipefd[1], &c, sizeof(c)) != sizeof(c)) 797 817 err("pipe write"); 798 818 if (pthread_join(uffd_mon, (void **)&userfaults)) 799 819 err("pthread_join()"); ··· 804 824 uffd_test_pass(); 805 825 } 806 826 807 - static void uffd_sigbus_test(uffd_test_args_t *args) 827 + static void uffd_sigbus_test(uffd_global_test_opts_t *gopts, uffd_test_args_t *args) 808 828 { 809 - uffd_sigbus_test_common(false); 829 + uffd_sigbus_test_common(gopts, false); 810 830 } 811 831 812 - static void uffd_sigbus_wp_test(uffd_test_args_t *args) 832 + static void uffd_sigbus_wp_test(uffd_global_test_opts_t *gopts, uffd_test_args_t *args) 813 833 { 814 - uffd_sigbus_test_common(true); 834 + uffd_sigbus_test_common(gopts, true); 815 835 } 816 836 817 - static void uffd_events_test_common(bool wp) 837 + static void uffd_events_test_common(uffd_global_test_opts_t *gopts, bool wp) 818 838 { 819 839 pthread_t uffd_mon; 820 840 pid_t pid; 821 841 int err; 822 842 char c; 823 843 struct uffd_args args = { 0 }; 844 + args.gopts = gopts; 824 845 825 - ready_for_fork = false; 846 + gopts->ready_for_fork = false; 826 847 827 - fcntl(uffd, F_SETFL, uffd_flags | O_NONBLOCK); 828 - if (uffd_register(uffd, area_dst, nr_pages * page_size, 848 + fcntl(gopts->uffd, F_SETFL, gopts->uffd_flags | O_NONBLOCK); 849 + if (uffd_register(gopts->uffd, gopts->area_dst, gopts->nr_pages * gopts->page_size, 829 850 true, wp, false)) 830 851 err("register failure"); 831 852 ··· 834 853 if (pthread_create(&uffd_mon, NULL, uffd_poll_thread, &args)) 835 854 err("uffd_poll_thread create"); 836 855 837 - while (!ready_for_fork) 856 + while (!gopts->ready_for_fork) 838 857 ; /* Wait for the poll_thread to start executing before forking */ 839 858 840 859 pid = fork(); ··· 842 861 err("fork"); 843 862 844 863 if (!pid) 845 - exit(faulting_process(0, wp)); 864 + exit(faulting_process(gopts, 0, wp)); 846 865 847 866 waitpid(pid, &err, 0); 848 867 if (err) 849 868 err("faulting process failed"); 850 - if (write(pipefd[1], &c, sizeof(c)) != sizeof(c)) 869 + if (write(gopts->pipefd[1], &c, sizeof(c)) != sizeof(c)) 851 870 err("pipe write"); 852 871 if (pthread_join(uffd_mon, NULL)) 853 872 err("pthread_join()"); 854 873 855 - if (args.missing_faults != nr_pages) 874 + if (args.missing_faults != gopts->nr_pages) 856 875 uffd_test_fail("Fault counts wrong"); 857 876 else 858 877 uffd_test_pass(); 859 878 } 860 879 861 - static void uffd_events_test(uffd_test_args_t *args) 880 + static void uffd_events_test(uffd_global_test_opts_t *gopts, uffd_test_args_t *args) 862 881 { 863 - uffd_events_test_common(false); 882 + uffd_events_test_common(gopts, false); 864 883 } 865 884 866 - static void uffd_events_wp_test(uffd_test_args_t *args) 885 + static void uffd_events_wp_test(uffd_global_test_opts_t *gopts, uffd_test_args_t *args) 867 886 { 868 - uffd_events_test_common(true); 887 + uffd_events_test_common(gopts, true); 869 888 } 870 889 871 - static void retry_uffdio_zeropage(int ufd, 890 + static void retry_uffdio_zeropage(uffd_global_test_opts_t *gopts, 872 891 struct uffdio_zeropage *uffdio_zeropage) 873 892 { 874 - uffd_test_ops->alias_mapping(&uffdio_zeropage->range.start, 893 + uffd_test_ops->alias_mapping(gopts, &uffdio_zeropage->range.start, 875 894 uffdio_zeropage->range.len, 876 895 0); 877 - if (ioctl(ufd, UFFDIO_ZEROPAGE, uffdio_zeropage)) { 896 + if (ioctl(gopts->uffd, UFFDIO_ZEROPAGE, uffdio_zeropage)) { 878 897 if (uffdio_zeropage->zeropage != -EEXIST) 879 898 err("UFFDIO_ZEROPAGE error: %"PRId64, 880 899 (int64_t)uffdio_zeropage->zeropage); ··· 884 903 } 885 904 } 886 905 887 - static bool do_uffdio_zeropage(int ufd, bool has_zeropage) 906 + static bool do_uffdio_zeropage(uffd_global_test_opts_t *gopts, bool has_zeropage) 888 907 { 889 908 struct uffdio_zeropage uffdio_zeropage = { 0 }; 890 909 int ret; 891 910 __s64 res; 892 911 893 - uffdio_zeropage.range.start = (unsigned long) area_dst; 894 - uffdio_zeropage.range.len = page_size; 912 + uffdio_zeropage.range.start = (unsigned long) gopts->area_dst; 913 + uffdio_zeropage.range.len = gopts->page_size; 895 914 uffdio_zeropage.mode = 0; 896 - ret = ioctl(ufd, UFFDIO_ZEROPAGE, &uffdio_zeropage); 915 + ret = ioctl(gopts->uffd, UFFDIO_ZEROPAGE, &uffdio_zeropage); 897 916 res = uffdio_zeropage.zeropage; 898 917 if (ret) { 899 918 /* real retval in ufdio_zeropage.zeropage */ ··· 902 921 else if (res != -EINVAL) 903 922 err("UFFDIO_ZEROPAGE not -EINVAL"); 904 923 } else if (has_zeropage) { 905 - if (res != page_size) 924 + if (res != gopts->page_size) 906 925 err("UFFDIO_ZEROPAGE unexpected size"); 907 926 else 908 - retry_uffdio_zeropage(ufd, &uffdio_zeropage); 927 + retry_uffdio_zeropage(gopts, &uffdio_zeropage); 909 928 return true; 910 929 } else 911 930 err("UFFDIO_ZEROPAGE succeeded"); ··· 931 950 } 932 951 933 952 /* exercise UFFDIO_ZEROPAGE */ 934 - static void uffd_zeropage_test(uffd_test_args_t *args) 953 + static void uffd_zeropage_test(uffd_global_test_opts_t *gopts, uffd_test_args_t *args) 935 954 { 936 955 bool has_zeropage; 937 956 int i; 938 957 939 - has_zeropage = uffd_register_detect_zeropage(uffd, area_dst, page_size); 940 - if (area_dst_alias) 958 + has_zeropage = uffd_register_detect_zeropage(gopts->uffd, 959 + gopts->area_dst, 960 + gopts->page_size); 961 + if (gopts->area_dst_alias) 941 962 /* Ignore the retval; we already have it */ 942 - uffd_register_detect_zeropage(uffd, area_dst_alias, page_size); 963 + uffd_register_detect_zeropage(gopts->uffd, gopts->area_dst_alias, gopts->page_size); 943 964 944 - if (do_uffdio_zeropage(uffd, has_zeropage)) 945 - for (i = 0; i < page_size; i++) 946 - if (area_dst[i] != 0) 965 + if (do_uffdio_zeropage(gopts, has_zeropage)) 966 + for (i = 0; i < gopts->page_size; i++) 967 + if (gopts->area_dst[i] != 0) 947 968 err("data non-zero at offset %d\n", i); 948 969 949 - if (uffd_unregister(uffd, area_dst, page_size)) 970 + if (uffd_unregister(gopts->uffd, gopts->area_dst, gopts->page_size)) 950 971 err("unregister"); 951 972 952 - if (area_dst_alias && uffd_unregister(uffd, area_dst_alias, page_size)) 973 + if (gopts->area_dst_alias && uffd_unregister(gopts->uffd, 974 + gopts->area_dst_alias, 975 + gopts->page_size)) 953 976 err("unregister"); 954 977 955 978 uffd_test_pass(); ··· 972 987 err("registered area doesn't support COPY and POISON ioctls"); 973 988 } 974 989 975 - static void do_uffdio_poison(int uffd, unsigned long offset) 990 + static void do_uffdio_poison(uffd_global_test_opts_t *gopts, unsigned long offset) 976 991 { 977 992 struct uffdio_poison uffdio_poison = { 0 }; 978 993 int ret; 979 994 __s64 res; 980 995 981 - uffdio_poison.range.start = (unsigned long) area_dst + offset; 982 - uffdio_poison.range.len = page_size; 996 + uffdio_poison.range.start = (unsigned long) gopts->area_dst + offset; 997 + uffdio_poison.range.len = gopts->page_size; 983 998 uffdio_poison.mode = 0; 984 - ret = ioctl(uffd, UFFDIO_POISON, &uffdio_poison); 999 + ret = ioctl(gopts->uffd, UFFDIO_POISON, &uffdio_poison); 985 1000 res = uffdio_poison.updated; 986 1001 987 1002 if (ret) 988 1003 err("UFFDIO_POISON error: %"PRId64, (int64_t)res); 989 - else if (res != page_size) 1004 + else if (res != gopts->page_size) 990 1005 err("UFFDIO_POISON unexpected size: %"PRId64, (int64_t)res); 991 1006 } 992 1007 993 - static void uffd_poison_handle_fault( 994 - struct uffd_msg *msg, struct uffd_args *args) 1008 + static void uffd_poison_handle_fault(uffd_global_test_opts_t *gopts, 1009 + struct uffd_msg *msg, 1010 + struct uffd_args *args) 995 1011 { 996 1012 unsigned long offset; 997 1013 ··· 1003 1017 (UFFD_PAGEFAULT_FLAG_WP | UFFD_PAGEFAULT_FLAG_MINOR)) 1004 1018 err("unexpected fault type %llu", msg->arg.pagefault.flags); 1005 1019 1006 - offset = (char *)(unsigned long)msg->arg.pagefault.address - area_dst; 1007 - offset &= ~(page_size-1); 1020 + offset = (char *)(unsigned long)msg->arg.pagefault.address - gopts->area_dst; 1021 + offset &= ~(gopts->page_size-1); 1008 1022 1009 1023 /* Odd pages -> copy zeroed page; even pages -> poison. */ 1010 - if (offset & page_size) 1011 - copy_page(uffd, offset, false); 1024 + if (offset & gopts->page_size) 1025 + copy_page(gopts, offset, false); 1012 1026 else 1013 - do_uffdio_poison(uffd, offset); 1027 + do_uffdio_poison(gopts, offset); 1014 1028 } 1015 1029 1016 1030 /* Make sure to cover odd/even, and minimum duplications */ 1017 1031 #define UFFD_POISON_TEST_NPAGES 4 1018 1032 1019 - static void uffd_poison_test(uffd_test_args_t *targs) 1033 + static void uffd_poison_test(uffd_global_test_opts_t *gopts, uffd_test_args_t *targs) 1020 1034 { 1021 1035 pthread_t uffd_mon; 1022 1036 char c; ··· 1025 1039 unsigned long nr_sigbus = 0; 1026 1040 unsigned long nr, poison_pages = UFFD_POISON_TEST_NPAGES; 1027 1041 1028 - if (nr_pages < poison_pages) { 1029 - uffd_test_skip("Too few pages for POISON test"); 1042 + if (gopts->nr_pages < poison_pages) { 1043 + uffd_test_skip("Too less pages for POISON test"); 1030 1044 return; 1031 1045 } 1032 1046 1033 - fcntl(uffd, F_SETFL, uffd_flags | O_NONBLOCK); 1047 + args.gopts = gopts; 1034 1048 1035 - uffd_register_poison(uffd, area_dst, poison_pages * page_size); 1036 - memset(area_src, 0, poison_pages * page_size); 1049 + fcntl(gopts->uffd, F_SETFL, gopts->uffd_flags | O_NONBLOCK); 1050 + 1051 + uffd_register_poison(gopts->uffd, gopts->area_dst, poison_pages * gopts->page_size); 1052 + memset(gopts->area_src, 0, poison_pages * gopts->page_size); 1037 1053 1038 1054 args.handle_fault = uffd_poison_handle_fault; 1039 1055 if (pthread_create(&uffd_mon, NULL, uffd_poll_thread, &args)) ··· 1048 1060 err("sigaction"); 1049 1061 1050 1062 for (nr = 0; nr < poison_pages; ++nr) { 1051 - unsigned long offset = nr * page_size; 1052 - const char *bytes = (const char *) area_dst + offset; 1063 + unsigned long offset = nr * gopts->page_size; 1064 + const char *bytes = (const char *) gopts->area_dst + offset; 1053 1065 const char *i; 1054 1066 1055 1067 if (sigsetjmp(*sigbuf, 1)) { ··· 1062 1074 continue; 1063 1075 } 1064 1076 1065 - for (i = bytes; i < bytes + page_size; ++i) { 1077 + for (i = bytes; i < bytes + gopts->page_size; ++i) { 1066 1078 if (*i) 1067 1079 err("nonzero byte in area_dst (%p) at %p: %u", 1068 - area_dst, i, *i); 1080 + gopts->area_dst, i, *i); 1069 1081 } 1070 1082 } 1071 1083 1072 - if (write(pipefd[1], &c, sizeof(c)) != sizeof(c)) 1084 + if (write(gopts->pipefd[1], &c, sizeof(c)) != sizeof(c)) 1073 1085 err("pipe write"); 1074 1086 if (pthread_join(uffd_mon, NULL)) 1075 1087 err("pthread_join()"); ··· 1082 1094 } 1083 1095 1084 1096 static void 1085 - uffd_move_handle_fault_common(struct uffd_msg *msg, struct uffd_args *args, 1097 + uffd_move_handle_fault_common(uffd_global_test_opts_t *gopts, 1098 + struct uffd_msg *msg, 1099 + struct uffd_args *args, 1086 1100 unsigned long len) 1087 1101 { 1088 1102 unsigned long offset; ··· 1096 1106 (UFFD_PAGEFAULT_FLAG_WP | UFFD_PAGEFAULT_FLAG_MINOR | UFFD_PAGEFAULT_FLAG_WRITE)) 1097 1107 err("unexpected fault type %llu", msg->arg.pagefault.flags); 1098 1108 1099 - offset = (char *)(unsigned long)msg->arg.pagefault.address - area_dst; 1109 + offset = (char *)(unsigned long)msg->arg.pagefault.address - gopts->area_dst; 1100 1110 offset &= ~(len-1); 1101 1111 1102 - if (move_page(uffd, offset, len)) 1112 + if (move_page(gopts, offset, len)) 1103 1113 args->missing_faults++; 1104 1114 } 1105 1115 1106 - static void uffd_move_handle_fault(struct uffd_msg *msg, 1116 + static void uffd_move_handle_fault(uffd_global_test_opts_t *gopts, struct uffd_msg *msg, 1107 1117 struct uffd_args *args) 1108 1118 { 1109 - uffd_move_handle_fault_common(msg, args, page_size); 1119 + uffd_move_handle_fault_common(gopts, msg, args, gopts->page_size); 1110 1120 } 1111 1121 1112 - static void uffd_move_pmd_handle_fault(struct uffd_msg *msg, 1122 + static void uffd_move_pmd_handle_fault(uffd_global_test_opts_t *gopts, struct uffd_msg *msg, 1113 1123 struct uffd_args *args) 1114 1124 { 1115 - uffd_move_handle_fault_common(msg, args, read_pmd_pagesize()); 1125 + uffd_move_handle_fault_common(gopts, msg, args, read_pmd_pagesize()); 1116 1126 } 1117 1127 1118 1128 static void 1119 - uffd_move_test_common(uffd_test_args_t *targs, unsigned long chunk_size, 1120 - void (*handle_fault)(struct uffd_msg *msg, struct uffd_args *args)) 1129 + uffd_move_test_common(uffd_global_test_opts_t *gopts, 1130 + uffd_test_args_t *targs, 1131 + unsigned long chunk_size, 1132 + void (*handle_fault)(struct uffd_global_test_opts *gopts, 1133 + struct uffd_msg *msg, struct uffd_args *args) 1134 + ) 1121 1135 { 1122 1136 unsigned long nr; 1123 1137 pthread_t uffd_mon; ··· 1133 1139 unsigned long src_offs = 0; 1134 1140 unsigned long dst_offs = 0; 1135 1141 1142 + args.gopts = gopts; 1143 + 1136 1144 /* Prevent source pages from being mapped more than once */ 1137 - if (madvise(area_src, nr_pages * page_size, MADV_DONTFORK)) 1145 + if (madvise(gopts->area_src, gopts->nr_pages * gopts->page_size, MADV_DONTFORK)) 1138 1146 err("madvise(MADV_DONTFORK) failure"); 1139 1147 1140 - if (uffd_register(uffd, area_dst, nr_pages * page_size, 1148 + if (uffd_register(gopts->uffd, gopts->area_dst, gopts->nr_pages * gopts->page_size, 1141 1149 true, false, false)) 1142 1150 err("register failure"); 1143 1151 ··· 1147 1151 if (pthread_create(&uffd_mon, NULL, uffd_poll_thread, &args)) 1148 1152 err("uffd_poll_thread create"); 1149 1153 1150 - step_size = chunk_size / page_size; 1151 - step_count = nr_pages / step_size; 1154 + step_size = chunk_size / gopts->page_size; 1155 + step_count = gopts->nr_pages / step_size; 1152 1156 1153 - if (chunk_size > page_size) { 1154 - char *aligned_src = ALIGN_UP(area_src, chunk_size); 1155 - char *aligned_dst = ALIGN_UP(area_dst, chunk_size); 1157 + if (chunk_size > gopts->page_size) { 1158 + char *aligned_src = ALIGN_UP(gopts->area_src, chunk_size); 1159 + char *aligned_dst = ALIGN_UP(gopts->area_dst, chunk_size); 1156 1160 1157 - if (aligned_src != area_src || aligned_dst != area_dst) { 1158 - src_offs = (aligned_src - area_src) / page_size; 1159 - dst_offs = (aligned_dst - area_dst) / page_size; 1161 + if (aligned_src != gopts->area_src || aligned_dst != gopts->area_dst) { 1162 + src_offs = (aligned_src - gopts->area_src) / gopts->page_size; 1163 + dst_offs = (aligned_dst - gopts->area_dst) / gopts->page_size; 1160 1164 step_count--; 1161 1165 } 1162 - orig_area_src = area_src; 1163 - orig_area_dst = area_dst; 1164 - area_src = aligned_src; 1165 - area_dst = aligned_dst; 1166 + orig_area_src = gopts->area_src; 1167 + orig_area_dst = gopts->area_dst; 1168 + gopts->area_src = aligned_src; 1169 + gopts->area_dst = aligned_dst; 1166 1170 } 1167 1171 1168 1172 /* ··· 1176 1180 1177 1181 /* Check area_src content */ 1178 1182 for (i = 0; i < step_size; i++) { 1179 - count = *area_count(area_src, nr + i); 1180 - if (count != count_verify[src_offs + nr + i]) 1183 + count = *area_count(gopts->area_src, nr + i, gopts); 1184 + if (count != gopts->count_verify[src_offs + nr + i]) 1181 1185 err("nr %lu source memory invalid %llu %llu\n", 1182 - nr + i, count, count_verify[src_offs + nr + i]); 1186 + nr + i, count, gopts->count_verify[src_offs + nr + i]); 1183 1187 } 1184 1188 1185 1189 /* Faulting into area_dst should move the page or the huge page */ 1186 1190 for (i = 0; i < step_size; i++) { 1187 - count = *area_count(area_dst, nr + i); 1188 - if (count != count_verify[dst_offs + nr + i]) 1191 + count = *area_count(gopts->area_dst, nr + i, gopts); 1192 + if (count != gopts->count_verify[dst_offs + nr + i]) 1189 1193 err("nr %lu memory corruption %llu %llu\n", 1190 - nr, count, count_verify[dst_offs + nr + i]); 1194 + nr, count, gopts->count_verify[dst_offs + nr + i]); 1191 1195 } 1192 1196 1193 1197 /* Re-check area_src content which should be empty */ 1194 1198 for (i = 0; i < step_size; i++) { 1195 - count = *area_count(area_src, nr + i); 1199 + count = *area_count(gopts->area_src, nr + i, gopts); 1196 1200 if (count != 0) 1197 1201 err("nr %lu move failed %llu %llu\n", 1198 - nr, count, count_verify[src_offs + nr + i]); 1202 + nr, count, gopts->count_verify[src_offs + nr + i]); 1199 1203 } 1200 1204 } 1201 - if (chunk_size > page_size) { 1202 - area_src = orig_area_src; 1203 - area_dst = orig_area_dst; 1205 + if (chunk_size > gopts->page_size) { 1206 + gopts->area_src = orig_area_src; 1207 + gopts->area_dst = orig_area_dst; 1204 1208 } 1205 1209 1206 - if (write(pipefd[1], &c, sizeof(c)) != sizeof(c)) 1210 + if (write(gopts->pipefd[1], &c, sizeof(c)) != sizeof(c)) 1207 1211 err("pipe write"); 1208 1212 if (pthread_join(uffd_mon, NULL)) 1209 1213 err("join() failed"); ··· 1214 1218 uffd_test_pass(); 1215 1219 } 1216 1220 1217 - static void uffd_move_test(uffd_test_args_t *targs) 1221 + static void uffd_move_test(uffd_global_test_opts_t *gopts, uffd_test_args_t *targs) 1218 1222 { 1219 - uffd_move_test_common(targs, page_size, uffd_move_handle_fault); 1223 + uffd_move_test_common(gopts, targs, gopts->page_size, uffd_move_handle_fault); 1220 1224 } 1221 1225 1222 - static void uffd_move_pmd_test(uffd_test_args_t *targs) 1226 + static void uffd_move_pmd_test(uffd_global_test_opts_t *gopts, uffd_test_args_t *targs) 1223 1227 { 1224 - if (madvise(area_dst, nr_pages * page_size, MADV_HUGEPAGE)) 1228 + if (madvise(gopts->area_dst, gopts->nr_pages * gopts->page_size, MADV_HUGEPAGE)) 1225 1229 err("madvise(MADV_HUGEPAGE) failure"); 1226 - uffd_move_test_common(targs, read_pmd_pagesize(), 1230 + uffd_move_test_common(gopts, targs, read_pmd_pagesize(), 1227 1231 uffd_move_pmd_handle_fault); 1228 1232 } 1229 1233 1230 - static void uffd_move_pmd_split_test(uffd_test_args_t *targs) 1234 + static void uffd_move_pmd_split_test(uffd_global_test_opts_t *gopts, uffd_test_args_t *targs) 1231 1235 { 1232 - if (madvise(area_dst, nr_pages * page_size, MADV_NOHUGEPAGE)) 1236 + if (madvise(gopts->area_dst, gopts->nr_pages * gopts->page_size, MADV_NOHUGEPAGE)) 1233 1237 err("madvise(MADV_NOHUGEPAGE) failure"); 1234 - uffd_move_test_common(targs, read_pmd_pagesize(), 1238 + uffd_move_test_common(gopts, targs, read_pmd_pagesize(), 1235 1239 uffd_move_pmd_handle_fault); 1236 1240 } 1237 1241 ··· 1291 1295 THR_STATE_UNINTERRUPTIBLE, 1292 1296 } thread_state; 1293 1297 1298 + typedef struct { 1299 + uffd_global_test_opts_t *gopts; 1300 + volatile pid_t *pid; 1301 + } mmap_changing_thread_args; 1302 + 1294 1303 static void sleep_short(void) 1295 1304 { 1296 1305 usleep(1000); ··· 1338 1337 1339 1338 static void *uffd_mmap_changing_thread(void *opaque) 1340 1339 { 1341 - volatile pid_t *pid = opaque; 1340 + mmap_changing_thread_args *args = opaque; 1341 + uffd_global_test_opts_t *gopts = args->gopts; 1342 + volatile pid_t *pid = args->pid; 1342 1343 int ret; 1343 1344 1344 1345 /* Unfortunately, it's only fetch-able from the thread itself.. */ ··· 1348 1345 *pid = syscall(SYS_gettid); 1349 1346 1350 1347 /* Inject an event, this will hang solid until the event read */ 1351 - ret = madvise(area_dst, page_size, MADV_REMOVE); 1348 + ret = madvise(gopts->area_dst, gopts->page_size, MADV_REMOVE); 1352 1349 if (ret) 1353 1350 err("madvise(MADV_REMOVE) failed"); 1354 1351 1355 1352 return NULL; 1356 1353 } 1357 1354 1358 - static void uffd_consume_message(int fd) 1355 + static void uffd_consume_message(uffd_global_test_opts_t *gopts) 1359 1356 { 1360 1357 struct uffd_msg msg = { 0 }; 1361 1358 1362 - while (uffd_read_msg(fd, &msg)); 1359 + while (uffd_read_msg(gopts, &msg)); 1363 1360 } 1364 1361 1365 - static void uffd_mmap_changing_test(uffd_test_args_t *targs) 1362 + static void uffd_mmap_changing_test(uffd_global_test_opts_t *gopts, uffd_test_args_t *targs) 1366 1363 { 1367 1364 /* 1368 1365 * This stores the real PID (which can be different from how tid is ··· 1371 1368 pid_t pid = 0; 1372 1369 pthread_t tid; 1373 1370 int ret; 1371 + mmap_changing_thread_args args = { gopts, &pid }; 1374 1372 1375 - if (uffd_register(uffd, area_dst, nr_pages * page_size, 1373 + if (uffd_register(gopts->uffd, gopts->area_dst, gopts->nr_pages * gopts->page_size, 1376 1374 true, false, false)) 1377 1375 err("uffd_register() failed"); 1378 1376 1379 1377 /* Create a thread to generate the racy event */ 1380 - ret = pthread_create(&tid, NULL, uffd_mmap_changing_thread, &pid); 1378 + ret = pthread_create(&tid, NULL, uffd_mmap_changing_thread, &args); 1381 1379 if (ret) 1382 1380 err("pthread_create() failed"); 1383 1381 ··· 1392 1388 /* Wait until the thread hangs at REMOVE event */ 1393 1389 thread_state_until(pid, THR_STATE_UNINTERRUPTIBLE); 1394 1390 1395 - if (!uffdio_mmap_changing_test_copy(uffd)) 1391 + if (!uffdio_mmap_changing_test_copy(gopts->uffd)) 1396 1392 return; 1397 1393 1398 - if (!uffdio_mmap_changing_test_zeropage(uffd)) 1394 + if (!uffdio_mmap_changing_test_zeropage(gopts->uffd)) 1399 1395 return; 1400 1396 1401 - if (!uffdio_mmap_changing_test_move(uffd)) 1397 + if (!uffdio_mmap_changing_test_move(gopts->uffd)) 1402 1398 return; 1403 1399 1404 - if (!uffdio_mmap_changing_test_poison(uffd)) 1400 + if (!uffdio_mmap_changing_test_poison(gopts->uffd)) 1405 1401 return; 1406 1402 1407 - if (!uffdio_mmap_changing_test_continue(uffd)) 1403 + if (!uffdio_mmap_changing_test_continue(gopts->uffd)) 1408 1404 return; 1409 1405 1410 1406 /* 1411 1407 * All succeeded above! Recycle everything. Start by reading the 1412 1408 * event so as to kick the thread roll again.. 1413 1409 */ 1414 - uffd_consume_message(uffd); 1410 + uffd_consume_message(gopts); 1415 1411 1416 1412 ret = pthread_join(tid, NULL); 1417 1413 assert(ret == 0); ··· 1419 1415 uffd_test_pass(); 1420 1416 } 1421 1417 1422 - static int prevent_hugepages(const char **errmsg) 1418 + static int prevent_hugepages(uffd_global_test_opts_t *gopts, const char **errmsg) 1423 1419 { 1424 1420 /* This should be done before source area is populated */ 1425 - if (madvise(area_src, nr_pages * page_size, MADV_NOHUGEPAGE)) { 1421 + if (madvise(gopts->area_src, gopts->nr_pages * gopts->page_size, MADV_NOHUGEPAGE)) { 1426 1422 /* Ignore only if CONFIG_TRANSPARENT_HUGEPAGE=n */ 1427 1423 if (errno != EINVAL) { 1428 1424 if (errmsg) ··· 1433 1429 return 0; 1434 1430 } 1435 1431 1436 - static int request_hugepages(const char **errmsg) 1432 + static int request_hugepages(uffd_global_test_opts_t *gopts, const char **errmsg) 1437 1433 { 1438 1434 /* This should be done before source area is populated */ 1439 - if (madvise(area_src, nr_pages * page_size, MADV_HUGEPAGE)) { 1435 + if (madvise(gopts->area_src, gopts->nr_pages * gopts->page_size, MADV_HUGEPAGE)) { 1440 1436 if (errmsg) { 1441 1437 *errmsg = (errno == EINVAL) ? 1442 1438 "CONFIG_TRANSPARENT_HUGEPAGE is not set" : ··· 1460 1456 * Note that _UFFDIO_ZEROPAGE is tested separately in the zeropage test. 1461 1457 */ 1462 1458 static void 1463 - do_register_ioctls_test(uffd_test_args_t *args, bool miss, bool wp, bool minor) 1459 + do_register_ioctls_test(uffd_global_test_opts_t *gopts, 1460 + uffd_test_args_t *args, 1461 + bool miss, 1462 + bool wp, 1463 + bool minor) 1464 1464 { 1465 1465 uint64_t ioctls = 0, expected = BIT_ULL(_UFFDIO_WAKE); 1466 1466 mem_type_t *mem_type = args->mem_type; 1467 1467 int ret; 1468 1468 1469 - ret = uffd_register_with_ioctls(uffd, area_dst, page_size, 1469 + ret = uffd_register_with_ioctls(gopts->uffd, gopts->area_dst, gopts->page_size, 1470 1470 miss, wp, minor, &ioctls); 1471 1471 1472 1472 /* ··· 1501 1493 "(miss=%d, wp=%d, minor=%d): expected=0x%"PRIx64", " 1502 1494 "returned=0x%"PRIx64, miss, wp, minor, expected, ioctls); 1503 1495 1504 - if (uffd_unregister(uffd, area_dst, page_size)) 1496 + if (uffd_unregister(gopts->uffd, gopts->area_dst, gopts->page_size)) 1505 1497 err("unregister"); 1506 1498 } 1507 1499 1508 - static void uffd_register_ioctls_test(uffd_test_args_t *args) 1500 + static void uffd_register_ioctls_test(uffd_global_test_opts_t *gopts, uffd_test_args_t *args) 1509 1501 { 1510 1502 int miss, wp, minor; 1511 1503 1512 1504 for (miss = 0; miss <= 1; miss++) 1513 1505 for (wp = 0; wp <= 1; wp++) 1514 1506 for (minor = 0; minor <= 1; minor++) 1515 - do_register_ioctls_test(args, miss, wp, minor); 1507 + do_register_ioctls_test(gopts, args, miss, wp, minor); 1516 1508 1517 1509 uffd_test_pass(); 1518 1510 } ··· 1750 1742 } 1751 1743 for (j = 0; j < n_mems; j++) { 1752 1744 mem_type = &mem_types[j]; 1745 + 1746 + /* Initialize global test options */ 1747 + uffd_global_test_opts_t gopts = { 0 }; 1748 + 1749 + gopts.map_shared = mem_type->shared; 1750 + uffd_test_ops = mem_type->mem_ops; 1751 + uffd_test_case_ops = test->test_case_ops; 1752 + 1753 + if (mem_type->mem_flag & (MEM_HUGETLB_PRIVATE | MEM_HUGETLB)) 1754 + gopts.page_size = default_huge_page_size(); 1755 + else 1756 + gopts.page_size = psize(); 1757 + 1758 + /* Ensure we have at least 2 pages */ 1759 + gopts.nr_pages = MAX(UFFD_TEST_MEM_SIZE, gopts.page_size * 2) 1760 + / gopts.page_size; 1761 + 1762 + gopts.nr_parallel = 1; 1763 + 1764 + /* Initialize test arguments */ 1765 + args.mem_type = mem_type; 1766 + 1753 1767 if (!(test->mem_targets & mem_type->mem_flag)) 1754 1768 continue; 1755 1769 ··· 1786 1756 uffd_test_skip("feature missing"); 1787 1757 continue; 1788 1758 } 1789 - if (uffd_setup_environment(&args, test, mem_type, 1790 - &errmsg)) { 1759 + if (uffd_test_ctx_init(&gopts, test->uffd_feature_required, &errmsg)) { 1791 1760 uffd_test_skip(errmsg); 1792 1761 continue; 1793 1762 } 1794 - test->uffd_fn(&args); 1795 - uffd_test_ctx_clear(); 1763 + test->uffd_fn(&gopts, &args); 1764 + uffd_test_ctx_clear(&gopts); 1796 1765 } 1797 1766 } 1798 1767

+11 -9

tools/testing/selftests/mm/uffd-wp-mremap.c

··· 152 152 return true; 153 153 } 154 154 155 - static void test_one_folio(size_t size, bool private, bool swapout, bool hugetlb) 155 + static void test_one_folio(uffd_global_test_opts_t *gopts, size_t size, bool private, 156 + bool swapout, bool hugetlb) 156 157 { 157 158 struct uffdio_writeprotect wp_prms; 158 159 uint64_t features = 0; ··· 177 176 } 178 177 179 178 /* Register range for uffd-wp. */ 180 - if (userfaultfd_open(&features)) { 179 + if (userfaultfd_open(gopts, &features)) { 181 180 if (errno == ENOENT) 182 181 ksft_test_result_skip("userfaultfd not available\n"); 183 182 else 184 183 ksft_test_result_fail("userfaultfd_open() failed\n"); 185 184 goto out; 186 185 } 187 - if (uffd_register(uffd, mem, size, false, true, false)) { 186 + if (uffd_register(gopts->uffd, mem, size, false, true, false)) { 188 187 ksft_test_result_fail("uffd_register() failed\n"); 189 188 goto out; 190 189 } 191 190 wp_prms.mode = UFFDIO_WRITEPROTECT_MODE_WP; 192 191 wp_prms.range.start = (uintptr_t)mem; 193 192 wp_prms.range.len = size; 194 - if (ioctl(uffd, UFFDIO_WRITEPROTECT, &wp_prms)) { 193 + if (ioctl(gopts->uffd, UFFDIO_WRITEPROTECT, &wp_prms)) { 195 194 ksft_test_result_fail("ioctl(UFFDIO_WRITEPROTECT) failed\n"); 196 195 goto out; 197 196 } ··· 238 237 out: 239 238 if (mem) 240 239 munmap(mem, size); 241 - if (uffd >= 0) { 242 - close(uffd); 243 - uffd = -1; 240 + if (gopts->uffd >= 0) { 241 + close(gopts->uffd); 242 + gopts->uffd = -1; 244 243 } 245 244 } 246 245 ··· 332 331 333 332 int main(int argc, char **argv) 334 333 { 334 + uffd_global_test_opts_t gopts = { 0 }; 335 335 struct thp_settings settings; 336 336 int i, j, plan = 0; 337 337 ··· 364 362 const struct testcase *tc = &testcases[i]; 365 363 366 364 for (j = 0; j < *tc->nr_sizes; j++) 367 - test_one_folio(tc->sizes[j], tc->private, tc->swapout, 368 - tc->hugetlb); 365 + test_one_folio(&gopts, tc->sizes[j], tc->private, 366 + tc->swapout, tc->hugetlb); 369 367 } 370 368 371 369 /* If THP is supported, restore original THP settings. */

Configure Feed

Configure Feed