iommufd: Change the selftest to use iommupt instead of xarray

+39

drivers/iommu/generic_pt/iommu_pt.h

··· 264 264 } 265 265 EXPORT_SYMBOL_NS_GPL(DOMAIN_NS(read_and_clear_dirty), "GENERIC_PT_IOMMU"); 266 266 267 + static inline int __set_dirty(struct pt_range *range, void *arg, 268 + unsigned int level, struct pt_table_p *table) 269 + { 270 + struct pt_state pts = pt_init(range, level, table); 271 + 272 + switch (pt_load_single_entry(&pts)) { 273 + case PT_ENTRY_EMPTY: 274 + return -ENOENT; 275 + case PT_ENTRY_TABLE: 276 + return pt_descend(&pts, arg, __set_dirty); 277 + case PT_ENTRY_OA: 278 + if (!pt_entry_make_write_dirty(&pts)) 279 + return -EAGAIN; 280 + return 0; 281 + } 282 + return -ENOENT; 283 + } 284 + 285 + static int __maybe_unused NS(set_dirty)(struct pt_iommu *iommu_table, 286 + dma_addr_t iova) 287 + { 288 + struct pt_range range; 289 + int ret; 290 + 291 + ret = make_range(common_from_iommu(iommu_table), &range, iova, 1); 292 + if (ret) 293 + return ret; 294 + 295 + /* 296 + * Note: There is no locking here yet, if the test suite races this it 297 + * can crash. It should use RCU locking eventually. 298 + */ 299 + return pt_walk_range(&range, __set_dirty, NULL); 300 + } 301 + 267 302 struct pt_iommu_collect_args { 268 303 struct iommu_pages_list free_list; 269 304 /* Fail if any OAs are within the range */ ··· 992 957 } 993 958 994 959 static const struct pt_iommu_ops NS(ops) = { 960 + #if IS_ENABLED(CONFIG_IOMMUFD_DRIVER) && defined(pt_entry_is_write_dirty) && \ 961 + IS_ENABLED(CONFIG_IOMMUFD_TEST) && defined(pt_entry_make_write_dirty) 962 + .set_dirty = NS(set_dirty), 963 + #endif 995 964 .get_info = NS(get_info), 996 965 .deinit = NS(deinit), 997 966 };

+1

drivers/iommu/iommufd/Kconfig

··· 41 41 depends on DEBUG_KERNEL 42 42 depends on FAULT_INJECTION 43 43 depends on RUNTIME_TESTING_MENU 44 + depends on IOMMU_PT_AMDV1 44 45 select IOMMUFD_DRIVER 45 46 default n 46 47 help

+10 -1

drivers/iommu/iommufd/iommufd_test.h

··· 32 32 }; 33 33 34 34 enum { 35 + MOCK_IOMMUPT_DEFAULT = 0, 36 + MOCK_IOMMUPT_HUGE, 37 + MOCK_IOMMUPT_AMDV1, 38 + }; 39 + 40 + /* These values are true for MOCK_IOMMUPT_DEFAULT */ 41 + enum { 35 42 MOCK_APERTURE_START = 1UL << 24, 36 43 MOCK_APERTURE_LAST = (1UL << 31) - 1, 44 + MOCK_PAGE_SIZE = 2048, 45 + MOCK_HUGE_PAGE_SIZE = 512 * MOCK_PAGE_SIZE, 37 46 }; 38 47 39 48 enum { ··· 61 52 62 53 enum { 63 54 MOCK_FLAGS_DEVICE_NO_DIRTY = 1 << 0, 64 - MOCK_FLAGS_DEVICE_HUGE_IOVA = 1 << 1, 65 55 MOCK_FLAGS_DEVICE_PASID = 1 << 2, 66 56 }; 67 57 ··· 213 205 */ 214 206 struct iommu_hwpt_selftest { 215 207 __u32 iotlb; 208 + __u32 pagetable_type; 216 209 }; 217 210 218 211 /* Should not be equal to any defined value in enum iommu_hwpt_invalidate_data_type */

+177 -261

drivers/iommu/iommufd/selftest.c

··· 12 12 #include <linux/slab.h> 13 13 #include <linux/xarray.h> 14 14 #include <uapi/linux/iommufd.h> 15 + #include <linux/generic_pt/iommu.h> 16 + #include "../iommu-pages.h" 15 17 16 18 #include "../iommu-priv.h" 17 19 #include "io_pagetable.h" ··· 43 41 44 42 enum { 45 43 MOCK_DIRTY_TRACK = 1, 46 - MOCK_IO_PAGE_SIZE = PAGE_SIZE / 2, 47 - MOCK_HUGE_PAGE_SIZE = 512 * MOCK_IO_PAGE_SIZE, 48 - 49 - /* 50 - * Like a real page table alignment requires the low bits of the address 51 - * to be zero. xarray also requires the high bit to be zero, so we store 52 - * the pfns shifted. The upper bits are used for metadata. 53 - */ 54 - MOCK_PFN_MASK = ULONG_MAX / MOCK_IO_PAGE_SIZE, 55 - 56 - _MOCK_PFN_START = MOCK_PFN_MASK + 1, 57 - MOCK_PFN_START_IOVA = _MOCK_PFN_START, 58 - MOCK_PFN_LAST_IOVA = _MOCK_PFN_START, 59 - MOCK_PFN_DIRTY_IOVA = _MOCK_PFN_START << 1, 60 - MOCK_PFN_HUGE_IOVA = _MOCK_PFN_START << 2, 61 44 }; 62 45 63 46 static int mock_dev_enable_iopf(struct device *dev, struct iommu_domain *domain); ··· 111 124 } 112 125 113 126 struct mock_iommu_domain { 127 + union { 128 + struct iommu_domain domain; 129 + struct pt_iommu iommu; 130 + struct pt_iommu_amdv1 amdv1; 131 + }; 114 132 unsigned long flags; 115 - struct iommu_domain domain; 116 - struct xarray pfns; 117 133 }; 134 + PT_IOMMU_CHECK_DOMAIN(struct mock_iommu_domain, iommu, domain); 135 + PT_IOMMU_CHECK_DOMAIN(struct mock_iommu_domain, amdv1.iommu, domain); 118 136 119 137 static inline struct mock_iommu_domain * 120 138 to_mock_domain(struct iommu_domain *domain) ··· 336 344 return 0; 337 345 } 338 346 339 - static bool mock_test_and_clear_dirty(struct mock_iommu_domain *mock, 340 - unsigned long iova, size_t page_size, 341 - unsigned long flags) 342 - { 343 - unsigned long cur, end = iova + page_size - 1; 344 - bool dirty = false; 345 - void *ent, *old; 346 - 347 - for (cur = iova; cur < end; cur += MOCK_IO_PAGE_SIZE) { 348 - ent = xa_load(&mock->pfns, cur / MOCK_IO_PAGE_SIZE); 349 - if (!ent || !(xa_to_value(ent) & MOCK_PFN_DIRTY_IOVA)) 350 - continue; 351 - 352 - dirty = true; 353 - /* Clear dirty */ 354 - if (!(flags & IOMMU_DIRTY_NO_CLEAR)) { 355 - unsigned long val; 356 - 357 - val = xa_to_value(ent) & ~MOCK_PFN_DIRTY_IOVA; 358 - old = xa_store(&mock->pfns, cur / MOCK_IO_PAGE_SIZE, 359 - xa_mk_value(val), GFP_KERNEL); 360 - WARN_ON_ONCE(ent != old); 361 - } 362 - } 363 - 364 - return dirty; 365 - } 366 - 367 - static int mock_domain_read_and_clear_dirty(struct iommu_domain *domain, 368 - unsigned long iova, size_t size, 369 - unsigned long flags, 370 - struct iommu_dirty_bitmap *dirty) 371 - { 372 - struct mock_iommu_domain *mock = to_mock_domain(domain); 373 - unsigned long end = iova + size; 374 - void *ent; 375 - 376 - if (!(mock->flags & MOCK_DIRTY_TRACK) && dirty->bitmap) 377 - return -EINVAL; 378 - 379 - do { 380 - unsigned long pgsize = MOCK_IO_PAGE_SIZE; 381 - unsigned long head; 382 - 383 - ent = xa_load(&mock->pfns, iova / MOCK_IO_PAGE_SIZE); 384 - if (!ent) { 385 - iova += pgsize; 386 - continue; 387 - } 388 - 389 - if (xa_to_value(ent) & MOCK_PFN_HUGE_IOVA) 390 - pgsize = MOCK_HUGE_PAGE_SIZE; 391 - head = iova & ~(pgsize - 1); 392 - 393 - /* Clear dirty */ 394 - if (mock_test_and_clear_dirty(mock, head, pgsize, flags)) 395 - iommu_dirty_bitmap_record(dirty, iova, pgsize); 396 - iova += pgsize; 397 - } while (iova < end); 398 - 399 - return 0; 400 - } 401 - 402 - static const struct iommu_dirty_ops dirty_ops = { 403 - .set_dirty_tracking = mock_domain_set_dirty_tracking, 404 - .read_and_clear_dirty = mock_domain_read_and_clear_dirty, 405 - }; 406 - 407 347 static struct mock_iommu_domain_nested * 408 348 __mock_domain_alloc_nested(const struct iommu_user_data *user_data) 409 349 { ··· 370 446 371 447 if (flags & ~IOMMU_HWPT_ALLOC_PASID) 372 448 return ERR_PTR(-EOPNOTSUPP); 373 - if (!parent || parent->ops != mock_ops.default_domain_ops) 449 + if (!parent || !(parent->type & __IOMMU_DOMAIN_PAGING)) 374 450 return ERR_PTR(-EINVAL); 375 451 376 452 mock_parent = to_mock_domain(parent); ··· 383 459 return &mock_nested->domain; 384 460 } 385 461 462 + static void mock_domain_free(struct iommu_domain *domain) 463 + { 464 + struct mock_iommu_domain *mock = to_mock_domain(domain); 465 + 466 + pt_iommu_deinit(&mock->iommu); 467 + kfree(mock); 468 + } 469 + 470 + static void mock_iotlb_sync(struct iommu_domain *domain, 471 + struct iommu_iotlb_gather *gather) 472 + { 473 + iommu_put_pages_list(&gather->freelist); 474 + } 475 + 476 + static const struct iommu_domain_ops amdv1_mock_ops = { 477 + IOMMU_PT_DOMAIN_OPS(amdv1_mock), 478 + .free = mock_domain_free, 479 + .attach_dev = mock_domain_nop_attach, 480 + .set_dev_pasid = mock_domain_set_dev_pasid_nop, 481 + .iotlb_sync = &mock_iotlb_sync, 482 + }; 483 + 484 + static const struct iommu_domain_ops amdv1_mock_huge_ops = { 485 + IOMMU_PT_DOMAIN_OPS(amdv1_mock), 486 + .free = mock_domain_free, 487 + .attach_dev = mock_domain_nop_attach, 488 + .set_dev_pasid = mock_domain_set_dev_pasid_nop, 489 + .iotlb_sync = &mock_iotlb_sync, 490 + }; 491 + #undef pt_iommu_amdv1_mock_map_pages 492 + 493 + static const struct iommu_dirty_ops amdv1_mock_dirty_ops = { 494 + IOMMU_PT_DIRTY_OPS(amdv1_mock), 495 + .set_dirty_tracking = mock_domain_set_dirty_tracking, 496 + }; 497 + 498 + static const struct iommu_domain_ops amdv1_ops = { 499 + IOMMU_PT_DOMAIN_OPS(amdv1), 500 + .free = mock_domain_free, 501 + .attach_dev = mock_domain_nop_attach, 502 + .set_dev_pasid = mock_domain_set_dev_pasid_nop, 503 + .iotlb_sync = &mock_iotlb_sync, 504 + }; 505 + 506 + static const struct iommu_dirty_ops amdv1_dirty_ops = { 507 + IOMMU_PT_DIRTY_OPS(amdv1), 508 + .set_dirty_tracking = mock_domain_set_dirty_tracking, 509 + }; 510 + 511 + static struct mock_iommu_domain * 512 + mock_domain_alloc_pgtable(struct device *dev, 513 + const struct iommu_hwpt_selftest *user_cfg, u32 flags) 514 + { 515 + struct mock_iommu_domain *mock; 516 + int rc; 517 + 518 + mock = kzalloc(sizeof(*mock), GFP_KERNEL); 519 + if (!mock) 520 + return ERR_PTR(-ENOMEM); 521 + mock->domain.type = IOMMU_DOMAIN_UNMANAGED; 522 + 523 + mock->amdv1.iommu.nid = NUMA_NO_NODE; 524 + 525 + switch (user_cfg->pagetable_type) { 526 + case MOCK_IOMMUPT_DEFAULT: 527 + case MOCK_IOMMUPT_HUGE: { 528 + struct pt_iommu_amdv1_cfg cfg = {}; 529 + 530 + /* The mock version has a 2k page size */ 531 + cfg.common.hw_max_vasz_lg2 = 56; 532 + cfg.common.hw_max_oasz_lg2 = 51; 533 + cfg.starting_level = 2; 534 + if (user_cfg->pagetable_type == MOCK_IOMMUPT_HUGE) 535 + mock->domain.ops = &amdv1_mock_huge_ops; 536 + else 537 + mock->domain.ops = &amdv1_mock_ops; 538 + rc = pt_iommu_amdv1_mock_init(&mock->amdv1, &cfg, GFP_KERNEL); 539 + if (rc) 540 + goto err_free; 541 + 542 + /* 543 + * In huge mode userspace should only provide huge pages, we 544 + * have to include PAGE_SIZE for the domain to be accepted by 545 + * iommufd. 546 + */ 547 + if (user_cfg->pagetable_type == MOCK_IOMMUPT_HUGE) 548 + mock->domain.pgsize_bitmap = MOCK_HUGE_PAGE_SIZE | 549 + PAGE_SIZE; 550 + if (flags & IOMMU_HWPT_ALLOC_DIRTY_TRACKING) 551 + mock->domain.dirty_ops = &amdv1_mock_dirty_ops; 552 + break; 553 + } 554 + 555 + case MOCK_IOMMUPT_AMDV1: { 556 + struct pt_iommu_amdv1_cfg cfg = {}; 557 + 558 + cfg.common.hw_max_vasz_lg2 = 64; 559 + cfg.common.hw_max_oasz_lg2 = 52; 560 + cfg.common.features = BIT(PT_FEAT_DYNAMIC_TOP) | 561 + BIT(PT_FEAT_AMDV1_ENCRYPT_TABLES) | 562 + BIT(PT_FEAT_AMDV1_FORCE_COHERENCE); 563 + cfg.starting_level = 2; 564 + mock->domain.ops = &amdv1_ops; 565 + rc = pt_iommu_amdv1_init(&mock->amdv1, &cfg, GFP_KERNEL); 566 + if (rc) 567 + goto err_free; 568 + if (flags & IOMMU_HWPT_ALLOC_DIRTY_TRACKING) 569 + mock->domain.dirty_ops = &amdv1_dirty_ops; 570 + break; 571 + } 572 + default: 573 + rc = -EOPNOTSUPP; 574 + goto err_free; 575 + } 576 + 577 + /* 578 + * Override the real aperture to the MOCK aperture for test purposes. 579 + */ 580 + if (user_cfg->pagetable_type == MOCK_IOMMUPT_DEFAULT) { 581 + WARN_ON(mock->domain.geometry.aperture_start != 0); 582 + WARN_ON(mock->domain.geometry.aperture_end < MOCK_APERTURE_LAST); 583 + 584 + mock->domain.geometry.aperture_start = MOCK_APERTURE_START; 585 + mock->domain.geometry.aperture_end = MOCK_APERTURE_LAST; 586 + } 587 + 588 + return mock; 589 + err_free: 590 + kfree(mock); 591 + return ERR_PTR(rc); 592 + } 593 + 386 594 static struct iommu_domain * 387 595 mock_domain_alloc_paging_flags(struct device *dev, u32 flags, 388 596 const struct iommu_user_data *user_data) ··· 525 469 IOMMU_HWPT_ALLOC_PASID; 526 470 struct mock_dev *mdev = to_mock_dev(dev); 527 471 bool no_dirty_ops = mdev->flags & MOCK_FLAGS_DEVICE_NO_DIRTY; 472 + struct iommu_hwpt_selftest user_cfg = {}; 528 473 struct mock_iommu_domain *mock; 474 + int rc; 529 475 530 - if (user_data) 531 - return ERR_PTR(-EOPNOTSUPP); 532 476 if ((flags & ~PAGING_FLAGS) || (has_dirty_flag && no_dirty_ops)) 533 477 return ERR_PTR(-EOPNOTSUPP); 534 478 535 - mock = kzalloc(sizeof(*mock), GFP_KERNEL); 536 - if (!mock) 537 - return ERR_PTR(-ENOMEM); 538 - mock->domain.geometry.aperture_start = MOCK_APERTURE_START; 539 - mock->domain.geometry.aperture_end = MOCK_APERTURE_LAST; 540 - mock->domain.pgsize_bitmap = MOCK_IO_PAGE_SIZE; 541 - if (dev && mdev->flags & MOCK_FLAGS_DEVICE_HUGE_IOVA) 542 - mock->domain.pgsize_bitmap |= MOCK_HUGE_PAGE_SIZE; 543 - mock->domain.ops = mock_ops.default_domain_ops; 544 - mock->domain.type = IOMMU_DOMAIN_UNMANAGED; 545 - xa_init(&mock->pfns); 479 + if (user_data && (user_data->type != IOMMU_HWPT_DATA_SELFTEST && 480 + user_data->type != IOMMU_HWPT_DATA_NONE)) 481 + return ERR_PTR(-EOPNOTSUPP); 546 482 547 - if (has_dirty_flag) 548 - mock->domain.dirty_ops = &dirty_ops; 483 + if (user_data) { 484 + rc = iommu_copy_struct_from_user( 485 + &user_cfg, user_data, IOMMU_HWPT_DATA_SELFTEST, iotlb); 486 + if (rc) 487 + return ERR_PTR(rc); 488 + } 489 + 490 + mock = mock_domain_alloc_pgtable(dev, &user_cfg, flags); 491 + if (IS_ERR(mock)) 492 + return ERR_CAST(mock); 549 493 return &mock->domain; 550 - } 551 - 552 - static void mock_domain_free(struct iommu_domain *domain) 553 - { 554 - struct mock_iommu_domain *mock = to_mock_domain(domain); 555 - 556 - WARN_ON(!xa_empty(&mock->pfns)); 557 - kfree(mock); 558 - } 559 - 560 - static int mock_domain_map_pages(struct iommu_domain *domain, 561 - unsigned long iova, phys_addr_t paddr, 562 - size_t pgsize, size_t pgcount, int prot, 563 - gfp_t gfp, size_t *mapped) 564 - { 565 - struct mock_iommu_domain *mock = to_mock_domain(domain); 566 - unsigned long flags = MOCK_PFN_START_IOVA; 567 - unsigned long start_iova = iova; 568 - 569 - /* 570 - * xarray does not reliably work with fault injection because it does a 571 - * retry allocation, so put our own failure point. 572 - */ 573 - if (iommufd_should_fail()) 574 - return -ENOENT; 575 - 576 - WARN_ON(iova % MOCK_IO_PAGE_SIZE); 577 - WARN_ON(pgsize % MOCK_IO_PAGE_SIZE); 578 - for (; pgcount; pgcount--) { 579 - size_t cur; 580 - 581 - for (cur = 0; cur != pgsize; cur += MOCK_IO_PAGE_SIZE) { 582 - void *old; 583 - 584 - if (pgcount == 1 && cur + MOCK_IO_PAGE_SIZE == pgsize) 585 - flags = MOCK_PFN_LAST_IOVA; 586 - if (pgsize != MOCK_IO_PAGE_SIZE) { 587 - flags |= MOCK_PFN_HUGE_IOVA; 588 - } 589 - old = xa_store(&mock->pfns, iova / MOCK_IO_PAGE_SIZE, 590 - xa_mk_value((paddr / MOCK_IO_PAGE_SIZE) | 591 - flags), 592 - gfp); 593 - if (xa_is_err(old)) { 594 - for (; start_iova != iova; 595 - start_iova += MOCK_IO_PAGE_SIZE) 596 - xa_erase(&mock->pfns, 597 - start_iova / 598 - MOCK_IO_PAGE_SIZE); 599 - return xa_err(old); 600 - } 601 - WARN_ON(old); 602 - iova += MOCK_IO_PAGE_SIZE; 603 - paddr += MOCK_IO_PAGE_SIZE; 604 - *mapped += MOCK_IO_PAGE_SIZE; 605 - flags = 0; 606 - } 607 - } 608 - return 0; 609 - } 610 - 611 - static size_t mock_domain_unmap_pages(struct iommu_domain *domain, 612 - unsigned long iova, size_t pgsize, 613 - size_t pgcount, 614 - struct iommu_iotlb_gather *iotlb_gather) 615 - { 616 - struct mock_iommu_domain *mock = to_mock_domain(domain); 617 - bool first = true; 618 - size_t ret = 0; 619 - void *ent; 620 - 621 - WARN_ON(iova % MOCK_IO_PAGE_SIZE); 622 - WARN_ON(pgsize % MOCK_IO_PAGE_SIZE); 623 - 624 - for (; pgcount; pgcount--) { 625 - size_t cur; 626 - 627 - for (cur = 0; cur != pgsize; cur += MOCK_IO_PAGE_SIZE) { 628 - ent = xa_erase(&mock->pfns, iova / MOCK_IO_PAGE_SIZE); 629 - 630 - /* 631 - * iommufd generates unmaps that must be a strict 632 - * superset of the map's performend So every 633 - * starting/ending IOVA should have been an iova passed 634 - * to map. 635 - * 636 - * This simple logic doesn't work when the HUGE_PAGE is 637 - * turned on since the core code will automatically 638 - * switch between the two page sizes creating a break in 639 - * the unmap calls. The break can land in the middle of 640 - * contiguous IOVA. 641 - */ 642 - if (!(domain->pgsize_bitmap & MOCK_HUGE_PAGE_SIZE)) { 643 - if (first) { 644 - WARN_ON(ent && !(xa_to_value(ent) & 645 - MOCK_PFN_START_IOVA)); 646 - first = false; 647 - } 648 - if (pgcount == 1 && 649 - cur + MOCK_IO_PAGE_SIZE == pgsize) 650 - WARN_ON(ent && !(xa_to_value(ent) & 651 - MOCK_PFN_LAST_IOVA)); 652 - } 653 - 654 - iova += MOCK_IO_PAGE_SIZE; 655 - ret += MOCK_IO_PAGE_SIZE; 656 - } 657 - } 658 - return ret; 659 - } 660 - 661 - static phys_addr_t mock_domain_iova_to_phys(struct iommu_domain *domain, 662 - dma_addr_t iova) 663 - { 664 - struct mock_iommu_domain *mock = to_mock_domain(domain); 665 - void *ent; 666 - 667 - WARN_ON(iova % MOCK_IO_PAGE_SIZE); 668 - ent = xa_load(&mock->pfns, iova / MOCK_IO_PAGE_SIZE); 669 - WARN_ON(!ent); 670 - return (xa_to_value(ent) & MOCK_PFN_MASK) * MOCK_IO_PAGE_SIZE; 671 494 } 672 495 673 496 static bool mock_domain_capable(struct device *dev, enum iommu_cap cap) ··· 890 955 .user_pasid_table = true, 891 956 .get_viommu_size = mock_get_viommu_size, 892 957 .viommu_init = mock_viommu_init, 893 - .default_domain_ops = 894 - &(struct iommu_domain_ops){ 895 - .free = mock_domain_free, 896 - .attach_dev = mock_domain_nop_attach, 897 - .map_pages = mock_domain_map_pages, 898 - .unmap_pages = mock_domain_unmap_pages, 899 - .iova_to_phys = mock_domain_iova_to_phys, 900 - .set_dev_pasid = mock_domain_set_dev_pasid_nop, 901 - }, 902 958 }; 903 959 904 960 static void mock_domain_free_nested(struct iommu_domain *domain) ··· 973 1047 if (IS_ERR(hwpt)) 974 1048 return hwpt; 975 1049 if (hwpt->domain->type != IOMMU_DOMAIN_UNMANAGED || 976 - hwpt->domain->ops != mock_ops.default_domain_ops) { 1050 + hwpt->domain->owner != &mock_ops) { 977 1051 iommufd_put_object(ucmd->ictx, &hwpt->obj); 978 1052 return ERR_PTR(-EINVAL); 979 1053 } ··· 1014 1088 {}, 1015 1089 }; 1016 1090 const u32 valid_flags = MOCK_FLAGS_DEVICE_NO_DIRTY | 1017 - MOCK_FLAGS_DEVICE_HUGE_IOVA | 1018 1091 MOCK_FLAGS_DEVICE_PASID; 1019 1092 struct mock_dev *mdev; 1020 1093 int rc, i; ··· 1202 1277 { 1203 1278 struct iommufd_hw_pagetable *hwpt; 1204 1279 struct mock_iommu_domain *mock; 1280 + unsigned int page_size; 1205 1281 uintptr_t end; 1206 1282 int rc; 1207 - 1208 - if (iova % MOCK_IO_PAGE_SIZE || length % MOCK_IO_PAGE_SIZE || 1209 - (uintptr_t)uptr % MOCK_IO_PAGE_SIZE || 1210 - check_add_overflow((uintptr_t)uptr, (uintptr_t)length, &end)) 1211 - return -EINVAL; 1212 1283 1213 1284 hwpt = get_md_pagetable(ucmd, mockpt_id, &mock); 1214 1285 if (IS_ERR(hwpt)) 1215 1286 return PTR_ERR(hwpt); 1216 1287 1217 - for (; length; length -= MOCK_IO_PAGE_SIZE) { 1288 + page_size = 1 << __ffs(mock->domain.pgsize_bitmap); 1289 + if (iova % page_size || length % page_size || 1290 + (uintptr_t)uptr % page_size || 1291 + check_add_overflow((uintptr_t)uptr, (uintptr_t)length, &end)) 1292 + return -EINVAL; 1293 + 1294 + for (; length; length -= page_size) { 1218 1295 struct page *pages[1]; 1296 + phys_addr_t io_phys; 1219 1297 unsigned long pfn; 1220 1298 long npages; 1221 - void *ent; 1222 1299 1223 1300 npages = get_user_pages_fast((uintptr_t)uptr & PAGE_MASK, 1, 0, 1224 1301 pages); ··· 1235 1308 pfn = page_to_pfn(pages[0]); 1236 1309 put_page(pages[0]); 1237 1310 1238 - ent = xa_load(&mock->pfns, iova / MOCK_IO_PAGE_SIZE); 1239 - if (!ent || 1240 - (xa_to_value(ent) & MOCK_PFN_MASK) * MOCK_IO_PAGE_SIZE != 1241 - pfn * PAGE_SIZE + ((uintptr_t)uptr % PAGE_SIZE)) { 1311 + io_phys = mock->domain.ops->iova_to_phys(&mock->domain, iova); 1312 + if (io_phys != 1313 + pfn * PAGE_SIZE + ((uintptr_t)uptr % PAGE_SIZE)) { 1242 1314 rc = -EINVAL; 1243 1315 goto out_put; 1244 1316 } 1245 - iova += MOCK_IO_PAGE_SIZE; 1246 - uptr += MOCK_IO_PAGE_SIZE; 1317 + iova += page_size; 1318 + uptr += page_size; 1247 1319 } 1248 1320 rc = 0; 1249 1321 ··· 1721 1795 if (IS_ERR(hwpt)) 1722 1796 return PTR_ERR(hwpt); 1723 1797 1724 - if (!(mock->flags & MOCK_DIRTY_TRACK)) { 1798 + if (!(mock->flags & MOCK_DIRTY_TRACK) || !mock->iommu.ops->set_dirty) { 1725 1799 rc = -EINVAL; 1726 1800 goto out_put; 1727 1801 } ··· 1740 1814 } 1741 1815 1742 1816 for (i = 0; i < max; i++) { 1743 - unsigned long cur = iova + i * page_size; 1744 - void *ent, *old; 1745 - 1746 1817 if (!test_bit(i, (unsigned long *)tmp)) 1747 1818 continue; 1748 - 1749 - ent = xa_load(&mock->pfns, cur / page_size); 1750 - if (ent) { 1751 - unsigned long val; 1752 - 1753 - val = xa_to_value(ent) | MOCK_PFN_DIRTY_IOVA; 1754 - old = xa_store(&mock->pfns, cur / page_size, 1755 - xa_mk_value(val), GFP_KERNEL); 1756 - WARN_ON_ONCE(ent != old); 1757 - count++; 1758 - } 1819 + mock->iommu.ops->set_dirty(&mock->iommu, iova + i * page_size); 1820 + count++; 1759 1821 } 1760 1822 1761 1823 cmd->dirty.out_nr_dirty = count; ··· 2116 2202 platform_device_unregister(selftest_iommu_dev); 2117 2203 debugfs_remove_recursive(dbgfs_root); 2118 2204 } 2205 + 2206 + MODULE_IMPORT_NS("GENERIC_PT_IOMMU");

+12

include/linux/generic_pt/iommu.h

··· 74 74 75 75 struct pt_iommu_ops { 76 76 /** 77 + * @set_dirty: Make the iova write dirty 78 + * @iommu_table: Table to manipulate 79 + * @iova: IO virtual address to start 80 + * 81 + * This is only used by iommufd testing. It makes the iova dirty so that 82 + * read_and_clear_dirty() will see it as dirty. Unlike all the other ops 83 + * this one is safe to call without holding any locking. It may return 84 + * -EAGAIN if there is a race. 85 + */ 86 + int (*set_dirty)(struct pt_iommu *iommu_table, dma_addr_t iova); 87 + 88 + /** 77 89 * @get_info: Return the pt_iommu_info structure 78 90 * @iommu_table: Table to query 79 91 *

+38 -22

tools/testing/selftests/iommu/iommufd.c

··· 13 13 14 14 static unsigned long HUGEPAGE_SIZE; 15 15 16 - #define MOCK_PAGE_SIZE (PAGE_SIZE / 2) 17 - #define MOCK_HUGE_PAGE_SIZE (512 * MOCK_PAGE_SIZE) 18 - 19 16 static unsigned long get_huge_page_size(void) 20 17 { 21 18 char buf[80]; ··· 2055 2058 2056 2059 FIXTURE_SETUP(iommufd_dirty_tracking) 2057 2060 { 2061 + struct iommu_option cmd = { 2062 + .size = sizeof(cmd), 2063 + .option_id = IOMMU_OPTION_HUGE_PAGES, 2064 + .op = IOMMU_OPTION_OP_SET, 2065 + .val64 = 0, 2066 + }; 2058 2067 size_t mmap_buffer_size; 2059 2068 unsigned long size; 2060 2069 int mmap_flags; ··· 2069 2066 2070 2067 if (variant->buffer_size < MOCK_PAGE_SIZE) { 2071 2068 SKIP(return, 2072 - "Skipping buffer_size=%lu, less than MOCK_PAGE_SIZE=%lu", 2069 + "Skipping buffer_size=%lu, less than MOCK_PAGE_SIZE=%u", 2073 2070 variant->buffer_size, MOCK_PAGE_SIZE); 2074 2071 } 2075 2072 ··· 2117 2114 assert((uintptr_t)self->bitmap % PAGE_SIZE == 0); 2118 2115 2119 2116 test_ioctl_ioas_alloc(&self->ioas_id); 2120 - /* Enable 1M mock IOMMU hugepages */ 2121 - if (variant->hugepages) { 2122 - test_cmd_mock_domain_flags(self->ioas_id, 2123 - MOCK_FLAGS_DEVICE_HUGE_IOVA, 2124 - &self->stdev_id, &self->hwpt_id, 2125 - &self->idev_id); 2126 - } else { 2127 - test_cmd_mock_domain(self->ioas_id, &self->stdev_id, 2128 - &self->hwpt_id, &self->idev_id); 2129 - } 2117 + 2118 + /* 2119 + * For dirty testing it is important that the page size fed into 2120 + * the iommu page tables matches the size the dirty logic 2121 + * expects, or set_dirty can touch too much stuff. 2122 + */ 2123 + cmd.object_id = self->ioas_id; 2124 + if (!variant->hugepages) 2125 + ASSERT_EQ(0, ioctl(self->fd, IOMMU_OPTION, &cmd)); 2126 + 2127 + test_cmd_mock_domain(self->ioas_id, &self->stdev_id, &self->hwpt_id, 2128 + &self->idev_id); 2130 2129 } 2131 2130 2132 2131 FIXTURE_TEARDOWN(iommufd_dirty_tracking) ··· 2253 2248 TEST_F(iommufd_dirty_tracking, get_dirty_bitmap) 2254 2249 { 2255 2250 uint32_t page_size = MOCK_PAGE_SIZE; 2251 + uint32_t ioas_id = self->ioas_id; 2256 2252 uint32_t hwpt_id; 2257 - uint32_t ioas_id; 2258 2253 2259 2254 if (variant->hugepages) 2260 2255 page_size = MOCK_HUGE_PAGE_SIZE; 2261 2256 2262 - test_ioctl_ioas_alloc(&ioas_id); 2263 2257 test_ioctl_ioas_map_fixed_id(ioas_id, self->buffer, 2264 2258 variant->buffer_size, MOCK_APERTURE_START); 2265 2259 2266 - test_cmd_hwpt_alloc(self->idev_id, ioas_id, 2267 - IOMMU_HWPT_ALLOC_DIRTY_TRACKING, &hwpt_id); 2260 + if (variant->hugepages) 2261 + test_cmd_hwpt_alloc_iommupt(self->idev_id, ioas_id, 2262 + IOMMU_HWPT_ALLOC_DIRTY_TRACKING, 2263 + MOCK_IOMMUPT_HUGE, &hwpt_id); 2264 + else 2265 + test_cmd_hwpt_alloc_iommupt(self->idev_id, ioas_id, 2266 + IOMMU_HWPT_ALLOC_DIRTY_TRACKING, 2267 + MOCK_IOMMUPT_DEFAULT, &hwpt_id); 2268 2268 2269 2269 test_cmd_set_dirty_tracking(hwpt_id, true); 2270 2270 ··· 2295 2285 TEST_F(iommufd_dirty_tracking, get_dirty_bitmap_no_clear) 2296 2286 { 2297 2287 uint32_t page_size = MOCK_PAGE_SIZE; 2288 + uint32_t ioas_id = self->ioas_id; 2298 2289 uint32_t hwpt_id; 2299 - uint32_t ioas_id; 2300 2290 2301 2291 if (variant->hugepages) 2302 2292 page_size = MOCK_HUGE_PAGE_SIZE; 2303 2293 2304 - test_ioctl_ioas_alloc(&ioas_id); 2305 2294 test_ioctl_ioas_map_fixed_id(ioas_id, self->buffer, 2306 2295 variant->buffer_size, MOCK_APERTURE_START); 2307 2296 2308 - test_cmd_hwpt_alloc(self->idev_id, ioas_id, 2309 - IOMMU_HWPT_ALLOC_DIRTY_TRACKING, &hwpt_id); 2297 + 2298 + if (variant->hugepages) 2299 + test_cmd_hwpt_alloc_iommupt(self->idev_id, ioas_id, 2300 + IOMMU_HWPT_ALLOC_DIRTY_TRACKING, 2301 + MOCK_IOMMUPT_HUGE, &hwpt_id); 2302 + else 2303 + test_cmd_hwpt_alloc_iommupt(self->idev_id, ioas_id, 2304 + IOMMU_HWPT_ALLOC_DIRTY_TRACKING, 2305 + MOCK_IOMMUPT_DEFAULT, &hwpt_id); 2310 2306 2311 2307 test_cmd_set_dirty_tracking(hwpt_id, true); 2312 2308

+12

tools/testing/selftests/iommu/iommufd_utils.h

··· 215 215 ASSERT_EQ(0, _test_cmd_hwpt_alloc(self->fd, device_id, pt_id, 0, flags, \ 216 216 hwpt_id, IOMMU_HWPT_DATA_NONE, NULL, \ 217 217 0)) 218 + #define test_cmd_hwpt_alloc_iommupt(device_id, pt_id, flags, iommupt_type, \ 219 + hwpt_id) \ 220 + ({ \ 221 + struct iommu_hwpt_selftest user_cfg = { \ 222 + .pagetable_type = iommupt_type \ 223 + }; \ 224 + \ 225 + ASSERT_EQ(0, _test_cmd_hwpt_alloc( \ 226 + self->fd, device_id, pt_id, 0, flags, \ 227 + hwpt_id, IOMMU_HWPT_DATA_SELFTEST, \ 228 + &user_cfg, sizeof(user_cfg))); \ 229 + }) 218 230 #define test_err_hwpt_alloc(_errno, device_id, pt_id, flags, hwpt_id) \ 219 231 EXPECT_ERRNO(_errno, _test_cmd_hwpt_alloc( \ 220 232 self->fd, device_id, pt_id, 0, flags, \

Configure Feed

Configure Feed