Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
iommufd/device: Add pasid_attach array to track per-PASID attach
PASIDs of PASID-capable device can be attached to hwpt separately, hence
a pasid array to track per-PASID attachment is necessary. The index
IOMMU_NO_PASID is used by the RID path. Hence drop the igroup->attach.
Link: https://patch.msgid.link/r/20250321171940.7213-10-yi.l.liu@intel.com
Reviewed-by: Jason Gunthorpe <jgg@nvidia.com>
Signed-off-by: Yi Liu <yi.l.liu@intel.com>
Tested-by: Nicolin Chen <nicolinc@nvidia.com>
Signed-off-by: Jason Gunthorpe <jgg@nvidia.com>
authored by
Yi Liu
and committed by
Jason Gunthorpe
c0e301b2
831b40f8
+41
-20
+40
-19
drivers/iommu/iommufd/device.c
+40
-19
drivers/iommu/iommufd/device.c
···
27
27
struct iommufd_group *igroup =
28
28
container_of(kref, struct iommufd_group, ref);
29
29
30
-
WARN_ON(igroup->attach);
30
+
WARN_ON(!xa_empty(&igroup->pasid_attach));
31
31
32
32
xa_cmpxchg(&igroup->ictx->groups, iommu_group_id(igroup->group), igroup,
33
33
NULL, GFP_KERNEL);
···
94
94
95
95
kref_init(&new_igroup->ref);
96
96
mutex_init(&new_igroup->lock);
97
+
xa_init(&new_igroup->pasid_attach);
97
98
new_igroup->sw_msi_start = PHYS_ADDR_MAX;
98
99
/* group reference moves into new_igroup */
99
100
new_igroup->group = group;
···
298
297
}
299
298
EXPORT_SYMBOL_NS_GPL(iommufd_device_to_id, "IOMMUFD");
300
299
301
-
static unsigned int iommufd_group_device_num(struct iommufd_group *igroup)
300
+
static unsigned int iommufd_group_device_num(struct iommufd_group *igroup,
301
+
ioasid_t pasid)
302
302
{
303
+
struct iommufd_attach *attach;
303
304
struct iommufd_device *idev;
304
305
unsigned int count = 0;
305
306
unsigned long index;
306
307
307
308
lockdep_assert_held(&igroup->lock);
308
309
309
-
if (igroup->attach)
310
-
xa_for_each(&igroup->attach->device_array, index, idev)
310
+
attach = xa_load(&igroup->pasid_attach, pasid);
311
+
if (attach)
312
+
xa_for_each(&attach->device_array, index, idev)
311
313
count++;
312
314
return count;
313
315
}
···
355
351
iommufd_group_first_attach(struct iommufd_group *igroup, ioasid_t pasid)
356
352
{
357
353
lockdep_assert_held(&igroup->lock);
358
-
return !igroup->attach;
354
+
return !xa_load(&igroup->pasid_attach, pasid);
359
355
}
360
356
361
357
static int
···
386
382
387
383
/* The device attach/detach/replace helpers for attach_handle */
388
384
389
-
/* Check if idev is attached to igroup->hwpt */
390
-
static bool iommufd_device_is_attached(struct iommufd_device *idev)
385
+
static bool iommufd_device_is_attached(struct iommufd_device *idev,
386
+
ioasid_t pasid)
391
387
{
392
-
return xa_load(&idev->igroup->attach->device_array, idev->obj.id);
388
+
struct iommufd_attach *attach;
389
+
390
+
attach = xa_load(&idev->igroup->pasid_attach, pasid);
391
+
return xa_load(&attach->device_array, idev->obj.id);
393
392
}
394
393
395
394
static int iommufd_hwpt_attach_device(struct iommufd_hw_pagetable *hwpt,
···
519
512
520
513
mutex_lock(&igroup->lock);
521
514
522
-
attach = igroup->attach;
515
+
attach = xa_cmpxchg(&igroup->pasid_attach, pasid, NULL,
516
+
XA_ZERO_ENTRY, GFP_KERNEL);
517
+
if (xa_is_err(attach)) {
518
+
rc = xa_err(attach);
519
+
goto err_unlock;
520
+
}
521
+
523
522
if (!attach) {
524
523
attach = kzalloc(sizeof(*attach), GFP_KERNEL);
525
524
if (!attach) {
526
525
rc = -ENOMEM;
527
-
goto err_unlock;
526
+
goto err_release_pasid;
528
527
}
529
528
xa_init(&attach->device_array);
530
529
}
···
567
554
if (rc)
568
555
goto err_unresv;
569
556
attach->hwpt = hwpt;
570
-
igroup->attach = attach;
557
+
WARN_ON(xa_is_err(xa_store(&igroup->pasid_attach, pasid, attach,
558
+
GFP_KERNEL)));
571
559
}
572
560
refcount_inc(&hwpt->obj.users);
573
561
WARN_ON(xa_is_err(xa_store(&attach->device_array, idev->obj.id,
···
583
569
err_free_attach:
584
570
if (iommufd_group_first_attach(igroup, pasid))
585
571
kfree(attach);
572
+
err_release_pasid:
573
+
if (iommufd_group_first_attach(igroup, pasid))
574
+
xa_release(&igroup->pasid_attach, pasid);
586
575
err_unlock:
587
576
mutex_unlock(&igroup->lock);
588
577
return rc;
···
600
583
struct iommufd_attach *attach;
601
584
602
585
mutex_lock(&igroup->lock);
603
-
attach = igroup->attach;
586
+
attach = xa_load(&igroup->pasid_attach, pasid);
604
587
hwpt = attach->hwpt;
605
588
hwpt_paging = find_hwpt_paging(hwpt);
606
589
607
590
xa_erase(&attach->device_array, idev->obj.id);
608
591
if (xa_empty(&attach->device_array)) {
609
592
iommufd_hwpt_detach_device(hwpt, idev, pasid);
610
-
igroup->attach = NULL;
593
+
xa_erase(&igroup->pasid_attach, pasid);
611
594
kfree(attach);
612
595
}
613
596
if (hwpt_paging && pasid == IOMMU_NO_PASID)
···
634
617
iommufd_group_remove_reserved_iova(struct iommufd_group *igroup,
635
618
struct iommufd_hwpt_paging *hwpt_paging)
636
619
{
620
+
struct iommufd_attach *attach;
637
621
struct iommufd_device *cur;
638
622
unsigned long index;
639
623
640
624
lockdep_assert_held(&igroup->lock);
641
625
642
-
xa_for_each(&igroup->attach->device_array, index, cur)
626
+
attach = xa_load(&igroup->pasid_attach, IOMMU_NO_PASID);
627
+
xa_for_each(&attach->device_array, index, cur)
643
628
iopt_remove_reserved_iova(&hwpt_paging->ioas->iopt, cur->dev);
644
629
}
645
630
···
650
631
struct iommufd_hwpt_paging *hwpt_paging)
651
632
{
652
633
struct iommufd_hwpt_paging *old_hwpt_paging;
634
+
struct iommufd_attach *attach;
653
635
struct iommufd_device *cur;
654
636
unsigned long index;
655
637
int rc;
656
638
657
639
lockdep_assert_held(&igroup->lock);
658
640
659
-
old_hwpt_paging = find_hwpt_paging(igroup->attach->hwpt);
641
+
attach = xa_load(&igroup->pasid_attach, IOMMU_NO_PASID);
642
+
old_hwpt_paging = find_hwpt_paging(attach->hwpt);
660
643
if (!old_hwpt_paging || hwpt_paging->ioas != old_hwpt_paging->ioas) {
661
-
xa_for_each(&igroup->attach->device_array, index, cur) {
644
+
xa_for_each(&attach->device_array, index, cur) {
662
645
rc = iopt_table_enforce_dev_resv_regions(
663
646
&hwpt_paging->ioas->iopt, cur->dev, NULL);
664
647
if (rc)
···
693
672
694
673
mutex_lock(&igroup->lock);
695
674
696
-
attach = igroup->attach;
675
+
attach = xa_load(&igroup->pasid_attach, pasid);
697
676
if (!attach) {
698
677
rc = -EINVAL;
699
678
goto err_unlock;
···
703
682
704
683
WARN_ON(!old_hwpt || xa_empty(&attach->device_array));
705
684
706
-
if (!iommufd_device_is_attached(idev)) {
685
+
if (!iommufd_device_is_attached(idev, pasid)) {
707
686
rc = -EINVAL;
708
687
goto err_unlock;
709
688
}
···
730
709
731
710
attach->hwpt = hwpt;
732
711
733
-
num_devices = iommufd_group_device_num(igroup);
712
+
num_devices = iommufd_group_device_num(igroup, pasid);
734
713
/*
735
714
* Move the refcounts held by the device_array to the new hwpt. Retain a
736
715
* refcount for this thread as the caller will free it.
+1
-1
drivers/iommu/iommufd/iommufd_private.h
+1
-1
drivers/iommu/iommufd/iommufd_private.h