Merge tag 'iommu-updates-v6.16' of git://git.kernel.org/pub/scm/linux/kernel/git/iommu/linux

+4

Documentation/devicetree/bindings/iommu/mediatek,iommu.yaml

··· 74 74 - mediatek,mt2712-m4u # generation two 75 75 - mediatek,mt6779-m4u # generation two 76 76 - mediatek,mt6795-m4u # generation two 77 + - mediatek,mt6893-iommu-mm # generation two 77 78 - mediatek,mt8167-m4u # generation two 78 79 - mediatek,mt8173-m4u # generation two 79 80 - mediatek,mt8183-m4u # generation two ··· 132 131 dt-binding/memory/mt2712-larb-port.h for mt2712, 133 132 dt-binding/memory/mt6779-larb-port.h for mt6779, 134 133 dt-binding/memory/mt6795-larb-port.h for mt6795, 134 + dt-binding/memory/mediatek,mt6893-memory-port.h for mt6893, 135 135 dt-binding/memory/mt8167-larb-port.h for mt8167, 136 136 dt-binding/memory/mt8173-larb-port.h for mt8173, 137 137 dt-binding/memory/mt8183-larb-port.h for mt8183, ··· 159 157 - mediatek,mt2701-m4u 160 158 - mediatek,mt2712-m4u 161 159 - mediatek,mt6795-m4u 160 + - mediatek,mt6893-iommu-mm 162 161 - mediatek,mt8173-m4u 163 162 - mediatek,mt8186-iommu-mm 164 163 - mediatek,mt8188-iommu-vdo ··· 176 173 properties: 177 174 compatible: 178 175 enum: 176 + - mediatek,mt6893-iommu-mm 179 177 - mediatek,mt8186-iommu-mm 180 178 - mediatek,mt8188-iommu-vdo 181 179 - mediatek,mt8188-iommu-vpp

+3

arch/s390/include/asm/pci_dma.h

··· 25 25 #define ZPCI_KEY (PAGE_DEFAULT_KEY << 5) 26 26 27 27 #define ZPCI_TABLE_SIZE_RT (1UL << 42) 28 + #define ZPCI_TABLE_SIZE_RS (1UL << 53) 28 29 29 30 #define ZPCI_IOTA_STO_FLAG (ZPCI_IOTA_IOT_ENABLED | ZPCI_KEY | ZPCI_IOTA_DT_ST) 30 31 #define ZPCI_IOTA_RTTO_FLAG (ZPCI_IOTA_IOT_ENABLED | ZPCI_KEY | ZPCI_IOTA_DT_RT) ··· 56 55 #define ZPCI_PT_BITS 8 57 56 #define ZPCI_ST_SHIFT (ZPCI_PT_BITS + PAGE_SHIFT) 58 57 #define ZPCI_RT_SHIFT (ZPCI_ST_SHIFT + ZPCI_TABLE_BITS) 58 + #define ZPCI_RS_SHIFT (ZPCI_RT_SHIFT + ZPCI_TABLE_BITS) 59 + #define ZPCI_RF_SHIFT (ZPCI_RS_SHIFT + ZPCI_TABLE_BITS) 59 60 60 61 #define ZPCI_RTE_FLAG_MASK 0x3fffUL 61 62 #define ZPCI_RTE_ADDR_MASK (~ZPCI_RTE_FLAG_MASK)

+2 -11

drivers/accel/amdxdna/aie2_pci.c

··· 512 512 goto release_fw; 513 513 } 514 514 515 - ret = iommu_dev_enable_feature(&pdev->dev, IOMMU_DEV_FEAT_SVA); 516 - if (ret) { 517 - XDNA_ERR(xdna, "Enable PASID failed, ret %d", ret); 518 - goto free_irq; 519 - } 520 - 521 515 psp_conf.fw_size = fw->size; 522 516 psp_conf.fw_buf = fw->data; 523 517 for (i = 0; i < PSP_MAX_REGS; i++) ··· 520 526 if (!ndev->psp_hdl) { 521 527 XDNA_ERR(xdna, "failed to create psp"); 522 528 ret = -ENOMEM; 523 - goto disable_sva; 529 + goto free_irq; 524 530 } 525 531 xdna->dev_handle = ndev; 526 532 527 533 ret = aie2_hw_start(xdna); 528 534 if (ret) { 529 535 XDNA_ERR(xdna, "start npu failed, ret %d", ret); 530 - goto disable_sva; 536 + goto free_irq; 531 537 } 532 538 533 539 ret = aie2_mgmt_fw_query(ndev); ··· 578 584 aie2_error_async_events_free(ndev); 579 585 stop_hw: 580 586 aie2_hw_stop(xdna); 581 - disable_sva: 582 - iommu_dev_disable_feature(&pdev->dev, IOMMU_DEV_FEAT_SVA); 583 587 free_irq: 584 588 pci_free_irq_vectors(pdev); 585 589 release_fw: ··· 593 601 594 602 aie2_hw_stop(xdna); 595 603 aie2_error_async_events_free(ndev); 596 - iommu_dev_disable_feature(&pdev->dev, IOMMU_DEV_FEAT_SVA); 597 604 pci_free_irq_vectors(pdev); 598 605 } 599 606

+6 -35

drivers/dma/idxd/init.c

··· 702 702 idxd->pasid = IOMMU_PASID_INVALID; 703 703 } 704 704 705 - static int idxd_enable_sva(struct pci_dev *pdev) 706 - { 707 - int ret; 708 - 709 - ret = iommu_dev_enable_feature(&pdev->dev, IOMMU_DEV_FEAT_IOPF); 710 - if (ret) 711 - return ret; 712 - 713 - ret = iommu_dev_enable_feature(&pdev->dev, IOMMU_DEV_FEAT_SVA); 714 - if (ret) 715 - iommu_dev_disable_feature(&pdev->dev, IOMMU_DEV_FEAT_IOPF); 716 - 717 - return ret; 718 - } 719 - 720 - static void idxd_disable_sva(struct pci_dev *pdev) 721 - { 722 - iommu_dev_disable_feature(&pdev->dev, IOMMU_DEV_FEAT_SVA); 723 - iommu_dev_disable_feature(&pdev->dev, IOMMU_DEV_FEAT_IOPF); 724 - } 725 - 726 705 static int idxd_probe(struct idxd_device *idxd) 727 706 { 728 707 struct pci_dev *pdev = idxd->pdev; ··· 716 737 dev_dbg(dev, "IDXD reset complete\n"); 717 738 718 739 if (IS_ENABLED(CONFIG_INTEL_IDXD_SVM) && sva) { 719 - if (idxd_enable_sva(pdev)) { 720 - dev_warn(dev, "Unable to turn on user SVA feature.\n"); 721 - } else { 722 - set_bit(IDXD_FLAG_USER_PASID_ENABLED, &idxd->flags); 740 + set_bit(IDXD_FLAG_USER_PASID_ENABLED, &idxd->flags); 723 741 724 - rc = idxd_enable_system_pasid(idxd); 725 - if (rc) 726 - dev_warn(dev, "No in-kernel DMA with PASID. %d\n", rc); 727 - else 728 - set_bit(IDXD_FLAG_PASID_ENABLED, &idxd->flags); 729 - } 742 + rc = idxd_enable_system_pasid(idxd); 743 + if (rc) 744 + dev_warn(dev, "No in-kernel DMA with PASID. %d\n", rc); 745 + else 746 + set_bit(IDXD_FLAG_PASID_ENABLED, &idxd->flags); 730 747 } else if (!sva) { 731 748 dev_warn(dev, "User forced SVA off via module param.\n"); 732 749 } ··· 760 785 err: 761 786 if (device_pasid_enabled(idxd)) 762 787 idxd_disable_system_pasid(idxd); 763 - if (device_user_pasid_enabled(idxd)) 764 - idxd_disable_sva(pdev); 765 788 return rc; 766 789 } 767 790 ··· 770 797 idxd_cleanup_internals(idxd); 771 798 if (device_pasid_enabled(idxd)) 772 799 idxd_disable_system_pasid(idxd); 773 - if (device_user_pasid_enabled(idxd)) 774 - idxd_disable_sva(idxd->pdev); 775 800 } 776 801 777 802 /*

+1 -157

drivers/iommu/Kconfig

··· 192 192 If unsure, say N here. 193 193 194 194 source "drivers/iommu/amd/Kconfig" 195 + source "drivers/iommu/arm/Kconfig" 195 196 source "drivers/iommu/intel/Kconfig" 196 197 source "drivers/iommu/iommufd/Kconfig" 197 198 source "drivers/iommu/riscv/Kconfig" ··· 200 199 config IRQ_REMAP 201 200 bool "Support for Interrupt Remapping" 202 201 depends on X86_64 && X86_IO_APIC && PCI_MSI && ACPI 203 - select DMAR_TABLE if INTEL_IOMMU 204 202 help 205 203 Supports Interrupt remapping for IO-APIC and MSI devices. 206 204 To use x2apic mode in the CPU's which support x2APIC enhancements or ··· 314 314 315 315 Say Y here if you are using an Apple SoC. 316 316 317 - # ARM IOMMU support 318 - config ARM_SMMU 319 - tristate "ARM Ltd. System MMU (SMMU) Support" 320 - depends on ARM64 || ARM || COMPILE_TEST 321 - depends on !GENERIC_ATOMIC64 # for IOMMU_IO_PGTABLE_LPAE 322 - select IOMMU_API 323 - select IOMMU_IO_PGTABLE_LPAE 324 - select ARM_DMA_USE_IOMMU if ARM 325 - help 326 - Support for implementations of the ARM System MMU architecture 327 - versions 1 and 2. 328 - 329 - Say Y here if your SoC includes an IOMMU device implementing 330 - the ARM SMMU architecture. 331 - 332 - config ARM_SMMU_LEGACY_DT_BINDINGS 333 - bool "Support the legacy \"mmu-masters\" devicetree bindings" 334 - depends on ARM_SMMU=y && OF 335 - help 336 - Support for the badly designed and deprecated "mmu-masters" 337 - devicetree bindings. This allows some DMA masters to attach 338 - to the SMMU but does not provide any support via the DMA API. 339 - If you're lucky, you might be able to get VFIO up and running. 340 - 341 - If you say Y here then you'll make me very sad. Instead, say N 342 - and move your firmware to the utopian future that was 2016. 343 - 344 - config ARM_SMMU_DISABLE_BYPASS_BY_DEFAULT 345 - bool "Default to disabling bypass on ARM SMMU v1 and v2" 346 - depends on ARM_SMMU 347 - default y 348 - help 349 - Say Y here to (by default) disable bypass streams such that 350 - incoming transactions from devices that are not attached to 351 - an iommu domain will report an abort back to the device and 352 - will not be allowed to pass through the SMMU. 353 - 354 - Any old kernels that existed before this KConfig was 355 - introduced would default to _allowing_ bypass (AKA the 356 - equivalent of NO for this config). However the default for 357 - this option is YES because the old behavior is insecure. 358 - 359 - There are few reasons to allow unmatched stream bypass, and 360 - even fewer good ones. If saying YES here breaks your board 361 - you should work on fixing your board. This KConfig option 362 - is expected to be removed in the future and we'll simply 363 - hardcode the bypass disable in the code. 364 - 365 - NOTE: the kernel command line parameter 366 - 'arm-smmu.disable_bypass' will continue to override this 367 - config. 368 - 369 - config ARM_SMMU_MMU_500_CPRE_ERRATA 370 - bool "Enable errata workaround for CPRE in SMMU reset path" 371 - depends on ARM_SMMU 372 - default y 373 - help 374 - Say Y here (by default) to apply workaround to disable 375 - MMU-500's next-page prefetcher for sake of 4 known errata. 376 - 377 - Say N here only when it is sure that any errata related to 378 - prefetch enablement are not applicable on the platform. 379 - Refer silicon-errata.rst for info on errata IDs. 380 - 381 - config ARM_SMMU_QCOM 382 - def_tristate y 383 - depends on ARM_SMMU && ARCH_QCOM 384 - select QCOM_SCM 385 - help 386 - When running on a Qualcomm platform that has the custom variant 387 - of the ARM SMMU, this needs to be built into the SMMU driver. 388 - 389 - config ARM_SMMU_QCOM_DEBUG 390 - bool "ARM SMMU QCOM implementation defined debug support" 391 - depends on ARM_SMMU_QCOM=y 392 - help 393 - Support for implementation specific debug features in ARM SMMU 394 - hardware found in QTI platforms. This include support for 395 - the Translation Buffer Units (TBU) that can be used to obtain 396 - additional information when debugging memory management issues 397 - like context faults. 398 - 399 - Say Y here to enable debug for issues such as context faults 400 - or TLB sync timeouts which requires implementation defined 401 - register dumps. 402 - 403 - config ARM_SMMU_V3 404 - tristate "ARM Ltd. System MMU Version 3 (SMMUv3) Support" 405 - depends on ARM64 406 - select IOMMU_API 407 - select IOMMU_IO_PGTABLE_LPAE 408 - select GENERIC_MSI_IRQ 409 - select IOMMUFD_DRIVER if IOMMUFD 410 - help 411 - Support for implementations of the ARM System MMU architecture 412 - version 3 providing translation support to a PCIe root complex. 413 - 414 - Say Y here if your system includes an IOMMU device implementing 415 - the ARM SMMUv3 architecture. 416 - 417 - if ARM_SMMU_V3 418 - config ARM_SMMU_V3_SVA 419 - bool "Shared Virtual Addressing support for the ARM SMMUv3" 420 - select IOMMU_SVA 421 - select IOMMU_IOPF 422 - select MMU_NOTIFIER 423 - help 424 - Support for sharing process address spaces with devices using the 425 - SMMUv3. 426 - 427 - Say Y here if your system supports SVA extensions such as PCIe PASID 428 - and PRI. 429 - 430 - config ARM_SMMU_V3_IOMMUFD 431 - bool "Enable IOMMUFD features for ARM SMMUv3 (EXPERIMENTAL)" 432 - depends on IOMMUFD 433 - help 434 - Support for IOMMUFD features intended to support virtual machines 435 - with accelerated virtual IOMMUs. 436 - 437 - Say Y here if you are doing development and testing on this feature. 438 - 439 - config ARM_SMMU_V3_KUNIT_TEST 440 - tristate "KUnit tests for arm-smmu-v3 driver" if !KUNIT_ALL_TESTS 441 - depends on KUNIT 442 - depends on ARM_SMMU_V3_SVA 443 - default KUNIT_ALL_TESTS 444 - help 445 - Enable this option to unit-test arm-smmu-v3 driver functions. 446 - 447 - If unsure, say N. 448 - 449 - config TEGRA241_CMDQV 450 - bool "NVIDIA Tegra241 CMDQ-V extension support for ARM SMMUv3" 451 - depends on ACPI 452 - help 453 - Support for NVIDIA CMDQ-Virtualization extension for ARM SMMUv3. The 454 - CMDQ-V extension is similar to v3.3 ECMDQ for multi command queues 455 - support, except with virtualization capabilities. 456 - 457 - Say Y here if your system is NVIDIA Tegra241 (Grace) or it has the same 458 - CMDQ-V extension. 459 - endif 460 - 461 317 config S390_IOMMU 462 318 def_bool y if S390 && PCI 463 319 depends on S390 && PCI ··· 349 493 DMA memory accesses for the multimedia subsystem. 350 494 351 495 if unsure, say N here. 352 - 353 - config QCOM_IOMMU 354 - # Note: iommu drivers cannot (yet?) be built as modules 355 - bool "Qualcomm IOMMU Support" 356 - depends on ARCH_QCOM || COMPILE_TEST 357 - depends on !GENERIC_ATOMIC64 # for IOMMU_IO_PGTABLE_LPAE 358 - select QCOM_SCM 359 - select IOMMU_API 360 - select IOMMU_IO_PGTABLE_LPAE 361 - select ARM_DMA_USE_IOMMU 362 - help 363 - Support for IOMMU on certain Qualcomm SoCs. 364 496 365 497 config HYPERV_IOMMU 366 498 bool "Hyper-V IRQ Handling"

+5 -1

drivers/iommu/Makefile

··· 1 1 # SPDX-License-Identifier: GPL-2.0 2 - obj-y += amd/ intel/ arm/ iommufd/ riscv/ 2 + obj-y += arm/ iommufd/ 3 + obj-$(CONFIG_AMD_IOMMU) += amd/ 4 + obj-$(CONFIG_INTEL_IOMMU) += intel/ 5 + obj-$(CONFIG_RISCV_IOMMU) += riscv/ 3 6 obj-$(CONFIG_IOMMU_API) += iommu.o 7 + obj-$(CONFIG_IOMMU_SUPPORT) += iommu-pages.o 4 8 obj-$(CONFIG_IOMMU_API) += iommu-traces.o 5 9 obj-$(CONFIG_IOMMU_API) += iommu-sysfs.o 6 10 obj-$(CONFIG_IOMMU_DEBUGFS) += iommu-debugfs.o

+1 -1

drivers/iommu/amd/Makefile

··· 1 1 # SPDX-License-Identifier: GPL-2.0-only 2 - obj-$(CONFIG_AMD_IOMMU) += iommu.o init.o quirks.o io_pgtable.o io_pgtable_v2.o ppr.o pasid.o 2 + obj-y += iommu.o init.o quirks.o io_pgtable.o io_pgtable_v2.o ppr.o pasid.o 3 3 obj-$(CONFIG_AMD_IOMMU_DEBUGFS) += debugfs.o

+2

drivers/iommu/amd/amd_iommu.h

··· 147 147 return PCI_SEG_DEVID_TO_SBDF(seg, devid); 148 148 } 149 149 150 + bool amd_iommu_ht_range_ignore(void); 151 + 150 152 /* 151 153 * This must be called after device probe completes. During probe 152 154 * use rlookup_amd_iommu() get the iommu.

+2 -8

drivers/iommu/amd/amd_iommu_types.h

··· 29 29 * some size calculation constants 30 30 */ 31 31 #define DEV_TABLE_ENTRY_SIZE 32 32 - #define ALIAS_TABLE_ENTRY_SIZE 2 33 - #define RLOOKUP_TABLE_ENTRY_SIZE (sizeof(void *)) 34 32 35 33 /* Capability offsets used by the driver */ 36 34 #define MMIO_CAP_HDR_OFFSET 0x00 ··· 109 111 #define FEATURE_SNPAVICSUP GENMASK_ULL(7, 5) 110 112 #define FEATURE_SNPAVICSUP_GAM(x) \ 111 113 (FIELD_GET(FEATURE_SNPAVICSUP, x) == 0x1) 114 + #define FEATURE_HT_RANGE_IGNORE BIT_ULL(11) 112 115 113 116 #define FEATURE_NUM_INT_REMAP_SUP GENMASK_ULL(9, 8) 114 117 #define FEATURE_NUM_INT_REMAP_SUP_2K(x) \ ··· 315 316 #define DTE_IRQ_REMAP_INTCTL (2ULL << 60) 316 317 #define DTE_IRQ_REMAP_ENABLE 1ULL 317 318 319 + #define DTE_INTTAB_ALIGNMENT 128 318 320 #define DTE_INTTABLEN_MASK (0xfULL << 1) 319 321 #define DTE_INTTABLEN_VALUE_512 9ULL 320 322 #define DTE_INTTABLEN_512 (DTE_INTTABLEN_VALUE_512 << 1) ··· 615 615 616 616 /* Size of the device table */ 617 617 u32 dev_table_size; 618 - 619 - /* Size of the alias table */ 620 - u32 alias_table_size; 621 - 622 - /* Size of the rlookup table */ 623 - u32 rlookup_table_size; 624 618 625 619 /* 626 620 * device table virtual address

+44 -52

drivers/iommu/amd/init.c

··· 243 243 iommu->flags |= AMD_IOMMU_FLAG_TRANS_PRE_ENABLED; 244 244 } 245 245 246 - static inline unsigned long tbl_size(int entry_size, int last_bdf) 247 - { 248 - unsigned shift = PAGE_SHIFT + 249 - get_order((last_bdf + 1) * entry_size); 250 - 251 - return 1UL << shift; 252 - } 253 - 254 246 int amd_iommu_get_num_iommus(void) 255 247 { 256 248 return amd_iommus_present; 249 + } 250 + 251 + bool amd_iommu_ht_range_ignore(void) 252 + { 253 + return check_feature2(FEATURE_HT_RANGE_IGNORE); 257 254 } 258 255 259 256 /* ··· 631 634 /* Allocate per PCI segment device table */ 632 635 static inline int __init alloc_dev_table(struct amd_iommu_pci_seg *pci_seg) 633 636 { 634 - pci_seg->dev_table = iommu_alloc_pages(GFP_KERNEL | GFP_DMA32, 635 - get_order(pci_seg->dev_table_size)); 637 + pci_seg->dev_table = iommu_alloc_pages_sz(GFP_KERNEL | GFP_DMA32, 638 + pci_seg->dev_table_size); 636 639 if (!pci_seg->dev_table) 637 640 return -ENOMEM; 638 641 ··· 641 644 642 645 static inline void free_dev_table(struct amd_iommu_pci_seg *pci_seg) 643 646 { 644 - iommu_free_pages(pci_seg->dev_table, 645 - get_order(pci_seg->dev_table_size)); 647 + iommu_free_pages(pci_seg->dev_table); 646 648 pci_seg->dev_table = NULL; 647 649 } 648 650 649 651 /* Allocate per PCI segment IOMMU rlookup table. */ 650 652 static inline int __init alloc_rlookup_table(struct amd_iommu_pci_seg *pci_seg) 651 653 { 652 - pci_seg->rlookup_table = iommu_alloc_pages(GFP_KERNEL, 653 - get_order(pci_seg->rlookup_table_size)); 654 + pci_seg->rlookup_table = kvcalloc(pci_seg->last_bdf + 1, 655 + sizeof(*pci_seg->rlookup_table), 656 + GFP_KERNEL); 654 657 if (pci_seg->rlookup_table == NULL) 655 658 return -ENOMEM; 656 659 ··· 659 662 660 663 static inline void free_rlookup_table(struct amd_iommu_pci_seg *pci_seg) 661 664 { 662 - iommu_free_pages(pci_seg->rlookup_table, 663 - get_order(pci_seg->rlookup_table_size)); 665 + kvfree(pci_seg->rlookup_table); 664 666 pci_seg->rlookup_table = NULL; 665 667 } 666 668 667 669 static inline int __init alloc_irq_lookup_table(struct amd_iommu_pci_seg *pci_seg) 668 670 { 669 - pci_seg->irq_lookup_table = iommu_alloc_pages(GFP_KERNEL, 670 - get_order(pci_seg->rlookup_table_size)); 671 - kmemleak_alloc(pci_seg->irq_lookup_table, 672 - pci_seg->rlookup_table_size, 1, GFP_KERNEL); 671 + pci_seg->irq_lookup_table = kvcalloc(pci_seg->last_bdf + 1, 672 + sizeof(*pci_seg->irq_lookup_table), 673 + GFP_KERNEL); 673 674 if (pci_seg->irq_lookup_table == NULL) 674 675 return -ENOMEM; 675 676 ··· 676 681 677 682 static inline void free_irq_lookup_table(struct amd_iommu_pci_seg *pci_seg) 678 683 { 679 - kmemleak_free(pci_seg->irq_lookup_table); 680 - iommu_free_pages(pci_seg->irq_lookup_table, 681 - get_order(pci_seg->rlookup_table_size)); 684 + kvfree(pci_seg->irq_lookup_table); 682 685 pci_seg->irq_lookup_table = NULL; 683 686 } 684 687 ··· 684 691 { 685 692 int i; 686 693 687 - pci_seg->alias_table = iommu_alloc_pages(GFP_KERNEL, 688 - get_order(pci_seg->alias_table_size)); 694 + pci_seg->alias_table = kvmalloc_array(pci_seg->last_bdf + 1, 695 + sizeof(*pci_seg->alias_table), 696 + GFP_KERNEL); 689 697 if (!pci_seg->alias_table) 690 698 return -ENOMEM; 691 699 ··· 701 707 702 708 static void __init free_alias_table(struct amd_iommu_pci_seg *pci_seg) 703 709 { 704 - iommu_free_pages(pci_seg->alias_table, 705 - get_order(pci_seg->alias_table_size)); 710 + kvfree(pci_seg->alias_table); 706 711 pci_seg->alias_table = NULL; 707 712 } 708 713 ··· 712 719 */ 713 720 static int __init alloc_command_buffer(struct amd_iommu *iommu) 714 721 { 715 - iommu->cmd_buf = iommu_alloc_pages(GFP_KERNEL, 716 - get_order(CMD_BUFFER_SIZE)); 722 + iommu->cmd_buf = iommu_alloc_pages_sz(GFP_KERNEL, CMD_BUFFER_SIZE); 717 723 718 724 return iommu->cmd_buf ? 0 : -ENOMEM; 719 725 } ··· 809 817 810 818 static void __init free_command_buffer(struct amd_iommu *iommu) 811 819 { 812 - iommu_free_pages(iommu->cmd_buf, get_order(CMD_BUFFER_SIZE)); 820 + iommu_free_pages(iommu->cmd_buf); 813 821 } 814 822 815 823 void *__init iommu_alloc_4k_pages(struct amd_iommu *iommu, gfp_t gfp, 816 824 size_t size) 817 825 { 818 - int order = get_order(size); 819 - void *buf = iommu_alloc_pages(gfp, order); 826 + void *buf; 820 827 821 - if (buf && 822 - check_feature(FEATURE_SNP) && 823 - set_memory_4k((unsigned long)buf, (1 << order))) { 824 - iommu_free_pages(buf, order); 825 - buf = NULL; 828 + size = PAGE_ALIGN(size); 829 + buf = iommu_alloc_pages_sz(gfp, size); 830 + if (!buf) 831 + return NULL; 832 + if (check_feature(FEATURE_SNP) && 833 + set_memory_4k((unsigned long)buf, size / PAGE_SIZE)) { 834 + iommu_free_pages(buf); 835 + return NULL; 826 836 } 827 837 828 838 return buf; ··· 867 873 868 874 static void __init free_event_buffer(struct amd_iommu *iommu) 869 875 { 870 - iommu_free_pages(iommu->evt_buf, get_order(EVT_BUFFER_SIZE)); 876 + iommu_free_pages(iommu->evt_buf); 871 877 } 872 878 873 879 static void free_ga_log(struct amd_iommu *iommu) 874 880 { 875 881 #ifdef CONFIG_IRQ_REMAP 876 - iommu_free_pages(iommu->ga_log, get_order(GA_LOG_SIZE)); 877 - iommu_free_pages(iommu->ga_log_tail, get_order(8)); 882 + iommu_free_pages(iommu->ga_log); 883 + iommu_free_pages(iommu->ga_log_tail); 878 884 #endif 879 885 } 880 886 ··· 919 925 if (!AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir)) 920 926 return 0; 921 927 922 - iommu->ga_log = iommu_alloc_pages(GFP_KERNEL, get_order(GA_LOG_SIZE)); 928 + iommu->ga_log = iommu_alloc_pages_sz(GFP_KERNEL, GA_LOG_SIZE); 923 929 if (!iommu->ga_log) 924 930 goto err_out; 925 931 926 - iommu->ga_log_tail = iommu_alloc_pages(GFP_KERNEL, get_order(8)); 932 + iommu->ga_log_tail = iommu_alloc_pages_sz(GFP_KERNEL, 8); 927 933 if (!iommu->ga_log_tail) 928 934 goto err_out; 929 935 ··· 944 950 static void __init free_cwwb_sem(struct amd_iommu *iommu) 945 951 { 946 952 if (iommu->cmd_sem) 947 - iommu_free_page((void *)iommu->cmd_sem); 953 + iommu_free_pages((void *)iommu->cmd_sem); 948 954 } 949 955 950 956 static void iommu_enable_xt(struct amd_iommu *iommu) ··· 1018 1024 if (!old_devtb) 1019 1025 return false; 1020 1026 1021 - pci_seg->old_dev_tbl_cpy = iommu_alloc_pages(GFP_KERNEL | GFP_DMA32, 1022 - get_order(pci_seg->dev_table_size)); 1027 + pci_seg->old_dev_tbl_cpy = iommu_alloc_pages_sz( 1028 + GFP_KERNEL | GFP_DMA32, pci_seg->dev_table_size); 1023 1029 if (pci_seg->old_dev_tbl_cpy == NULL) { 1024 1030 pr_err("Failed to allocate memory for copying old device table!\n"); 1025 1031 memunmap(old_devtb); ··· 1593 1599 1594 1600 pci_seg->last_bdf = last_bdf; 1595 1601 DUMP_printk("PCI segment : 0x%0x, last bdf : 0x%04x\n", id, last_bdf); 1596 - pci_seg->dev_table_size = tbl_size(DEV_TABLE_ENTRY_SIZE, last_bdf); 1597 - pci_seg->alias_table_size = tbl_size(ALIAS_TABLE_ENTRY_SIZE, last_bdf); 1598 - pci_seg->rlookup_table_size = tbl_size(RLOOKUP_TABLE_ENTRY_SIZE, last_bdf); 1602 + pci_seg->dev_table_size = 1603 + max(roundup_pow_of_two((last_bdf + 1) * DEV_TABLE_ENTRY_SIZE), 1604 + SZ_4K); 1599 1605 1600 1606 pci_seg->id = id; 1601 1607 init_llist_head(&pci_seg->dev_data_list); ··· 2783 2789 2784 2790 for_each_pci_segment(pci_seg) { 2785 2791 if (pci_seg->old_dev_tbl_cpy != NULL) { 2786 - iommu_free_pages(pci_seg->old_dev_tbl_cpy, 2787 - get_order(pci_seg->dev_table_size)); 2792 + iommu_free_pages(pci_seg->old_dev_tbl_cpy); 2788 2793 pci_seg->old_dev_tbl_cpy = NULL; 2789 2794 } 2790 2795 } ··· 2796 2803 pr_info("Copied DEV table from previous kernel.\n"); 2797 2804 2798 2805 for_each_pci_segment(pci_seg) { 2799 - iommu_free_pages(pci_seg->dev_table, 2800 - get_order(pci_seg->dev_table_size)); 2806 + iommu_free_pages(pci_seg->dev_table); 2801 2807 pci_seg->dev_table = pci_seg->old_dev_tbl_cpy; 2802 2808 } 2803 2809

+17 -21

drivers/iommu/amd/io_pgtable.c

··· 47 47 return fpte; 48 48 } 49 49 50 - static void free_pt_page(u64 *pt, struct list_head *freelist) 51 - { 52 - struct page *p = virt_to_page(pt); 53 - 54 - list_add_tail(&p->lru, freelist); 55 - } 56 - 57 - static void free_pt_lvl(u64 *pt, struct list_head *freelist, int lvl) 50 + static void free_pt_lvl(u64 *pt, struct iommu_pages_list *freelist, int lvl) 58 51 { 59 52 u64 *p; 60 53 int i; ··· 70 77 if (lvl > 2) 71 78 free_pt_lvl(p, freelist, lvl - 1); 72 79 else 73 - free_pt_page(p, freelist); 80 + iommu_pages_list_add(freelist, p); 74 81 } 75 82 76 - free_pt_page(pt, freelist); 83 + iommu_pages_list_add(freelist, pt); 77 84 } 78 85 79 - static void free_sub_pt(u64 *root, int mode, struct list_head *freelist) 86 + static void free_sub_pt(u64 *root, int mode, struct iommu_pages_list *freelist) 80 87 { 81 88 switch (mode) { 82 89 case PAGE_MODE_NONE: 83 90 case PAGE_MODE_7_LEVEL: 84 91 break; 85 92 case PAGE_MODE_1_LEVEL: 86 - free_pt_page(root, freelist); 93 + iommu_pages_list_add(freelist, root); 87 94 break; 88 95 case PAGE_MODE_2_LEVEL: 89 96 case PAGE_MODE_3_LEVEL: ··· 114 121 bool ret = true; 115 122 u64 *pte; 116 123 117 - pte = iommu_alloc_page_node(cfg->amd.nid, gfp); 124 + pte = iommu_alloc_pages_node_sz(cfg->amd.nid, gfp, SZ_4K); 118 125 if (!pte) 119 126 return false; 120 127 ··· 139 146 140 147 out: 141 148 spin_unlock_irqrestore(&domain->lock, flags); 142 - iommu_free_page(pte); 149 + iommu_free_pages(pte); 143 150 144 151 return ret; 145 152 } ··· 206 213 207 214 if (!IOMMU_PTE_PRESENT(__pte) || 208 215 pte_level == PAGE_MODE_NONE) { 209 - page = iommu_alloc_page_node(cfg->amd.nid, gfp); 216 + page = iommu_alloc_pages_node_sz(cfg->amd.nid, gfp, 217 + SZ_4K); 210 218 211 219 if (!page) 212 220 return NULL; ··· 216 222 217 223 /* pte could have been changed somewhere. */ 218 224 if (!try_cmpxchg64(pte, &__pte, __npte)) 219 - iommu_free_page(page); 225 + iommu_free_pages(page); 220 226 else if (IOMMU_PTE_PRESENT(__pte)) 221 227 *updated = true; 222 228 ··· 293 299 return pte; 294 300 } 295 301 296 - static void free_clear_pte(u64 *pte, u64 pteval, struct list_head *freelist) 302 + static void free_clear_pte(u64 *pte, u64 pteval, 303 + struct iommu_pages_list *freelist) 297 304 { 298 305 u64 *pt; 299 306 int mode; ··· 323 328 int prot, gfp_t gfp, size_t *mapped) 324 329 { 325 330 struct amd_io_pgtable *pgtable = io_pgtable_ops_to_data(ops); 326 - LIST_HEAD(freelist); 331 + struct iommu_pages_list freelist = IOMMU_PAGES_LIST_INIT(freelist); 327 332 bool updated = false; 328 333 u64 __pte, *pte; 329 334 int ret, i, count; ··· 348 353 for (i = 0; i < count; ++i) 349 354 free_clear_pte(&pte[i], pte[i], &freelist); 350 355 351 - if (!list_empty(&freelist)) 356 + if (!iommu_pages_list_empty(&freelist)) 352 357 updated = true; 353 358 354 359 if (count > 1) { ··· 519 524 static void v1_free_pgtable(struct io_pgtable *iop) 520 525 { 521 526 struct amd_io_pgtable *pgtable = container_of(iop, struct amd_io_pgtable, pgtbl); 522 - LIST_HEAD(freelist); 527 + struct iommu_pages_list freelist = IOMMU_PAGES_LIST_INIT(freelist); 523 528 524 529 if (pgtable->mode == PAGE_MODE_NONE) 525 530 return; ··· 536 541 { 537 542 struct amd_io_pgtable *pgtable = io_pgtable_cfg_to_data(cfg); 538 543 539 - pgtable->root = iommu_alloc_page_node(cfg->amd.nid, GFP_KERNEL); 544 + pgtable->root = 545 + iommu_alloc_pages_node_sz(cfg->amd.nid, GFP_KERNEL, SZ_4K); 540 546 if (!pgtable->root) 541 547 return NULL; 542 548 pgtable->mode = PAGE_MODE_3_LEVEL;

+6 -6

drivers/iommu/amd/io_pgtable_v2.c

··· 121 121 if (level > 2) 122 122 free_pgtable(p, level - 1); 123 123 else 124 - iommu_free_page(p); 124 + iommu_free_pages(p); 125 125 } 126 126 127 - iommu_free_page(pt); 127 + iommu_free_pages(pt); 128 128 } 129 129 130 130 /* Allocate page table */ ··· 152 152 } 153 153 154 154 if (!IOMMU_PTE_PRESENT(__pte)) { 155 - page = iommu_alloc_page_node(nid, gfp); 155 + page = iommu_alloc_pages_node_sz(nid, gfp, SZ_4K); 156 156 if (!page) 157 157 return NULL; 158 158 159 159 __npte = set_pgtable_attr(page); 160 160 /* pte could have been changed somewhere. */ 161 161 if (!try_cmpxchg64(pte, &__pte, __npte)) 162 - iommu_free_page(page); 162 + iommu_free_pages(page); 163 163 else if (IOMMU_PTE_PRESENT(__pte)) 164 164 *updated = true; 165 165 ··· 181 181 if (pg_size == IOMMU_PAGE_SIZE_1G) 182 182 free_pgtable(__pte, end_level - 1); 183 183 else if (pg_size == IOMMU_PAGE_SIZE_2M) 184 - iommu_free_page(__pte); 184 + iommu_free_pages(__pte); 185 185 } 186 186 187 187 return pte; ··· 346 346 struct amd_io_pgtable *pgtable = io_pgtable_cfg_to_data(cfg); 347 347 int ias = IOMMU_IN_ADDR_BIT_SIZE; 348 348 349 - pgtable->pgd = iommu_alloc_page_node(cfg->amd.nid, GFP_KERNEL); 349 + pgtable->pgd = iommu_alloc_pages_node_sz(cfg->amd.nid, GFP_KERNEL, SZ_4K); 350 350 if (!pgtable->pgd) 351 351 return NULL; 352 352

+47 -47

drivers/iommu/amd/iommu.c

··· 241 241 struct acpihid_map_entry **entry) 242 242 { 243 243 struct acpi_device *adev = ACPI_COMPANION(dev); 244 - struct acpihid_map_entry *p; 244 + struct acpihid_map_entry *p, *p1 = NULL; 245 + int hid_count = 0; 246 + bool fw_bug; 245 247 246 248 if (!adev) 247 249 return -ENODEV; ··· 251 249 list_for_each_entry(p, &acpihid_map, list) { 252 250 if (acpi_dev_hid_uid_match(adev, p->hid, 253 251 p->uid[0] ? p->uid : NULL)) { 254 - if (entry) 255 - *entry = p; 256 - return p->devid; 252 + p1 = p; 253 + fw_bug = false; 254 + hid_count = 1; 255 + break; 256 + } 257 + 258 + /* 259 + * Count HID matches w/o UID, raise FW_BUG but allow exactly one match 260 + */ 261 + if (acpi_dev_hid_match(adev, p->hid)) { 262 + p1 = p; 263 + hid_count++; 264 + fw_bug = true; 257 265 } 258 266 } 259 - return -EINVAL; 267 + 268 + if (!p1) 269 + return -EINVAL; 270 + if (fw_bug) 271 + dev_err_once(dev, FW_BUG "No ACPI device matched UID, but %d device%s matched HID.\n", 272 + hid_count, hid_count > 1 ? "s" : ""); 273 + if (hid_count > 1) 274 + return -EINVAL; 275 + if (entry) 276 + *entry = p1; 277 + 278 + return p1->devid; 260 279 } 261 280 262 281 static inline int get_device_sbdf_id(struct device *dev) ··· 1004 981 int amd_iommu_register_ga_log_notifier(int (*notifier)(u32)) 1005 982 { 1006 983 iommu_ga_log_notifier = notifier; 984 + 985 + /* 986 + * Ensure all in-flight IRQ handlers run to completion before returning 987 + * to the caller, e.g. to ensure module code isn't unloaded while it's 988 + * being executed in the IRQ handler. 989 + */ 990 + if (!notifier) 991 + synchronize_rcu(); 1007 992 1008 993 return 0; 1009 994 } ··· 1843 1812 1844 1813 ptr = iommu_phys_to_virt(tbl[i] & PAGE_MASK); 1845 1814 1846 - iommu_free_page(ptr); 1815 + iommu_free_pages(ptr); 1847 1816 } 1848 1817 } 1849 1818 ··· 1876 1845 /* Free per device domain ID */ 1877 1846 pdom_id_free(gcr3_info->domid); 1878 1847 1879 - iommu_free_page(gcr3_info->gcr3_tbl); 1848 + iommu_free_pages(gcr3_info->gcr3_tbl); 1880 1849 gcr3_info->gcr3_tbl = NULL; 1881 1850 } 1882 1851 ··· 1915 1884 return -ENOSPC; 1916 1885 gcr3_info->domid = domid; 1917 1886 1918 - gcr3_info->gcr3_tbl = iommu_alloc_page_node(nid, GFP_ATOMIC); 1887 + gcr3_info->gcr3_tbl = iommu_alloc_pages_node_sz(nid, GFP_ATOMIC, SZ_4K); 1919 1888 if (gcr3_info->gcr3_tbl == NULL) { 1920 1889 pdom_id_free(domid); 1921 1890 return -ENOMEM; ··· 2939 2908 return; 2940 2909 list_add_tail(&region->list, head); 2941 2910 2911 + if (amd_iommu_ht_range_ignore()) 2912 + return; 2913 + 2942 2914 region = iommu_alloc_resv_region(HT_RANGE_START, 2943 2915 HT_RANGE_END - HT_RANGE_START + 1, 2944 2916 0, IOMMU_RESV_RESERVED, GFP_KERNEL); ··· 3018 2984 .read_and_clear_dirty = amd_iommu_read_and_clear_dirty, 3019 2985 }; 3020 2986 3021 - static int amd_iommu_dev_enable_feature(struct device *dev, 3022 - enum iommu_dev_features feat) 3023 - { 3024 - int ret = 0; 3025 - 3026 - switch (feat) { 3027 - case IOMMU_DEV_FEAT_IOPF: 3028 - case IOMMU_DEV_FEAT_SVA: 3029 - break; 3030 - default: 3031 - ret = -EINVAL; 3032 - break; 3033 - } 3034 - return ret; 3035 - } 3036 - 3037 - static int amd_iommu_dev_disable_feature(struct device *dev, 3038 - enum iommu_dev_features feat) 3039 - { 3040 - int ret = 0; 3041 - 3042 - switch (feat) { 3043 - case IOMMU_DEV_FEAT_IOPF: 3044 - case IOMMU_DEV_FEAT_SVA: 3045 - break; 3046 - default: 3047 - ret = -EINVAL; 3048 - break; 3049 - } 3050 - return ret; 3051 - } 3052 - 3053 2987 const struct iommu_ops amd_iommu_ops = { 3054 2988 .capable = amd_iommu_capable, 3055 2989 .blocked_domain = &blocked_domain, ··· 3031 3029 .get_resv_regions = amd_iommu_get_resv_regions, 3032 3030 .is_attach_deferred = amd_iommu_is_attach_deferred, 3033 3031 .def_domain_type = amd_iommu_def_domain_type, 3034 - .dev_enable_feat = amd_iommu_dev_enable_feature, 3035 - .dev_disable_feat = amd_iommu_dev_disable_feature, 3036 3032 .page_response = amd_iommu_page_response, 3037 3033 .default_domain_ops = &(const struct iommu_domain_ops) { 3038 3034 .attach_dev = amd_iommu_attach_device, ··· 3129 3129 return table; 3130 3130 } 3131 3131 3132 - static struct irq_remap_table *__alloc_irq_table(int nid, int order) 3132 + static struct irq_remap_table *__alloc_irq_table(int nid, size_t size) 3133 3133 { 3134 3134 struct irq_remap_table *table; 3135 3135 ··· 3137 3137 if (!table) 3138 3138 return NULL; 3139 3139 3140 - table->table = iommu_alloc_pages_node(nid, GFP_KERNEL, order); 3140 + table->table = iommu_alloc_pages_node_sz( 3141 + nid, GFP_KERNEL, max(DTE_INTTAB_ALIGNMENT, size)); 3141 3142 if (!table->table) { 3142 3143 kfree(table); 3143 3144 return NULL; ··· 3192 3191 struct irq_remap_table *new_table = NULL; 3193 3192 struct amd_iommu_pci_seg *pci_seg; 3194 3193 unsigned long flags; 3195 - int order = get_order(get_irq_table_size(max_irqs)); 3196 3194 int nid = iommu && iommu->dev ? dev_to_node(&iommu->dev->dev) : NUMA_NO_NODE; 3197 3195 u16 alias; 3198 3196 ··· 3211 3211 spin_unlock_irqrestore(&iommu_table_lock, flags); 3212 3212 3213 3213 /* Nothing there yet, allocate new irq remapping table */ 3214 - new_table = __alloc_irq_table(nid, order); 3214 + new_table = __alloc_irq_table(nid, get_irq_table_size(max_irqs)); 3215 3215 if (!new_table) 3216 3216 return NULL; 3217 3217 ··· 3246 3246 spin_unlock_irqrestore(&iommu_table_lock, flags); 3247 3247 3248 3248 if (new_table) { 3249 - iommu_free_pages(new_table->table, order); 3249 + iommu_free_pages(new_table->table); 3250 3250 kfree(new_table); 3251 3251 } 3252 3252 return table;

+1 -1

drivers/iommu/amd/ppr.c

··· 48 48 49 49 void __init amd_iommu_free_ppr_log(struct amd_iommu *iommu) 50 50 { 51 - iommu_free_pages(iommu->ppr_log, get_order(PPR_LOG_SIZE)); 51 + iommu_free_pages(iommu->ppr_log); 52 52 } 53 53 54 54 /*

+1 -2

drivers/iommu/apple-dart.c

··· 776 776 { 777 777 struct apple_dart_domain *dart_domain = to_dart_domain(domain); 778 778 779 - if (dart_domain->pgtbl_ops) 780 - free_io_pgtable_ops(dart_domain->pgtbl_ops); 779 + free_io_pgtable_ops(dart_domain->pgtbl_ops); 781 780 782 781 kfree(dart_domain); 783 782 }

+144

drivers/iommu/arm/Kconfig

··· 1 + # SPDX-License-Identifier: GPL-2.0-only 2 + # ARM IOMMU support 3 + config ARM_SMMU 4 + tristate "ARM Ltd. System MMU (SMMU) Support" 5 + depends on ARM64 || ARM || COMPILE_TEST 6 + depends on !GENERIC_ATOMIC64 # for IOMMU_IO_PGTABLE_LPAE 7 + select IOMMU_API 8 + select IOMMU_IO_PGTABLE_LPAE 9 + select ARM_DMA_USE_IOMMU if ARM 10 + help 11 + Support for implementations of the ARM System MMU architecture 12 + versions 1 and 2. 13 + 14 + Say Y here if your SoC includes an IOMMU device implementing 15 + the ARM SMMU architecture. 16 + 17 + if ARM_SMMU 18 + config ARM_SMMU_LEGACY_DT_BINDINGS 19 + bool "Support the legacy \"mmu-masters\" devicetree bindings" 20 + depends on ARM_SMMU=y && OF 21 + help 22 + Support for the badly designed and deprecated "mmu-masters" 23 + devicetree bindings. This allows some DMA masters to attach 24 + to the SMMU but does not provide any support via the DMA API. 25 + If you're lucky, you might be able to get VFIO up and running. 26 + 27 + If you say Y here then you'll make me very sad. Instead, say N 28 + and move your firmware to the utopian future that was 2016. 29 + 30 + config ARM_SMMU_DISABLE_BYPASS_BY_DEFAULT 31 + bool "Disable unmatched stream bypass by default" if EXPERT 32 + default y 33 + help 34 + If your firmware is broken and fails to describe StreamIDs which 35 + Linux should know about in order to manage the SMMU correctly and 36 + securely, and you don't want to boot with the 'arm-smmu.disable_bypass=0' 37 + command line parameter, then as a last resort you can turn it off 38 + by default here. But don't. This option may be removed at any time. 39 + 40 + Note that 'arm-smmu.disable_bypass=1' will still take precedence. 41 + 42 + config ARM_SMMU_MMU_500_CPRE_ERRATA 43 + bool "Enable errata workaround for CPRE in SMMU reset path" 44 + default y 45 + help 46 + Say Y here (by default) to apply workaround to disable 47 + MMU-500's next-page prefetcher for sake of 4 known errata. 48 + 49 + Say N here only when it is sure that any errata related to 50 + prefetch enablement are not applicable on the platform. 51 + Refer silicon-errata.rst for info on errata IDs. 52 + 53 + config ARM_SMMU_QCOM 54 + def_tristate y 55 + depends on ARCH_QCOM 56 + select QCOM_SCM 57 + help 58 + When running on a Qualcomm platform that has the custom variant 59 + of the ARM SMMU, this needs to be built into the SMMU driver. 60 + 61 + config ARM_SMMU_QCOM_DEBUG 62 + bool "ARM SMMU QCOM implementation defined debug support" 63 + depends on ARM_SMMU_QCOM=y 64 + help 65 + Support for implementation specific debug features in ARM SMMU 66 + hardware found in QTI platforms. This include support for 67 + the Translation Buffer Units (TBU) that can be used to obtain 68 + additional information when debugging memory management issues 69 + like context faults. 70 + 71 + Say Y here to enable debug for issues such as context faults 72 + or TLB sync timeouts which requires implementation defined 73 + register dumps. 74 + endif 75 + 76 + config ARM_SMMU_V3 77 + tristate "ARM Ltd. System MMU Version 3 (SMMUv3) Support" 78 + depends on ARM64 79 + select IOMMU_API 80 + select IOMMU_IO_PGTABLE_LPAE 81 + select GENERIC_MSI_IRQ 82 + select IOMMUFD_DRIVER if IOMMUFD 83 + help 84 + Support for implementations of the ARM System MMU architecture 85 + version 3 providing translation support to a PCIe root complex. 86 + 87 + Say Y here if your system includes an IOMMU device implementing 88 + the ARM SMMUv3 architecture. 89 + 90 + if ARM_SMMU_V3 91 + config ARM_SMMU_V3_SVA 92 + bool "Shared Virtual Addressing support for the ARM SMMUv3" 93 + select IOMMU_SVA 94 + select IOMMU_IOPF 95 + select MMU_NOTIFIER 96 + help 97 + Support for sharing process address spaces with devices using the 98 + SMMUv3. 99 + 100 + Say Y here if your system supports SVA extensions such as PCIe PASID 101 + and PRI. 102 + 103 + config ARM_SMMU_V3_IOMMUFD 104 + bool "Enable IOMMUFD features for ARM SMMUv3 (EXPERIMENTAL)" 105 + depends on IOMMUFD 106 + help 107 + Support for IOMMUFD features intended to support virtual machines 108 + with accelerated virtual IOMMUs. 109 + 110 + Say Y here if you are doing development and testing on this feature. 111 + 112 + config ARM_SMMU_V3_KUNIT_TEST 113 + tristate "KUnit tests for arm-smmu-v3 driver" if !KUNIT_ALL_TESTS 114 + depends on KUNIT 115 + depends on ARM_SMMU_V3_SVA 116 + default KUNIT_ALL_TESTS 117 + help 118 + Enable this option to unit-test arm-smmu-v3 driver functions. 119 + 120 + If unsure, say N. 121 + 122 + config TEGRA241_CMDQV 123 + bool "NVIDIA Tegra241 CMDQ-V extension support for ARM SMMUv3" 124 + depends on ACPI 125 + help 126 + Support for NVIDIA CMDQ-Virtualization extension for ARM SMMUv3. The 127 + CMDQ-V extension is similar to v3.3 ECMDQ for multi command queues 128 + support, except with virtualization capabilities. 129 + 130 + Say Y here if your system is NVIDIA Tegra241 (Grace) or it has the same 131 + CMDQ-V extension. 132 + endif 133 + 134 + config QCOM_IOMMU 135 + # Note: iommu drivers cannot (yet?) be built as modules 136 + bool "Qualcomm IOMMU Support" 137 + depends on ARCH_QCOM || COMPILE_TEST 138 + depends on !GENERIC_ATOMIC64 # for IOMMU_IO_PGTABLE_LPAE 139 + select QCOM_SCM 140 + select IOMMU_API 141 + select IOMMU_IO_PGTABLE_LPAE 142 + select ARM_DMA_USE_IOMMU 143 + help 144 + Support for IOMMU on certain Qualcomm SoCs.

+2 -1

drivers/iommu/arm/Makefile

··· 1 1 # SPDX-License-Identifier: GPL-2.0 2 - obj-y += arm-smmu/ arm-smmu-v3/ 2 + obj-y += arm-smmu/ 3 + obj-$(CONFIG_ARM_SMMU_V3) += arm-smmu-v3/

+1 -1

drivers/iommu/arm/arm-smmu-v3/Makefile

··· 1 1 # SPDX-License-Identifier: GPL-2.0 2 - obj-$(CONFIG_ARM_SMMU_V3) += arm_smmu_v3.o 2 + obj-y += arm_smmu_v3.o 3 3 arm_smmu_v3-y := arm-smmu-v3.o 4 4 arm_smmu_v3-$(CONFIG_ARM_SMMU_V3_IOMMUFD) += arm-smmu-v3-iommufd.o 5 5 arm_smmu_v3-$(CONFIG_ARM_SMMU_V3_SVA) += arm-smmu-v3-sva.o

+6 -80

drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c

··· 13 13 #include "arm-smmu-v3.h" 14 14 #include "../../io-pgtable-arm.h" 15 15 16 - static DEFINE_MUTEX(sva_lock); 17 - 18 16 static void __maybe_unused 19 17 arm_smmu_update_s1_domain_cd_entry(struct arm_smmu_domain *smmu_domain) 20 18 { ··· 255 257 return true; 256 258 } 257 259 258 - bool arm_smmu_master_iopf_supported(struct arm_smmu_master *master) 259 - { 260 - /* We're not keeping track of SIDs in fault events */ 261 - if (master->num_streams != 1) 262 - return false; 263 - 264 - return master->stall_enabled; 265 - } 266 - 267 - bool arm_smmu_master_sva_supported(struct arm_smmu_master *master) 268 - { 269 - if (!(master->smmu->features & ARM_SMMU_FEAT_SVA)) 270 - return false; 271 - 272 - /* SSID support is mandatory for the moment */ 273 - return master->ssid_bits; 274 - } 275 - 276 - bool arm_smmu_master_sva_enabled(struct arm_smmu_master *master) 277 - { 278 - bool enabled; 279 - 280 - mutex_lock(&sva_lock); 281 - enabled = master->sva_enabled; 282 - mutex_unlock(&sva_lock); 283 - return enabled; 284 - } 285 - 286 - static int arm_smmu_master_sva_enable_iopf(struct arm_smmu_master *master) 287 - { 288 - struct device *dev = master->dev; 289 - 290 - /* 291 - * Drivers for devices supporting PRI or stall should enable IOPF first. 292 - * Others have device-specific fault handlers and don't need IOPF. 293 - */ 294 - if (!arm_smmu_master_iopf_supported(master)) 295 - return 0; 296 - 297 - if (!master->iopf_enabled) 298 - return -EINVAL; 299 - 300 - return iopf_queue_add_device(master->smmu->evtq.iopf, dev); 301 - } 302 - 303 - static void arm_smmu_master_sva_disable_iopf(struct arm_smmu_master *master) 304 - { 305 - struct device *dev = master->dev; 306 - 307 - if (!master->iopf_enabled) 308 - return; 309 - 310 - iopf_queue_remove_device(master->smmu->evtq.iopf, dev); 311 - } 312 - 313 - int arm_smmu_master_enable_sva(struct arm_smmu_master *master) 314 - { 315 - int ret; 316 - 317 - mutex_lock(&sva_lock); 318 - ret = arm_smmu_master_sva_enable_iopf(master); 319 - if (!ret) 320 - master->sva_enabled = true; 321 - mutex_unlock(&sva_lock); 322 - 323 - return ret; 324 - } 325 - 326 - int arm_smmu_master_disable_sva(struct arm_smmu_master *master) 327 - { 328 - mutex_lock(&sva_lock); 329 - arm_smmu_master_sva_disable_iopf(master); 330 - master->sva_enabled = false; 331 - mutex_unlock(&sva_lock); 332 - 333 - return 0; 334 - } 335 - 336 260 void arm_smmu_sva_notifier_synchronize(void) 337 261 { 338 262 /* ··· 272 352 struct arm_smmu_master *master = dev_iommu_priv_get(dev); 273 353 struct arm_smmu_cd target; 274 354 int ret; 355 + 356 + if (!(master->smmu->features & ARM_SMMU_FEAT_SVA)) 357 + return -EOPNOTSUPP; 275 358 276 359 /* Prevent arm_smmu_mm_release from being called while we are attaching */ 277 360 if (!mmget_not_zero(domain->mm)) ··· 328 405 struct arm_smmu_domain *smmu_domain; 329 406 u32 asid; 330 407 int ret; 408 + 409 + if (!(master->smmu->features & ARM_SMMU_FEAT_SVA)) 410 + return ERR_PTR(-EOPNOTSUPP); 331 411 332 412 smmu_domain = arm_smmu_domain_alloc(); 333 413 if (IS_ERR(smmu_domain))

+76 -62

drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c

··· 2720 2720 2721 2721 static struct arm_smmu_master_domain * 2722 2722 arm_smmu_find_master_domain(struct arm_smmu_domain *smmu_domain, 2723 + struct iommu_domain *domain, 2723 2724 struct arm_smmu_master *master, 2724 2725 ioasid_t ssid, bool nested_ats_flush) 2725 2726 { ··· 2731 2730 list_for_each_entry(master_domain, &smmu_domain->devices, 2732 2731 devices_elm) { 2733 2732 if (master_domain->master == master && 2733 + master_domain->domain == domain && 2734 2734 master_domain->ssid == ssid && 2735 2735 master_domain->nested_ats_flush == nested_ats_flush) 2736 2736 return master_domain; ··· 2758 2756 return NULL; 2759 2757 } 2760 2758 2759 + static int arm_smmu_enable_iopf(struct arm_smmu_master *master, 2760 + struct arm_smmu_master_domain *master_domain) 2761 + { 2762 + int ret; 2763 + 2764 + iommu_group_mutex_assert(master->dev); 2765 + 2766 + if (!IS_ENABLED(CONFIG_ARM_SMMU_V3_SVA)) 2767 + return -EOPNOTSUPP; 2768 + 2769 + /* 2770 + * Drivers for devices supporting PRI or stall require iopf others have 2771 + * device-specific fault handlers and don't need IOPF, so this is not a 2772 + * failure. 2773 + */ 2774 + if (!master->stall_enabled) 2775 + return 0; 2776 + 2777 + /* We're not keeping track of SIDs in fault events */ 2778 + if (master->num_streams != 1) 2779 + return -EOPNOTSUPP; 2780 + 2781 + if (master->iopf_refcount) { 2782 + master->iopf_refcount++; 2783 + master_domain->using_iopf = true; 2784 + return 0; 2785 + } 2786 + 2787 + ret = iopf_queue_add_device(master->smmu->evtq.iopf, master->dev); 2788 + if (ret) 2789 + return ret; 2790 + master->iopf_refcount = 1; 2791 + master_domain->using_iopf = true; 2792 + return 0; 2793 + } 2794 + 2795 + static void arm_smmu_disable_iopf(struct arm_smmu_master *master, 2796 + struct arm_smmu_master_domain *master_domain) 2797 + { 2798 + iommu_group_mutex_assert(master->dev); 2799 + 2800 + if (!IS_ENABLED(CONFIG_ARM_SMMU_V3_SVA)) 2801 + return; 2802 + 2803 + if (!master_domain || !master_domain->using_iopf) 2804 + return; 2805 + 2806 + master->iopf_refcount--; 2807 + if (master->iopf_refcount == 0) 2808 + iopf_queue_remove_device(master->smmu->evtq.iopf, master->dev); 2809 + } 2810 + 2761 2811 static void arm_smmu_remove_master_domain(struct arm_smmu_master *master, 2762 2812 struct iommu_domain *domain, 2763 2813 ioasid_t ssid) ··· 2826 2772 nested_ats_flush = to_smmu_nested_domain(domain)->enable_ats; 2827 2773 2828 2774 spin_lock_irqsave(&smmu_domain->devices_lock, flags); 2829 - master_domain = arm_smmu_find_master_domain(smmu_domain, master, ssid, 2830 - nested_ats_flush); 2775 + master_domain = arm_smmu_find_master_domain(smmu_domain, domain, master, 2776 + ssid, nested_ats_flush); 2831 2777 if (master_domain) { 2832 2778 list_del(&master_domain->devices_elm); 2833 - kfree(master_domain); 2834 2779 if (master->ats_enabled) 2835 2780 atomic_dec(&smmu_domain->nr_ats_masters); 2836 2781 } 2837 2782 spin_unlock_irqrestore(&smmu_domain->devices_lock, flags); 2783 + 2784 + arm_smmu_disable_iopf(master, master_domain); 2785 + kfree(master_domain); 2838 2786 } 2839 2787 2840 2788 /* ··· 2909 2853 kfree(state->vmaster); 2910 2854 return -ENOMEM; 2911 2855 } 2856 + master_domain->domain = new_domain; 2912 2857 master_domain->master = master; 2913 2858 master_domain->ssid = state->ssid; 2914 2859 if (new_domain->type == IOMMU_DOMAIN_NESTED) 2915 2860 master_domain->nested_ats_flush = 2916 2861 to_smmu_nested_domain(new_domain)->enable_ats; 2862 + 2863 + if (new_domain->iopf_handler) { 2864 + ret = arm_smmu_enable_iopf(master, master_domain); 2865 + if (ret) 2866 + goto err_free_master_domain; 2867 + } 2917 2868 2918 2869 /* 2919 2870 * During prepare we want the current smmu_domain and new ··· 2941 2878 !arm_smmu_master_canwbs(master)) { 2942 2879 spin_unlock_irqrestore(&smmu_domain->devices_lock, 2943 2880 flags); 2944 - kfree(master_domain); 2945 2881 kfree(state->vmaster); 2946 - return -EINVAL; 2882 + ret = -EINVAL; 2883 + goto err_iopf; 2947 2884 } 2948 2885 2949 2886 if (state->ats_enabled) ··· 2962 2899 wmb(); 2963 2900 } 2964 2901 return 0; 2902 + 2903 + err_iopf: 2904 + arm_smmu_disable_iopf(master, master_domain); 2905 + err_free_master_domain: 2906 + kfree(master_domain); 2907 + return ret; 2965 2908 } 2966 2909 2967 2910 /* ··· 3022 2953 smmu = master->smmu; 3023 2954 3024 2955 if (smmu_domain->smmu != smmu) 3025 - return ret; 2956 + return -EINVAL; 3026 2957 3027 2958 if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) { 3028 2959 cdptr = arm_smmu_alloc_cd_ptr(master, IOMMU_NO_PASID); ··· 3579 3510 { 3580 3511 struct arm_smmu_master *master = dev_iommu_priv_get(dev); 3581 3512 3582 - if (WARN_ON(arm_smmu_master_sva_enabled(master))) 3583 - iopf_queue_remove_device(master->smmu->evtq.iopf, dev); 3513 + WARN_ON(master->iopf_refcount); 3584 3514 3585 3515 /* Put the STE back to what arm_smmu_init_strtab() sets */ 3586 3516 if (dev->iommu->require_direct) ··· 3654 3586 iommu_dma_get_resv_regions(dev, head); 3655 3587 } 3656 3588 3657 - static int arm_smmu_dev_enable_feature(struct device *dev, 3658 - enum iommu_dev_features feat) 3659 - { 3660 - struct arm_smmu_master *master = dev_iommu_priv_get(dev); 3661 - 3662 - if (!master) 3663 - return -ENODEV; 3664 - 3665 - switch (feat) { 3666 - case IOMMU_DEV_FEAT_IOPF: 3667 - if (!arm_smmu_master_iopf_supported(master)) 3668 - return -EINVAL; 3669 - if (master->iopf_enabled) 3670 - return -EBUSY; 3671 - master->iopf_enabled = true; 3672 - return 0; 3673 - case IOMMU_DEV_FEAT_SVA: 3674 - if (!arm_smmu_master_sva_supported(master)) 3675 - return -EINVAL; 3676 - if (arm_smmu_master_sva_enabled(master)) 3677 - return -EBUSY; 3678 - return arm_smmu_master_enable_sva(master); 3679 - default: 3680 - return -EINVAL; 3681 - } 3682 - } 3683 - 3684 - static int arm_smmu_dev_disable_feature(struct device *dev, 3685 - enum iommu_dev_features feat) 3686 - { 3687 - struct arm_smmu_master *master = dev_iommu_priv_get(dev); 3688 - 3689 - if (!master) 3690 - return -EINVAL; 3691 - 3692 - switch (feat) { 3693 - case IOMMU_DEV_FEAT_IOPF: 3694 - if (!master->iopf_enabled) 3695 - return -EINVAL; 3696 - if (master->sva_enabled) 3697 - return -EBUSY; 3698 - master->iopf_enabled = false; 3699 - return 0; 3700 - case IOMMU_DEV_FEAT_SVA: 3701 - if (!arm_smmu_master_sva_enabled(master)) 3702 - return -EINVAL; 3703 - return arm_smmu_master_disable_sva(master); 3704 - default: 3705 - return -EINVAL; 3706 - } 3707 - } 3708 - 3709 3589 /* 3710 3590 * HiSilicon PCIe tune and trace device can be used to trace TLP headers on the 3711 3591 * PCIe link and save the data to memory by DMA. The hardware is restricted to ··· 3686 3670 .device_group = arm_smmu_device_group, 3687 3671 .of_xlate = arm_smmu_of_xlate, 3688 3672 .get_resv_regions = arm_smmu_get_resv_regions, 3689 - .dev_enable_feat = arm_smmu_dev_enable_feature, 3690 - .dev_disable_feat = arm_smmu_dev_disable_feature, 3691 3673 .page_response = arm_smmu_page_response, 3692 3674 .def_domain_type = arm_smmu_def_domain_type, 3693 3675 .viommu_alloc = arm_vsmmu_alloc,

+7 -32

drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h

··· 837 837 bool ats_enabled : 1; 838 838 bool ste_ats_enabled : 1; 839 839 bool stall_enabled; 840 - bool sva_enabled; 841 - bool iopf_enabled; 842 840 unsigned int ssid_bits; 841 + unsigned int iopf_refcount; 843 842 }; 844 843 845 844 /* SMMU private data for an IOMMU domain */ ··· 914 915 struct arm_smmu_master_domain { 915 916 struct list_head devices_elm; 916 917 struct arm_smmu_master *master; 918 + /* 919 + * For nested domains the master_domain is threaded onto the S2 parent, 920 + * this points to the IOMMU_DOMAIN_NESTED to disambiguate the masters. 921 + */ 922 + struct iommu_domain *domain; 917 923 ioasid_t ssid; 918 924 bool nested_ats_flush : 1; 925 + bool using_iopf : 1; 919 926 }; 920 927 921 928 static inline struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom) ··· 1000 995 1001 996 #ifdef CONFIG_ARM_SMMU_V3_SVA 1002 997 bool arm_smmu_sva_supported(struct arm_smmu_device *smmu); 1003 - bool arm_smmu_master_sva_supported(struct arm_smmu_master *master); 1004 - bool arm_smmu_master_sva_enabled(struct arm_smmu_master *master); 1005 - int arm_smmu_master_enable_sva(struct arm_smmu_master *master); 1006 - int arm_smmu_master_disable_sva(struct arm_smmu_master *master); 1007 - bool arm_smmu_master_iopf_supported(struct arm_smmu_master *master); 1008 998 void arm_smmu_sva_notifier_synchronize(void); 1009 999 struct iommu_domain *arm_smmu_sva_domain_alloc(struct device *dev, 1010 1000 struct mm_struct *mm); 1011 1001 #else /* CONFIG_ARM_SMMU_V3_SVA */ 1012 1002 static inline bool arm_smmu_sva_supported(struct arm_smmu_device *smmu) 1013 - { 1014 - return false; 1015 - } 1016 - 1017 - static inline bool arm_smmu_master_sva_supported(struct arm_smmu_master *master) 1018 - { 1019 - return false; 1020 - } 1021 - 1022 - static inline bool arm_smmu_master_sva_enabled(struct arm_smmu_master *master) 1023 - { 1024 - return false; 1025 - } 1026 - 1027 - static inline int arm_smmu_master_enable_sva(struct arm_smmu_master *master) 1028 - { 1029 - return -ENODEV; 1030 - } 1031 - 1032 - static inline int arm_smmu_master_disable_sva(struct arm_smmu_master *master) 1033 - { 1034 - return -ENODEV; 1035 - } 1036 - 1037 - static inline bool arm_smmu_master_iopf_supported(struct arm_smmu_master *master) 1038 1003 { 1039 1004 return false; 1040 1005 }

+9

drivers/iommu/arm/arm-smmu/arm-smmu-qcom-debug.c

··· 406 406 arm_smmu_print_context_fault_info(smmu, idx, &cfi); 407 407 408 408 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_FSR, cfi.fsr); 409 + 410 + if (cfi.fsr & ARM_SMMU_CB_FSR_SS) { 411 + arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_RESUME, 412 + ret == -EAGAIN ? 0 : ARM_SMMU_RESUME_TERMINATE); 413 + } 414 + 409 415 return IRQ_HANDLED; 410 416 } 411 417 ··· 422 416 if (!tmp || tmp == -EBUSY) { 423 417 ret = IRQ_HANDLED; 424 418 resume = ARM_SMMU_RESUME_TERMINATE; 419 + } else if (tmp == -EAGAIN) { 420 + ret = IRQ_HANDLED; 421 + resume = 0; 425 422 } else { 426 423 phys_addr_t phys_atos = qcom_smmu_verify_fault(smmu_domain, cfi.iova, cfi.fsr); 427 424

+30 -14

drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c

··· 112 112 { 113 113 struct arm_smmu_domain *smmu_domain = (void *)cookie; 114 114 struct arm_smmu_cfg *cfg = &smmu_domain->cfg; 115 - struct qcom_smmu *qsmmu = to_qcom_smmu(smmu_domain->smmu); 115 + struct arm_smmu_device *smmu = smmu_domain->smmu; 116 + struct qcom_smmu *qsmmu = to_qcom_smmu(smmu); 117 + u32 mask = BIT(cfg->cbndx); 118 + bool stall_changed = !!(qsmmu->stall_enabled & mask) != enabled; 119 + unsigned long flags; 116 120 117 121 if (enabled) 118 - qsmmu->stall_enabled |= BIT(cfg->cbndx); 122 + qsmmu->stall_enabled |= mask; 119 123 else 120 - qsmmu->stall_enabled &= ~BIT(cfg->cbndx); 121 - } 124 + qsmmu->stall_enabled &= ~mask; 122 125 123 - static void qcom_adreno_smmu_resume_translation(const void *cookie, bool terminate) 124 - { 125 - struct arm_smmu_domain *smmu_domain = (void *)cookie; 126 - struct arm_smmu_cfg *cfg = &smmu_domain->cfg; 127 - struct arm_smmu_device *smmu = smmu_domain->smmu; 128 - u32 reg = 0; 126 + /* 127 + * If the device is on and we changed the setting, update the register. 128 + * The spec pseudocode says that CFCFG is resampled after a fault, and 129 + * we believe that no implementations cache it in the TLB, so it should 130 + * be safe to change it without a TLB invalidation. 131 + */ 132 + if (stall_changed && pm_runtime_get_if_active(smmu->dev) > 0) { 133 + u32 reg; 129 134 130 - if (terminate) 131 - reg |= ARM_SMMU_RESUME_TERMINATE; 135 + spin_lock_irqsave(&smmu_domain->cb_lock, flags); 136 + reg = arm_smmu_cb_read(smmu, cfg->cbndx, ARM_SMMU_CB_SCTLR); 132 137 133 - arm_smmu_cb_write(smmu, cfg->cbndx, ARM_SMMU_CB_RESUME, reg); 138 + if (enabled) 139 + reg |= ARM_SMMU_SCTLR_CFCFG; 140 + else 141 + reg &= ~ARM_SMMU_SCTLR_CFCFG; 142 + 143 + arm_smmu_cb_write(smmu, cfg->cbndx, ARM_SMMU_CB_SCTLR, reg); 144 + spin_unlock_irqrestore(&smmu_domain->cb_lock, flags); 145 + 146 + pm_runtime_put_autosuspend(smmu->dev); 147 + } 134 148 } 135 149 136 150 static void qcom_adreno_smmu_set_prr_bit(const void *cookie, bool set) ··· 351 337 priv->set_ttbr0_cfg = qcom_adreno_smmu_set_ttbr0_cfg; 352 338 priv->get_fault_info = qcom_adreno_smmu_get_fault_info; 353 339 priv->set_stall = qcom_adreno_smmu_set_stall; 354 - priv->resume_translation = qcom_adreno_smmu_resume_translation; 355 340 priv->set_prr_bit = NULL; 356 341 priv->set_prr_addr = NULL; 357 342 ··· 369 356 { .compatible = "qcom,mdp4" }, 370 357 { .compatible = "qcom,mdss" }, 371 358 { .compatible = "qcom,qcm2290-mdss" }, 359 + { .compatible = "qcom,sar2130p-mdss" }, 372 360 { .compatible = "qcom,sc7180-mdss" }, 373 361 { .compatible = "qcom,sc7180-mss-pil" }, 374 362 { .compatible = "qcom,sc7280-mdss" }, ··· 599 585 .alloc_context_bank = qcom_adreno_smmu_alloc_context_bank, 600 586 .write_sctlr = qcom_adreno_smmu_write_sctlr, 601 587 .tlb_sync = qcom_smmu_tlb_sync, 588 + .context_fault_needs_threaded_irq = true, 602 589 }; 603 590 604 591 static const struct arm_smmu_impl qcom_adreno_smmu_500_impl = { ··· 609 594 .alloc_context_bank = qcom_adreno_smmu_alloc_context_bank, 610 595 .write_sctlr = qcom_adreno_smmu_write_sctlr, 611 596 .tlb_sync = qcom_smmu_tlb_sync, 597 + .context_fault_needs_threaded_irq = true, 612 598 }; 613 599 614 600 static struct arm_smmu_device *qcom_smmu_create(struct arm_smmu_device *smmu,

+6

drivers/iommu/arm/arm-smmu/arm-smmu.c

··· 474 474 arm_smmu_print_context_fault_info(smmu, idx, &cfi); 475 475 476 476 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_FSR, cfi.fsr); 477 + 478 + if (cfi.fsr & ARM_SMMU_CB_FSR_SS) { 479 + arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_RESUME, 480 + ret == -EAGAIN ? 0 : ARM_SMMU_RESUME_TERMINATE); 481 + } 482 + 477 483 return IRQ_HANDLED; 478 484 } 479 485

+7 -4

drivers/iommu/dma-iommu.c

··· 106 106 struct iova_fq_entry { 107 107 unsigned long iova_pfn; 108 108 unsigned long pages; 109 - struct list_head freelist; 109 + struct iommu_pages_list freelist; 110 110 u64 counter; /* Flush counter when this entry was added */ 111 111 }; 112 112 ··· 155 155 fq->entries[idx].iova_pfn, 156 156 fq->entries[idx].pages); 157 157 158 + fq->entries[idx].freelist = 159 + IOMMU_PAGES_LIST_INIT(fq->entries[idx].freelist); 158 160 fq->head = (fq->head + 1) & fq->mod_mask; 159 161 } 160 162 } ··· 195 193 196 194 static void queue_iova(struct iommu_dma_cookie *cookie, 197 195 unsigned long pfn, unsigned long pages, 198 - struct list_head *freelist) 196 + struct iommu_pages_list *freelist) 199 197 { 200 198 struct iova_fq *fq; 201 199 unsigned long flags; ··· 234 232 fq->entries[idx].iova_pfn = pfn; 235 233 fq->entries[idx].pages = pages; 236 234 fq->entries[idx].counter = atomic64_read(&cookie->fq_flush_start_cnt); 237 - list_splice(freelist, &fq->entries[idx].freelist); 235 + iommu_pages_list_splice(freelist, &fq->entries[idx].freelist); 238 236 239 237 spin_unlock_irqrestore(&fq->lock, flags); 240 238 ··· 292 290 spin_lock_init(&fq->lock); 293 291 294 292 for (i = 0; i < fq_size; i++) 295 - INIT_LIST_HEAD(&fq->entries[i].freelist); 293 + fq->entries[i].freelist = 294 + IOMMU_PAGES_LIST_INIT(fq->entries[i].freelist); 296 295 } 297 296 298 297 static int iommu_dma_init_fq_single(struct iommu_dma_cookie *cookie)

+6 -6

drivers/iommu/exynos-iommu.c

··· 902 902 if (!domain) 903 903 return NULL; 904 904 905 - domain->pgtable = iommu_alloc_pages(GFP_KERNEL, 2); 905 + domain->pgtable = iommu_alloc_pages_sz(GFP_KERNEL, SZ_16K); 906 906 if (!domain->pgtable) 907 907 goto err_pgtable; 908 908 909 - domain->lv2entcnt = iommu_alloc_pages(GFP_KERNEL, 1); 909 + domain->lv2entcnt = iommu_alloc_pages_sz(GFP_KERNEL, SZ_8K); 910 910 if (!domain->lv2entcnt) 911 911 goto err_counter; 912 912 ··· 932 932 return &domain->domain; 933 933 934 934 err_lv2ent: 935 - iommu_free_pages(domain->lv2entcnt, 1); 935 + iommu_free_pages(domain->lv2entcnt); 936 936 err_counter: 937 - iommu_free_pages(domain->pgtable, 2); 937 + iommu_free_pages(domain->pgtable); 938 938 err_pgtable: 939 939 kfree(domain); 940 940 return NULL; ··· 975 975 phys_to_virt(base)); 976 976 } 977 977 978 - iommu_free_pages(domain->pgtable, 2); 979 - iommu_free_pages(domain->lv2entcnt, 1); 978 + iommu_free_pages(domain->pgtable); 979 + iommu_free_pages(domain->lv2entcnt); 980 980 kfree(domain); 981 981 } 982 982

+1 -1

drivers/iommu/fsl_pamu_domain.c

··· 64 64 spin_lock_irqsave(&iommu_lock, flags); 65 65 ret = pamu_update_paace_stash(liodn, val); 66 66 if (ret) { 67 - pr_debug("Failed to update SPAACE for liodn %d\n ", liodn); 67 + pr_debug("Failed to update SPAACE for liodn %d\n", liodn); 68 68 spin_unlock_irqrestore(&iommu_lock, flags); 69 69 return ret; 70 70 }

+2 -5

drivers/iommu/intel/Makefile

··· 1 1 # SPDX-License-Identifier: GPL-2.0 2 - obj-$(CONFIG_DMAR_TABLE) += dmar.o 3 - obj-$(CONFIG_INTEL_IOMMU) += iommu.o pasid.o nested.o cache.o prq.o 4 - obj-$(CONFIG_DMAR_TABLE) += trace.o 2 + obj-y += iommu.o pasid.o nested.o cache.o prq.o 3 + obj-$(CONFIG_DMAR_TABLE) += dmar.o trace.o 5 4 obj-$(CONFIG_DMAR_PERF) += perf.o 6 5 obj-$(CONFIG_INTEL_IOMMU_DEBUGFS) += debugfs.o 7 6 obj-$(CONFIG_INTEL_IOMMU_SVM) += svm.o 8 - ifdef CONFIG_INTEL_IOMMU 9 7 obj-$(CONFIG_IRQ_REMAP) += irq_remapping.o 10 - endif 11 8 obj-$(CONFIG_INTEL_IOMMU_PERF_EVENTS) += perfmon.o

+9 -5

drivers/iommu/intel/dmar.c

··· 1099 1099 spin_lock_init(&iommu->device_rbtree_lock); 1100 1100 mutex_init(&iommu->iopf_lock); 1101 1101 iommu->node = NUMA_NO_NODE; 1102 + spin_lock_init(&iommu->lock); 1103 + ida_init(&iommu->domain_ida); 1104 + mutex_init(&iommu->did_lock); 1102 1105 1103 1106 ver = readl(iommu->reg + DMAR_VER_REG); 1104 1107 pr_info("%s: reg_base_addr %llx ver %d:%d cap %llx ecap %llx\n", ··· 1190 1187 } 1191 1188 1192 1189 if (iommu->qi) { 1193 - iommu_free_page(iommu->qi->desc); 1190 + iommu_free_pages(iommu->qi->desc); 1194 1191 kfree(iommu->qi->desc_status); 1195 1192 kfree(iommu->qi); 1196 1193 } ··· 1198 1195 if (iommu->reg) 1199 1196 unmap_iommu(iommu); 1200 1197 1198 + ida_destroy(&iommu->domain_ida); 1201 1199 ida_free(&dmar_seq_ids, iommu->seq_id); 1202 1200 kfree(iommu); 1203 1201 } ··· 1685 1681 { 1686 1682 struct q_inval *qi; 1687 1683 void *desc; 1688 - int order; 1689 1684 1690 1685 if (!ecap_qis(iommu->ecap)) 1691 1686 return -ENOENT; ··· 1705 1702 * Need two pages to accommodate 256 descriptors of 256 bits each 1706 1703 * if the remapping hardware supports scalable mode translation. 1707 1704 */ 1708 - order = ecap_smts(iommu->ecap) ? 1 : 0; 1709 - desc = iommu_alloc_pages_node(iommu->node, GFP_ATOMIC, order); 1705 + desc = iommu_alloc_pages_node_sz(iommu->node, GFP_ATOMIC, 1706 + ecap_smts(iommu->ecap) ? SZ_8K : 1707 + SZ_4K); 1710 1708 if (!desc) { 1711 1709 kfree(qi); 1712 1710 iommu->qi = NULL; ··· 1718 1714 1719 1715 qi->desc_status = kcalloc(QI_LENGTH, sizeof(int), GFP_ATOMIC); 1720 1716 if (!qi->desc_status) { 1721 - iommu_free_page(qi->desc); 1717 + iommu_free_pages(qi->desc); 1722 1718 kfree(qi); 1723 1719 iommu->qi = NULL; 1724 1720 return -ENOMEM;

+102 -142

drivers/iommu/intel/iommu.c

··· 397 397 if (!alloc) 398 398 return NULL; 399 399 400 - context = iommu_alloc_page_node(iommu->node, GFP_ATOMIC); 400 + context = iommu_alloc_pages_node_sz(iommu->node, GFP_ATOMIC, 401 + SZ_4K); 401 402 if (!context) 402 403 return NULL; 403 404 ··· 572 571 for (i = 0; i < ROOT_ENTRY_NR; i++) { 573 572 context = iommu_context_addr(iommu, i, 0, 0); 574 573 if (context) 575 - iommu_free_page(context); 574 + iommu_free_pages(context); 576 575 577 576 if (!sm_supported(iommu)) 578 577 continue; 579 578 580 579 context = iommu_context_addr(iommu, i, 0x80, 0); 581 580 if (context) 582 - iommu_free_page(context); 581 + iommu_free_pages(context); 583 582 } 584 583 585 - iommu_free_page(iommu->root_entry); 584 + iommu_free_pages(iommu->root_entry); 586 585 iommu->root_entry = NULL; 587 586 } 588 587 ··· 732 731 if (!dma_pte_present(pte)) { 733 732 uint64_t pteval, tmp; 734 733 735 - tmp_page = iommu_alloc_page_node(domain->nid, gfp); 734 + tmp_page = iommu_alloc_pages_node_sz(domain->nid, gfp, 735 + SZ_4K); 736 736 737 737 if (!tmp_page) 738 738 return NULL; ··· 747 745 tmp = 0ULL; 748 746 if (!try_cmpxchg64(&pte->val, &tmp, pteval)) 749 747 /* Someone else set it while we were thinking; use theirs. */ 750 - iommu_free_page(tmp_page); 748 + iommu_free_pages(tmp_page); 751 749 else 752 750 domain_flush_cache(domain, pte, sizeof(*pte)); 753 751 } ··· 860 858 last_pfn < level_pfn + level_size(level) - 1)) { 861 859 dma_clear_pte(pte); 862 860 domain_flush_cache(domain, pte, sizeof(*pte)); 863 - iommu_free_page(level_pte); 861 + iommu_free_pages(level_pte); 864 862 } 865 863 next: 866 864 pfn += level_size(level); ··· 884 882 885 883 /* free pgd */ 886 884 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) { 887 - iommu_free_page(domain->pgd); 885 + iommu_free_pages(domain->pgd); 888 886 domain->pgd = NULL; 889 887 } 890 888 } ··· 896 894 The 'pte' argument is the *parent* PTE, pointing to the page that is to 897 895 be freed. */ 898 896 static void dma_pte_list_pagetables(struct dmar_domain *domain, 899 - int level, struct dma_pte *pte, 900 - struct list_head *freelist) 897 + int level, struct dma_pte *parent_pte, 898 + struct iommu_pages_list *freelist) 901 899 { 902 - struct page *pg; 900 + struct dma_pte *pte = phys_to_virt(dma_pte_addr(parent_pte)); 903 901 904 - pg = pfn_to_page(dma_pte_addr(pte) >> PAGE_SHIFT); 905 - list_add_tail(&pg->lru, freelist); 902 + iommu_pages_list_add(freelist, pte); 906 903 907 904 if (level == 1) 908 905 return; 909 906 910 - pte = page_address(pg); 911 907 do { 912 908 if (dma_pte_present(pte) && !dma_pte_superpage(pte)) 913 909 dma_pte_list_pagetables(domain, level - 1, pte, freelist); ··· 916 916 static void dma_pte_clear_level(struct dmar_domain *domain, int level, 917 917 struct dma_pte *pte, unsigned long pfn, 918 918 unsigned long start_pfn, unsigned long last_pfn, 919 - struct list_head *freelist) 919 + struct iommu_pages_list *freelist) 920 920 { 921 921 struct dma_pte *first_pte = NULL, *last_pte = NULL; 922 922 ··· 961 961 the page tables, and may have cached the intermediate levels. The 962 962 pages can only be freed after the IOTLB flush has been done. */ 963 963 static void domain_unmap(struct dmar_domain *domain, unsigned long start_pfn, 964 - unsigned long last_pfn, struct list_head *freelist) 964 + unsigned long last_pfn, 965 + struct iommu_pages_list *freelist) 965 966 { 966 967 if (WARN_ON(!domain_pfn_supported(domain, last_pfn)) || 967 968 WARN_ON(start_pfn > last_pfn)) ··· 974 973 975 974 /* free pgd */ 976 975 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) { 977 - struct page *pgd_page = virt_to_page(domain->pgd); 978 - list_add_tail(&pgd_page->lru, freelist); 976 + iommu_pages_list_add(freelist, domain->pgd); 979 977 domain->pgd = NULL; 980 978 } 981 979 } ··· 984 984 { 985 985 struct root_entry *root; 986 986 987 - root = iommu_alloc_page_node(iommu->node, GFP_ATOMIC); 987 + root = iommu_alloc_pages_node_sz(iommu->node, GFP_ATOMIC, SZ_4K); 988 988 if (!root) { 989 989 pr_err("Allocating root entry for %s failed\n", 990 990 iommu->name); ··· 1289 1289 raw_spin_unlock_irqrestore(&iommu->register_lock, flag); 1290 1290 } 1291 1291 1292 - static int iommu_init_domains(struct intel_iommu *iommu) 1293 - { 1294 - u32 ndomains; 1295 - 1296 - ndomains = cap_ndoms(iommu->cap); 1297 - pr_debug("%s: Number of Domains supported <%d>\n", 1298 - iommu->name, ndomains); 1299 - 1300 - spin_lock_init(&iommu->lock); 1301 - 1302 - iommu->domain_ids = bitmap_zalloc(ndomains, GFP_KERNEL); 1303 - if (!iommu->domain_ids) 1304 - return -ENOMEM; 1305 - 1306 - /* 1307 - * If Caching mode is set, then invalid translations are tagged 1308 - * with domain-id 0, hence we need to pre-allocate it. We also 1309 - * use domain-id 0 as a marker for non-allocated domain-id, so 1310 - * make sure it is not used for a real domain. 1311 - */ 1312 - set_bit(0, iommu->domain_ids); 1313 - 1314 - /* 1315 - * Vt-d spec rev3.0 (section 6.2.3.1) requires that each pasid 1316 - * entry for first-level or pass-through translation modes should 1317 - * be programmed with a domain id different from those used for 1318 - * second-level or nested translation. We reserve a domain id for 1319 - * this purpose. This domain id is also used for identity domain 1320 - * in legacy mode. 1321 - */ 1322 - set_bit(FLPT_DEFAULT_DID, iommu->domain_ids); 1323 - 1324 - return 0; 1325 - } 1326 - 1327 1292 static void disable_dmar_iommu(struct intel_iommu *iommu) 1328 1293 { 1329 - if (!iommu->domain_ids) 1330 - return; 1331 - 1332 1294 /* 1333 1295 * All iommu domains must have been detached from the devices, 1334 1296 * hence there should be no domain IDs in use. 1335 1297 */ 1336 - if (WARN_ON(bitmap_weight(iommu->domain_ids, cap_ndoms(iommu->cap)) 1337 - > NUM_RESERVED_DID)) 1298 + if (WARN_ON(!ida_is_empty(&iommu->domain_ida))) 1338 1299 return; 1339 1300 1340 1301 if (iommu->gcmd & DMA_GCMD_TE) ··· 1304 1343 1305 1344 static void free_dmar_iommu(struct intel_iommu *iommu) 1306 1345 { 1307 - if (iommu->domain_ids) { 1308 - bitmap_free(iommu->domain_ids); 1309 - iommu->domain_ids = NULL; 1310 - } 1311 - 1312 1346 if (iommu->copied_tables) { 1313 1347 bitmap_free(iommu->copied_tables); 1314 1348 iommu->copied_tables = NULL; ··· 1336 1380 int domain_attach_iommu(struct dmar_domain *domain, struct intel_iommu *iommu) 1337 1381 { 1338 1382 struct iommu_domain_info *info, *curr; 1339 - unsigned long ndomains; 1340 1383 int num, ret = -ENOSPC; 1341 1384 1342 1385 if (domain->domain.type == IOMMU_DOMAIN_SVA) ··· 1345 1390 if (!info) 1346 1391 return -ENOMEM; 1347 1392 1348 - spin_lock(&iommu->lock); 1393 + guard(mutex)(&iommu->did_lock); 1349 1394 curr = xa_load(&domain->iommu_array, iommu->seq_id); 1350 1395 if (curr) { 1351 1396 curr->refcnt++; 1352 - spin_unlock(&iommu->lock); 1353 1397 kfree(info); 1354 1398 return 0; 1355 1399 } 1356 1400 1357 - ndomains = cap_ndoms(iommu->cap); 1358 - num = find_first_zero_bit(iommu->domain_ids, ndomains); 1359 - if (num >= ndomains) { 1401 + num = ida_alloc_range(&iommu->domain_ida, IDA_START_DID, 1402 + cap_ndoms(iommu->cap) - 1, GFP_KERNEL); 1403 + if (num < 0) { 1360 1404 pr_err("%s: No free domain ids\n", iommu->name); 1361 1405 goto err_unlock; 1362 1406 } 1363 1407 1364 - set_bit(num, iommu->domain_ids); 1365 1408 info->refcnt = 1; 1366 1409 info->did = num; 1367 1410 info->iommu = iommu; 1368 1411 curr = xa_cmpxchg(&domain->iommu_array, iommu->seq_id, 1369 - NULL, info, GFP_ATOMIC); 1412 + NULL, info, GFP_KERNEL); 1370 1413 if (curr) { 1371 1414 ret = xa_err(curr) ? : -EBUSY; 1372 1415 goto err_clear; 1373 1416 } 1374 1417 1375 - spin_unlock(&iommu->lock); 1376 1418 return 0; 1377 1419 1378 1420 err_clear: 1379 - clear_bit(info->did, iommu->domain_ids); 1421 + ida_free(&iommu->domain_ida, info->did); 1380 1422 err_unlock: 1381 - spin_unlock(&iommu->lock); 1382 1423 kfree(info); 1383 1424 return ret; 1384 1425 } ··· 1386 1435 if (domain->domain.type == IOMMU_DOMAIN_SVA) 1387 1436 return; 1388 1437 1389 - spin_lock(&iommu->lock); 1438 + guard(mutex)(&iommu->did_lock); 1390 1439 info = xa_load(&domain->iommu_array, iommu->seq_id); 1391 1440 if (--info->refcnt == 0) { 1392 - clear_bit(info->did, iommu->domain_ids); 1441 + ida_free(&iommu->domain_ida, info->did); 1393 1442 xa_erase(&domain->iommu_array, iommu->seq_id); 1394 1443 domain->nid = NUMA_NO_NODE; 1395 1444 kfree(info); 1396 1445 } 1397 - spin_unlock(&iommu->lock); 1398 1446 } 1399 1447 1400 1448 static void domain_exit(struct dmar_domain *domain) 1401 1449 { 1402 1450 if (domain->pgd) { 1403 - LIST_HEAD(freelist); 1451 + struct iommu_pages_list freelist = 1452 + IOMMU_PAGES_LIST_INIT(freelist); 1404 1453 1405 1454 domain_unmap(domain, 0, DOMAIN_MAX_PFN(domain->gaw), &freelist); 1406 1455 iommu_put_pages_list(&freelist); ··· 1632 1681 } 1633 1682 1634 1683 attr = prot & (DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP); 1635 - attr |= DMA_FL_PTE_PRESENT; 1636 1684 if (domain->use_first_level) { 1637 - attr |= DMA_FL_PTE_US | DMA_FL_PTE_ACCESS; 1685 + attr |= DMA_FL_PTE_PRESENT | DMA_FL_PTE_US | DMA_FL_PTE_ACCESS; 1638 1686 if (prot & DMA_PTE_WRITE) 1639 1687 attr |= DMA_FL_PTE_DIRTY; 1640 1688 } ··· 1809 1859 return ret; 1810 1860 1811 1861 info->domain = domain; 1862 + info->domain_attached = true; 1812 1863 spin_lock_irqsave(&domain->lock, flags); 1813 1864 list_add(&info->link, &domain->devices); 1814 1865 spin_unlock_irqrestore(&domain->lock, flags); ··· 1978 2027 if (!old_ce) 1979 2028 goto out; 1980 2029 1981 - new_ce = iommu_alloc_page_node(iommu->node, GFP_KERNEL); 2030 + new_ce = iommu_alloc_pages_node_sz(iommu->node, 2031 + GFP_KERNEL, SZ_4K); 1982 2032 if (!new_ce) 1983 2033 goto out_unmap; 1984 2034 ··· 1994 2042 1995 2043 did = context_domain_id(&ce); 1996 2044 if (did >= 0 && did < cap_ndoms(iommu->cap)) 1997 - set_bit(did, iommu->domain_ids); 2045 + ida_alloc_range(&iommu->domain_ida, did, did, GFP_KERNEL); 1998 2046 1999 2047 set_context_copied(iommu, bus, devfn); 2000 2048 new_ce[idx] = ce; ··· 2121 2169 } 2122 2170 2123 2171 intel_iommu_init_qi(iommu); 2124 - 2125 - ret = iommu_init_domains(iommu); 2126 - if (ret) 2127 - goto free_iommu; 2128 - 2129 2172 init_translation_status(iommu); 2130 2173 2131 2174 if (translation_pre_enabled(iommu) && !is_kdump_kernel()) { ··· 2598 2651 if (iommu->gcmd & DMA_GCMD_TE) 2599 2652 iommu_disable_translation(iommu); 2600 2653 2601 - ret = iommu_init_domains(iommu); 2602 - if (ret == 0) 2603 - ret = iommu_alloc_root_entry(iommu); 2654 + ret = iommu_alloc_root_entry(iommu); 2604 2655 if (ret) 2605 2656 goto out; 2606 2657 ··· 2689 2744 struct device *tmp; 2690 2745 int i; 2691 2746 2692 - dev = pci_physfn(dev); 2693 2747 rcu_read_lock(); 2694 2748 2695 2749 list_for_each_entry_rcu(satcu, &dmar_satc_units, list) { ··· 2705 2761 return satcu; 2706 2762 } 2707 2763 2708 - static int dmar_ats_supported(struct pci_dev *dev, struct intel_iommu *iommu) 2764 + static bool dmar_ats_supported(struct pci_dev *dev, struct intel_iommu *iommu) 2709 2765 { 2710 - int i, ret = 1; 2711 - struct pci_bus *bus; 2712 2766 struct pci_dev *bridge = NULL; 2713 - struct device *tmp; 2714 - struct acpi_dmar_atsr *atsr; 2715 2767 struct dmar_atsr_unit *atsru; 2716 2768 struct dmar_satc_unit *satcu; 2769 + struct acpi_dmar_atsr *atsr; 2770 + bool supported = true; 2771 + struct pci_bus *bus; 2772 + struct device *tmp; 2773 + int i; 2717 2774 2718 2775 dev = pci_physfn(dev); 2719 2776 satcu = dmar_find_matched_satc_unit(dev); ··· 2732 2787 bridge = bus->self; 2733 2788 /* If it's an integrated device, allow ATS */ 2734 2789 if (!bridge) 2735 - return 1; 2790 + return true; 2736 2791 /* Connected via non-PCIe: no ATS */ 2737 2792 if (!pci_is_pcie(bridge) || 2738 2793 pci_pcie_type(bridge) == PCI_EXP_TYPE_PCI_BRIDGE) 2739 - return 0; 2794 + return false; 2740 2795 /* If we found the root port, look it up in the ATSR */ 2741 2796 if (pci_pcie_type(bridge) == PCI_EXP_TYPE_ROOT_PORT) 2742 2797 break; ··· 2755 2810 if (atsru->include_all) 2756 2811 goto out; 2757 2812 } 2758 - ret = 0; 2813 + supported = false; 2759 2814 out: 2760 2815 rcu_read_unlock(); 2761 2816 2762 - return ret; 2817 + return supported; 2763 2818 } 2764 2819 2765 2820 int dmar_iommu_notify_scope_dev(struct dmar_pci_notify_info *info) ··· 2917 2972 struct device_attribute *attr, char *buf) 2918 2973 { 2919 2974 struct intel_iommu *iommu = dev_to_intel_iommu(dev); 2920 - return sysfs_emit(buf, "%d\n", 2921 - bitmap_weight(iommu->domain_ids, 2922 - cap_ndoms(iommu->cap))); 2975 + unsigned int count = 0; 2976 + int id; 2977 + 2978 + for (id = 0; id < cap_ndoms(iommu->cap); id++) 2979 + if (ida_exists(&iommu->domain_ida, id)) 2980 + count++; 2981 + 2982 + return sysfs_emit(buf, "%d\n", count); 2923 2983 } 2924 2984 static DEVICE_ATTR_RO(domains_used); 2925 2985 ··· 3207 3257 struct intel_iommu *iommu = info->iommu; 3208 3258 unsigned long flags; 3209 3259 3260 + /* Device in DMA blocking state. Noting to do. */ 3261 + if (!info->domain_attached) 3262 + return; 3263 + 3210 3264 if (info->domain) 3211 3265 cache_tag_unassign_domain(info->domain, dev, IOMMU_NO_PASID); 3212 3266 ··· 3221 3267 else 3222 3268 domain_context_clear(info); 3223 3269 } 3270 + 3271 + /* Device now in DMA blocking state. */ 3272 + info->domain_attached = false; 3224 3273 3225 3274 if (!info->domain) 3226 3275 return; ··· 3239 3282 static int blocking_domain_attach_dev(struct iommu_domain *domain, 3240 3283 struct device *dev) 3241 3284 { 3285 + struct device_domain_info *info = dev_iommu_priv_get(dev); 3286 + 3287 + iopf_for_domain_remove(info->domain ? &info->domain->domain : NULL, dev); 3242 3288 device_block_translation(dev); 3243 3289 return 0; 3244 3290 } ··· 3320 3360 domain->domain.geometry.aperture_end = __DOMAIN_MAX_ADDR(domain->gaw); 3321 3361 3322 3362 /* always allocate the top pgd */ 3323 - domain->pgd = iommu_alloc_page_node(domain->nid, GFP_KERNEL); 3363 + domain->pgd = iommu_alloc_pages_node_sz(domain->nid, GFP_KERNEL, SZ_4K); 3324 3364 if (!domain->pgd) { 3325 3365 kfree(domain); 3326 3366 return ERR_PTR(-ENOMEM); ··· 3452 3492 if (ret) 3453 3493 return ret; 3454 3494 3455 - return dmar_domain_attach_device(to_dmar_domain(domain), dev); 3495 + ret = iopf_for_domain_set(domain, dev); 3496 + if (ret) 3497 + return ret; 3498 + 3499 + ret = dmar_domain_attach_device(to_dmar_domain(domain), dev); 3500 + if (ret) 3501 + iopf_for_domain_remove(domain, dev); 3502 + 3503 + return ret; 3456 3504 } 3457 3505 3458 3506 static int intel_iommu_map(struct iommu_domain *domain, ··· 3571 3603 struct iommu_iotlb_gather *gather) 3572 3604 { 3573 3605 cache_tag_flush_range(to_dmar_domain(domain), gather->start, 3574 - gather->end, list_empty(&gather->freelist)); 3606 + gather->end, 3607 + iommu_pages_list_empty(&gather->freelist)); 3575 3608 iommu_put_pages_list(&gather->freelist); 3576 3609 } 3577 3610 ··· 3887 3918 if (!info->pri_enabled) 3888 3919 return -ENODEV; 3889 3920 3921 + /* pri_enabled is protected by the group mutex. */ 3922 + iommu_group_mutex_assert(dev); 3890 3923 if (info->iopf_refcount) { 3891 3924 info->iopf_refcount++; 3892 3925 return 0; ··· 3911 3940 if (WARN_ON(!info->pri_enabled || !info->iopf_refcount)) 3912 3941 return; 3913 3942 3943 + iommu_group_mutex_assert(dev); 3914 3944 if (--info->iopf_refcount) 3915 3945 return; 3916 3946 3917 3947 iopf_queue_remove_device(iommu->iopf_queue, dev); 3918 - } 3919 - 3920 - static int 3921 - intel_iommu_dev_enable_feat(struct device *dev, enum iommu_dev_features feat) 3922 - { 3923 - switch (feat) { 3924 - case IOMMU_DEV_FEAT_IOPF: 3925 - return intel_iommu_enable_iopf(dev); 3926 - 3927 - case IOMMU_DEV_FEAT_SVA: 3928 - return 0; 3929 - 3930 - default: 3931 - return -ENODEV; 3932 - } 3933 - } 3934 - 3935 - static int 3936 - intel_iommu_dev_disable_feat(struct device *dev, enum iommu_dev_features feat) 3937 - { 3938 - switch (feat) { 3939 - case IOMMU_DEV_FEAT_IOPF: 3940 - intel_iommu_disable_iopf(dev); 3941 - return 0; 3942 - 3943 - case IOMMU_DEV_FEAT_SVA: 3944 - return 0; 3945 - 3946 - default: 3947 - return -ENODEV; 3948 - } 3949 3948 } 3950 3949 3951 3950 static bool intel_iommu_is_attach_deferred(struct device *dev) ··· 3991 4050 { 3992 4051 struct device_domain_info *info = dev_iommu_priv_get(dev); 3993 4052 4053 + iopf_for_domain_remove(old, dev); 3994 4054 intel_pasid_tear_down_entry(info->iommu, dev, pasid, false); 3995 4055 domain_remove_dev_pasid(old, dev, pasid); 3996 4056 ··· 4065 4123 if (IS_ERR(dev_pasid)) 4066 4124 return PTR_ERR(dev_pasid); 4067 4125 4126 + ret = iopf_for_domain_replace(domain, old, dev); 4127 + if (ret) 4128 + goto out_remove_dev_pasid; 4129 + 4068 4130 if (dmar_domain->use_first_level) 4069 4131 ret = domain_setup_first_level(iommu, dmar_domain, 4070 4132 dev, pasid, old); ··· 4076 4130 ret = domain_setup_second_level(iommu, dmar_domain, 4077 4131 dev, pasid, old); 4078 4132 if (ret) 4079 - goto out_remove_dev_pasid; 4133 + goto out_unwind_iopf; 4080 4134 4081 4135 domain_remove_dev_pasid(old, dev, pasid); 4082 4136 ··· 4084 4138 4085 4139 return 0; 4086 4140 4141 + out_unwind_iopf: 4142 + iopf_for_domain_replace(old, domain, dev); 4087 4143 out_remove_dev_pasid: 4088 4144 domain_remove_dev_pasid(domain, dev, pasid); 4089 4145 return ret; ··· 4300 4352 if (dev_is_real_dma_subdevice(dev)) 4301 4353 return 0; 4302 4354 4355 + /* 4356 + * No PRI support with the global identity domain. No need to enable or 4357 + * disable PRI in this path as the iommu has been put in the blocking 4358 + * state. 4359 + */ 4303 4360 if (sm_supported(iommu)) 4304 4361 ret = intel_pasid_setup_pass_through(iommu, dev, IOMMU_NO_PASID); 4305 4362 else 4306 4363 ret = device_setup_pass_through(dev); 4364 + 4365 + if (!ret) 4366 + info->domain_attached = true; 4307 4367 4308 4368 return ret; 4309 4369 } ··· 4327 4371 if (!pasid_supported(iommu) || dev_is_real_dma_subdevice(dev)) 4328 4372 return -EOPNOTSUPP; 4329 4373 4330 - ret = domain_setup_passthrough(iommu, dev, pasid, old); 4374 + ret = iopf_for_domain_replace(domain, old, dev); 4331 4375 if (ret) 4332 4376 return ret; 4377 + 4378 + ret = domain_setup_passthrough(iommu, dev, pasid, old); 4379 + if (ret) { 4380 + iopf_for_domain_replace(old, domain, dev); 4381 + return ret; 4382 + } 4333 4383 4334 4384 domain_remove_dev_pasid(old, dev, pasid); 4335 4385 return 0; ··· 4363 4401 .release_device = intel_iommu_release_device, 4364 4402 .get_resv_regions = intel_iommu_get_resv_regions, 4365 4403 .device_group = intel_iommu_device_group, 4366 - .dev_enable_feat = intel_iommu_dev_enable_feat, 4367 - .dev_disable_feat = intel_iommu_dev_disable_feat, 4368 4404 .is_attach_deferred = intel_iommu_is_attach_deferred, 4369 4405 .def_domain_type = device_def_domain_type, 4370 4406 .pgsize_bitmap = SZ_4K,

+54 -8

drivers/iommu/intel/iommu.h

··· 493 493 494 494 /* Page Request Queue depth */ 495 495 #define PRQ_ORDER 4 496 - #define PRQ_RING_MASK ((0x1000 << PRQ_ORDER) - 0x20) 497 - #define PRQ_DEPTH ((0x1000 << PRQ_ORDER) >> 5) 496 + #define PRQ_SIZE (SZ_4K << PRQ_ORDER) 497 + #define PRQ_RING_MASK (PRQ_SIZE - 0x20) 498 + #define PRQ_DEPTH (PRQ_SIZE >> 5) 498 499 499 500 struct dmar_pci_notify_info; 500 501 501 502 #ifdef CONFIG_IRQ_REMAP 502 - /* 1MB - maximum possible interrupt remapping table size */ 503 - #define INTR_REMAP_PAGE_ORDER 8 504 503 #define INTR_REMAP_TABLE_REG_SIZE 0xf 505 504 #define INTR_REMAP_TABLE_REG_SIZE_MASK 0xf 506 505 ··· 721 722 unsigned char name[16]; /* Device Name */ 722 723 723 724 #ifdef CONFIG_INTEL_IOMMU 724 - unsigned long *domain_ids; /* bitmap of domains */ 725 + /* mutex to protect domain_ida */ 726 + struct mutex did_lock; 727 + struct ida domain_ida; /* domain id allocator */ 725 728 unsigned long *copied_tables; /* bitmap of copied tables */ 726 729 spinlock_t lock; /* protect context, domain ids */ 727 730 struct root_entry *root_entry; /* virtual address */ ··· 774 773 u8 ats_supported:1; 775 774 u8 ats_enabled:1; 776 775 u8 dtlb_extra_inval:1; /* Quirk for devices need extra flush */ 776 + u8 domain_attached:1; /* Device has domain attached */ 777 777 u8 ats_qdep; 778 778 unsigned int iopf_refcount; 779 779 struct device *dev; /* it's NULL for PCIe-to-PCI bridge */ ··· 811 809 } 812 810 813 811 /* 814 - * Domain ID reserved for pasid entries programmed for first-level 815 - * only and pass-through transfer modes. 812 + * Domain ID 0 and 1 are reserved: 813 + * 814 + * If Caching mode is set, then invalid translations are tagged 815 + * with domain-id 0, hence we need to pre-allocate it. We also 816 + * use domain-id 0 as a marker for non-allocated domain-id, so 817 + * make sure it is not used for a real domain. 818 + * 819 + * Vt-d spec rev3.0 (section 6.2.3.1) requires that each pasid 820 + * entry for first-level or pass-through translation modes should 821 + * be programmed with a domain id different from those used for 822 + * second-level or nested translation. We reserve a domain id for 823 + * this purpose. This domain id is also used for identity domain 824 + * in legacy mode. 816 825 */ 817 826 #define FLPT_DEFAULT_DID 1 818 - #define NUM_RESERVED_DID 2 827 + #define IDA_START_DID 2 819 828 820 829 /* Retrieve the domain ID which has allocated to the domain */ 821 830 static inline u16 ··· 1310 1297 1311 1298 int intel_iommu_enable_iopf(struct device *dev); 1312 1299 void intel_iommu_disable_iopf(struct device *dev); 1300 + 1301 + static inline int iopf_for_domain_set(struct iommu_domain *domain, 1302 + struct device *dev) 1303 + { 1304 + if (!domain || !domain->iopf_handler) 1305 + return 0; 1306 + 1307 + return intel_iommu_enable_iopf(dev); 1308 + } 1309 + 1310 + static inline void iopf_for_domain_remove(struct iommu_domain *domain, 1311 + struct device *dev) 1312 + { 1313 + if (!domain || !domain->iopf_handler) 1314 + return; 1315 + 1316 + intel_iommu_disable_iopf(dev); 1317 + } 1318 + 1319 + static inline int iopf_for_domain_replace(struct iommu_domain *new, 1320 + struct iommu_domain *old, 1321 + struct device *dev) 1322 + { 1323 + int ret; 1324 + 1325 + ret = iopf_for_domain_set(new, dev); 1326 + if (ret) 1327 + return ret; 1328 + 1329 + iopf_for_domain_remove(old, dev); 1330 + 1331 + return 0; 1332 + } 1313 1333 1314 1334 #ifdef CONFIG_INTEL_IOMMU_SVM 1315 1335 void intel_svm_check(struct intel_iommu *iommu);

+6 -6

drivers/iommu/intel/irq_remapping.c

··· 530 530 if (!ir_table) 531 531 return -ENOMEM; 532 532 533 - ir_table_base = iommu_alloc_pages_node(iommu->node, GFP_KERNEL, 534 - INTR_REMAP_PAGE_ORDER); 533 + /* 1MB - maximum possible interrupt remapping table size */ 534 + ir_table_base = 535 + iommu_alloc_pages_node_sz(iommu->node, GFP_KERNEL, SZ_1M); 535 536 if (!ir_table_base) { 536 - pr_err("IR%d: failed to allocate pages of order %d\n", 537 - iommu->seq_id, INTR_REMAP_PAGE_ORDER); 537 + pr_err("IR%d: failed to allocate 1M of pages\n", iommu->seq_id); 538 538 goto out_free_table; 539 539 } 540 540 ··· 612 612 out_free_bitmap: 613 613 bitmap_free(bitmap); 614 614 out_free_pages: 615 - iommu_free_pages(ir_table_base, INTR_REMAP_PAGE_ORDER); 615 + iommu_free_pages(ir_table_base); 616 616 out_free_table: 617 617 kfree(ir_table); 618 618 ··· 633 633 irq_domain_free_fwnode(fn); 634 634 iommu->ir_domain = NULL; 635 635 } 636 - iommu_free_pages(iommu->ir_table->base, INTR_REMAP_PAGE_ORDER); 636 + iommu_free_pages(iommu->ir_table->base); 637 637 bitmap_free(iommu->ir_table->bitmap); 638 638 kfree(iommu->ir_table); 639 639 iommu->ir_table = NULL;

+17 -5

drivers/iommu/intel/nested.c

··· 27 27 unsigned long flags; 28 28 int ret = 0; 29 29 30 - if (info->domain) 31 - device_block_translation(dev); 30 + device_block_translation(dev); 32 31 33 32 if (iommu->agaw < dmar_domain->s2_domain->agaw) { 34 33 dev_err_ratelimited(dev, "Adjusted guest address width not compatible\n"); ··· 55 56 if (ret) 56 57 goto detach_iommu; 57 58 58 - ret = intel_pasid_setup_nested(iommu, dev, 59 - IOMMU_NO_PASID, dmar_domain); 59 + ret = iopf_for_domain_set(domain, dev); 60 60 if (ret) 61 61 goto unassign_tag; 62 62 63 + ret = intel_pasid_setup_nested(iommu, dev, 64 + IOMMU_NO_PASID, dmar_domain); 65 + if (ret) 66 + goto disable_iopf; 67 + 63 68 info->domain = dmar_domain; 69 + info->domain_attached = true; 64 70 spin_lock_irqsave(&dmar_domain->lock, flags); 65 71 list_add(&info->link, &dmar_domain->devices); 66 72 spin_unlock_irqrestore(&dmar_domain->lock, flags); 67 73 68 74 return 0; 75 + disable_iopf: 76 + iopf_for_domain_remove(domain, dev); 69 77 unassign_tag: 70 78 cache_tag_unassign_domain(dmar_domain, dev, IOMMU_NO_PASID); 71 79 detach_iommu: ··· 172 166 if (IS_ERR(dev_pasid)) 173 167 return PTR_ERR(dev_pasid); 174 168 175 - ret = domain_setup_nested(iommu, dmar_domain, dev, pasid, old); 169 + ret = iopf_for_domain_replace(domain, old, dev); 176 170 if (ret) 177 171 goto out_remove_dev_pasid; 172 + 173 + ret = domain_setup_nested(iommu, dmar_domain, dev, pasid, old); 174 + if (ret) 175 + goto out_unwind_iopf; 178 176 179 177 domain_remove_dev_pasid(old, dev, pasid); 180 178 181 179 return 0; 182 180 181 + out_unwind_iopf: 182 + iopf_for_domain_replace(old, domain, dev); 183 183 out_remove_dev_pasid: 184 184 domain_remove_dev_pasid(domain, dev, pasid); 185 185 return ret;

+7 -6

drivers/iommu/intel/pasid.c

··· 60 60 61 61 size = max_pasid >> (PASID_PDE_SHIFT - 3); 62 62 order = size ? get_order(size) : 0; 63 - dir = iommu_alloc_pages_node(info->iommu->node, GFP_KERNEL, order); 63 + dir = iommu_alloc_pages_node_sz(info->iommu->node, GFP_KERNEL, 64 + 1 << (order + PAGE_SHIFT)); 64 65 if (!dir) { 65 66 kfree(pasid_table); 66 67 return -ENOMEM; 67 68 } 68 69 69 70 pasid_table->table = dir; 70 - pasid_table->order = order; 71 71 pasid_table->max_pasid = 1 << (order + PAGE_SHIFT + 3); 72 72 info->pasid_table = pasid_table; 73 73 ··· 97 97 max_pde = pasid_table->max_pasid >> PASID_PDE_SHIFT; 98 98 for (i = 0; i < max_pde; i++) { 99 99 table = get_pasid_table_from_pde(&dir[i]); 100 - iommu_free_page(table); 100 + iommu_free_pages(table); 101 101 } 102 102 103 - iommu_free_pages(pasid_table->table, pasid_table->order); 103 + iommu_free_pages(pasid_table->table); 104 104 kfree(pasid_table); 105 105 } 106 106 ··· 148 148 if (!entries) { 149 149 u64 tmp; 150 150 151 - entries = iommu_alloc_page_node(info->iommu->node, GFP_ATOMIC); 151 + entries = iommu_alloc_pages_node_sz(info->iommu->node, 152 + GFP_ATOMIC, SZ_4K); 152 153 if (!entries) 153 154 return NULL; 154 155 ··· 162 161 tmp = 0ULL; 163 162 if (!try_cmpxchg64(&dir[dir_index].val, &tmp, 164 163 (u64)virt_to_phys(entries) | PASID_PTE_PRESENT)) { 165 - iommu_free_page(entries); 164 + iommu_free_pages(entries); 166 165 goto retry; 167 166 } 168 167 if (!ecap_coherent(info->iommu->ecap)) {

-1

drivers/iommu/intel/pasid.h

··· 47 47 /* The representative of a PASID table */ 48 48 struct pasid_table { 49 49 void *table; /* pasid table pointer */ 50 - int order; /* page order of pasid table */ 51 50 u32 max_pasid; /* max pasid */ 52 51 }; 53 52

+4 -3

drivers/iommu/intel/prq.c

··· 290 290 struct iopf_queue *iopfq; 291 291 int irq, ret; 292 292 293 - iommu->prq = iommu_alloc_pages_node(iommu->node, GFP_KERNEL, PRQ_ORDER); 293 + iommu->prq = 294 + iommu_alloc_pages_node_sz(iommu->node, GFP_KERNEL, PRQ_SIZE); 294 295 if (!iommu->prq) { 295 296 pr_warn("IOMMU: %s: Failed to allocate page request queue\n", 296 297 iommu->name); ··· 341 340 dmar_free_hwirq(irq); 342 341 iommu->pr_irq = 0; 343 342 free_prq: 344 - iommu_free_pages(iommu->prq, PRQ_ORDER); 343 + iommu_free_pages(iommu->prq); 345 344 iommu->prq = NULL; 346 345 347 346 return ret; ··· 364 363 iommu->iopf_queue = NULL; 365 364 } 366 365 367 - iommu_free_pages(iommu->prq, PRQ_ORDER); 366 + iommu_free_pages(iommu->prq); 368 367 iommu->prq = NULL; 369 368 370 369 return 0;

+7 -2

drivers/iommu/intel/svm.c

··· 164 164 if (IS_ERR(dev_pasid)) 165 165 return PTR_ERR(dev_pasid); 166 166 167 + ret = iopf_for_domain_replace(domain, old, dev); 168 + if (ret) 169 + goto out_remove_dev_pasid; 170 + 167 171 /* Setup the pasid table: */ 168 172 sflags = cpu_feature_enabled(X86_FEATURE_LA57) ? PASID_FLAG_FL5LP : 0; 169 173 ret = __domain_setup_first_level(iommu, dev, pasid, 170 174 FLPT_DEFAULT_DID, mm->pgd, 171 175 sflags, old); 172 176 if (ret) 173 - goto out_remove_dev_pasid; 177 + goto out_unwind_iopf; 174 178 175 179 domain_remove_dev_pasid(old, dev, pasid); 176 180 177 181 return 0; 178 - 182 + out_unwind_iopf: 183 + iopf_for_domain_replace(old, domain, dev); 179 184 out_remove_dev_pasid: 180 185 domain_remove_dev_pasid(domain, dev, pasid); 181 186 return ret;

+34 -24

drivers/iommu/io-pgtable-arm.c

··· 13 13 #include <linux/bitops.h> 14 14 #include <linux/io-pgtable.h> 15 15 #include <linux/kernel.h> 16 + #include <linux/device/faux.h> 16 17 #include <linux/sizes.h> 17 18 #include <linux/slab.h> 18 19 #include <linux/types.h> ··· 252 251 (data->start_level == 1) && (oas == 40); 253 252 } 254 253 255 - static bool selftest_running = false; 256 - 257 254 static dma_addr_t __arm_lpae_dma_addr(void *pages) 258 255 { 259 256 return (dma_addr_t)virt_to_phys(pages); ··· 262 263 void *cookie) 263 264 { 264 265 struct device *dev = cfg->iommu_dev; 265 - int order = get_order(size); 266 + size_t alloc_size; 266 267 dma_addr_t dma; 267 268 void *pages; 268 269 269 - VM_BUG_ON((gfp & __GFP_HIGHMEM)); 270 - 270 + /* 271 + * For very small starting-level translation tables the HW requires a 272 + * minimum alignment of at least 64 to cover all cases. 273 + */ 274 + alloc_size = max(size, 64); 271 275 if (cfg->alloc) 272 - pages = cfg->alloc(cookie, size, gfp); 276 + pages = cfg->alloc(cookie, alloc_size, gfp); 273 277 else 274 - pages = iommu_alloc_pages_node(dev_to_node(dev), gfp, order); 278 + pages = iommu_alloc_pages_node_sz(dev_to_node(dev), gfp, 279 + alloc_size); 275 280 276 281 if (!pages) 277 282 return NULL; ··· 303 300 if (cfg->free) 304 301 cfg->free(cookie, pages, size); 305 302 else 306 - iommu_free_pages(pages, order); 303 + iommu_free_pages(pages); 307 304 308 305 return NULL; 309 306 } ··· 319 316 if (cfg->free) 320 317 cfg->free(cookie, pages, size); 321 318 else 322 - iommu_free_pages(pages, get_order(size)); 319 + iommu_free_pages(pages); 323 320 } 324 321 325 322 static void __arm_lpae_sync_pte(arm_lpae_iopte *ptep, int num_entries, ··· 374 371 for (i = 0; i < num_entries; i++) 375 372 if (iopte_leaf(ptep[i], lvl, data->iop.fmt)) { 376 373 /* We require an unmap first */ 377 - WARN_ON(!selftest_running); 374 + WARN_ON(!(data->iop.cfg.quirks & IO_PGTABLE_QUIRK_NO_WARN)); 378 375 return -EEXIST; 379 376 } else if (iopte_type(ptep[i]) == ARM_LPAE_PTE_TYPE_TABLE) { 380 377 /* ··· 476 473 cptep = iopte_deref(pte, data); 477 474 } else if (pte) { 478 475 /* We require an unmap first */ 479 - WARN_ON(!selftest_running); 476 + WARN_ON(!(cfg->quirks & IO_PGTABLE_QUIRK_NO_WARN)); 480 477 return -EEXIST; 481 478 } 482 479 ··· 644 641 unmap_idx_start = ARM_LPAE_LVL_IDX(iova, lvl, data); 645 642 ptep += unmap_idx_start; 646 643 pte = READ_ONCE(*ptep); 647 - if (WARN_ON(!pte)) 648 - return 0; 644 + if (!pte) { 645 + WARN_ON(!(data->iop.cfg.quirks & IO_PGTABLE_QUIRK_NO_WARN)); 646 + return -ENOENT; 647 + } 649 648 650 649 /* If the size matches this level, we're in the right place */ 651 650 if (size == ARM_LPAE_BLOCK_SIZE(lvl, data)) { ··· 657 652 /* Find and handle non-leaf entries */ 658 653 for (i = 0; i < num_entries; i++) { 659 654 pte = READ_ONCE(ptep[i]); 660 - if (WARN_ON(!pte)) 655 + if (!pte) { 656 + WARN_ON(!(data->iop.cfg.quirks & IO_PGTABLE_QUIRK_NO_WARN)); 661 657 break; 658 + } 662 659 663 660 if (!iopte_leaf(pte, lvl, iop->fmt)) { 664 661 __arm_lpae_clear_pte(&ptep[i], &iop->cfg, 1); ··· 975 968 if (cfg->quirks & ~(IO_PGTABLE_QUIRK_ARM_NS | 976 969 IO_PGTABLE_QUIRK_ARM_TTBR1 | 977 970 IO_PGTABLE_QUIRK_ARM_OUTER_WBWA | 978 - IO_PGTABLE_QUIRK_ARM_HD)) 971 + IO_PGTABLE_QUIRK_ARM_HD | 972 + IO_PGTABLE_QUIRK_NO_WARN)) 979 973 return NULL; 980 974 981 975 data = arm_lpae_alloc_pgtable(cfg); ··· 1077 1069 struct arm_lpae_io_pgtable *data; 1078 1070 typeof(&cfg->arm_lpae_s2_cfg.vtcr) vtcr = &cfg->arm_lpae_s2_cfg.vtcr; 1079 1071 1080 - if (cfg->quirks & ~(IO_PGTABLE_QUIRK_ARM_S2FWB)) 1072 + if (cfg->quirks & ~(IO_PGTABLE_QUIRK_ARM_S2FWB | 1073 + IO_PGTABLE_QUIRK_NO_WARN)) 1081 1074 return NULL; 1082 1075 1083 1076 data = arm_lpae_alloc_pgtable(cfg); ··· 1319 1310 #define __FAIL(ops, i) ({ \ 1320 1311 WARN(1, "selftest: test failed for fmt idx %d\n", (i)); \ 1321 1312 arm_lpae_dump_ops(ops); \ 1322 - selftest_running = false; \ 1323 1313 -EFAULT; \ 1324 1314 }) 1325 1315 ··· 1333 1325 unsigned long iova; 1334 1326 size_t size, mapped; 1335 1327 struct io_pgtable_ops *ops; 1336 - 1337 - selftest_running = true; 1338 1328 1339 1329 for (i = 0; i < ARRAY_SIZE(fmts); ++i) { 1340 1330 cfg_cookie = cfg; ··· 1422 1416 free_io_pgtable_ops(ops); 1423 1417 } 1424 1418 1425 - selftest_running = false; 1426 1419 return 0; 1427 1420 } 1428 1421 ··· 1438 1433 }; 1439 1434 1440 1435 int i, j, k, pass = 0, fail = 0; 1441 - struct device dev; 1436 + struct faux_device *dev; 1442 1437 struct io_pgtable_cfg cfg = { 1443 1438 .tlb = &dummy_tlb_ops, 1444 1439 .coherent_walk = true, 1445 - .iommu_dev = &dev, 1440 + .quirks = IO_PGTABLE_QUIRK_NO_WARN, 1446 1441 }; 1447 1442 1448 - /* __arm_lpae_alloc_pages() merely needs dev_to_node() to work */ 1449 - set_dev_node(&dev, NUMA_NO_NODE); 1443 + dev = faux_device_create("io-pgtable-test", NULL, 0); 1444 + if (!dev) 1445 + return -ENOMEM; 1446 + 1447 + cfg.iommu_dev = &dev->dev; 1450 1448 1451 1449 for (i = 0; i < ARRAY_SIZE(pgsize); ++i) { 1452 1450 for (j = 0; j < ARRAY_SIZE(address_size); ++j) { ··· 1469 1461 } 1470 1462 1471 1463 pr_info("selftest: completed with %d PASS %d FAIL\n", pass, fail); 1464 + faux_device_destroy(dev); 1465 + 1472 1466 return fail ? -EFAULT : 0; 1473 1467 } 1474 1468 subsys_initcall(arm_lpae_do_selftests);

+7 -16

drivers/iommu/io-pgtable-dart.c

··· 107 107 return paddr; 108 108 } 109 109 110 - static void *__dart_alloc_pages(size_t size, gfp_t gfp) 111 - { 112 - int order = get_order(size); 113 - 114 - VM_BUG_ON((gfp & __GFP_HIGHMEM)); 115 - return iommu_alloc_pages(gfp, order); 116 - } 117 - 118 110 static int dart_init_pte(struct dart_io_pgtable *data, 119 111 unsigned long iova, phys_addr_t paddr, 120 112 dart_iopte prot, int num_entries, ··· 248 256 249 257 /* no L2 table present */ 250 258 if (!pte) { 251 - cptep = __dart_alloc_pages(tblsz, gfp); 259 + cptep = iommu_alloc_pages_sz(gfp, tblsz); 252 260 if (!cptep) 253 261 return -ENOMEM; 254 262 255 263 pte = dart_install_table(cptep, ptep, 0, data); 256 264 if (pte) 257 - iommu_free_pages(cptep, get_order(tblsz)); 265 + iommu_free_pages(cptep); 258 266 259 267 /* L2 table is present (now) */ 260 268 pte = READ_ONCE(*ptep); ··· 405 413 cfg->apple_dart_cfg.n_ttbrs = 1 << data->tbl_bits; 406 414 407 415 for (i = 0; i < cfg->apple_dart_cfg.n_ttbrs; ++i) { 408 - data->pgd[i] = __dart_alloc_pages(DART_GRANULE(data), GFP_KERNEL); 416 + data->pgd[i] = 417 + iommu_alloc_pages_sz(GFP_KERNEL, DART_GRANULE(data)); 409 418 if (!data->pgd[i]) 410 419 goto out_free_data; 411 420 cfg->apple_dart_cfg.ttbr[i] = virt_to_phys(data->pgd[i]); ··· 416 423 417 424 out_free_data: 418 425 while (--i >= 0) { 419 - iommu_free_pages(data->pgd[i], 420 - get_order(DART_GRANULE(data))); 426 + iommu_free_pages(data->pgd[i]); 421 427 } 422 428 kfree(data); 423 429 return NULL; ··· 425 433 static void apple_dart_free_pgtable(struct io_pgtable *iop) 426 434 { 427 435 struct dart_io_pgtable *data = io_pgtable_to_data(iop); 428 - int order = get_order(DART_GRANULE(data)); 429 436 dart_iopte *ptep, *end; 430 437 int i; 431 438 ··· 436 445 dart_iopte pte = *ptep++; 437 446 438 447 if (pte) 439 - iommu_free_pages(iopte_deref(pte, data), order); 448 + iommu_free_pages(iopte_deref(pte, data)); 440 449 } 441 - iommu_free_pages(data->pgd[i], order); 450 + iommu_free_pages(data->pgd[i]); 442 451 } 443 452 444 453 kfree(data);

+119

drivers/iommu/iommu-pages.c

··· 1 + // SPDX-License-Identifier: GPL-2.0-only 2 + /* 3 + * Copyright (c) 2024, Google LLC. 4 + * Pasha Tatashin <pasha.tatashin@soleen.com> 5 + */ 6 + #include "iommu-pages.h" 7 + #include <linux/gfp.h> 8 + #include <linux/mm.h> 9 + 10 + #define IOPTDESC_MATCH(pg_elm, elm) \ 11 + static_assert(offsetof(struct page, pg_elm) == \ 12 + offsetof(struct ioptdesc, elm)) 13 + IOPTDESC_MATCH(flags, __page_flags); 14 + IOPTDESC_MATCH(lru, iopt_freelist_elm); /* Ensure bit 0 is clear */ 15 + IOPTDESC_MATCH(mapping, __page_mapping); 16 + IOPTDESC_MATCH(private, _private); 17 + IOPTDESC_MATCH(page_type, __page_type); 18 + IOPTDESC_MATCH(_refcount, __page_refcount); 19 + #ifdef CONFIG_MEMCG 20 + IOPTDESC_MATCH(memcg_data, memcg_data); 21 + #endif 22 + #undef IOPTDESC_MATCH 23 + static_assert(sizeof(struct ioptdesc) <= sizeof(struct page)); 24 + 25 + /** 26 + * iommu_alloc_pages_node_sz - Allocate a zeroed page of a given size from 27 + * specific NUMA node 28 + * @nid: memory NUMA node id 29 + * @gfp: buddy allocator flags 30 + * @size: Memory size to allocate, rounded up to a power of 2 31 + * 32 + * Returns the virtual address of the allocated page. The page must be freed 33 + * either by calling iommu_free_pages() or via iommu_put_pages_list(). The 34 + * returned allocation is round_up_pow_two(size) big, and is physically aligned 35 + * to its size. 36 + */ 37 + void *iommu_alloc_pages_node_sz(int nid, gfp_t gfp, size_t size) 38 + { 39 + unsigned long pgcnt; 40 + struct folio *folio; 41 + unsigned int order; 42 + 43 + /* This uses page_address() on the memory. */ 44 + if (WARN_ON(gfp & __GFP_HIGHMEM)) 45 + return NULL; 46 + 47 + /* 48 + * Currently sub page allocations result in a full page being returned. 49 + */ 50 + order = get_order(size); 51 + 52 + /* 53 + * __folio_alloc_node() does not handle NUMA_NO_NODE like 54 + * alloc_pages_node() did. 55 + */ 56 + if (nid == NUMA_NO_NODE) 57 + nid = numa_mem_id(); 58 + 59 + folio = __folio_alloc_node(gfp | __GFP_ZERO, order, nid); 60 + if (unlikely(!folio)) 61 + return NULL; 62 + 63 + /* 64 + * All page allocations that should be reported to as "iommu-pagetables" 65 + * to userspace must use one of the functions below. This includes 66 + * allocations of page-tables and other per-iommu_domain configuration 67 + * structures. 68 + * 69 + * This is necessary for the proper accounting as IOMMU state can be 70 + * rather large, i.e. multiple gigabytes in size. 71 + */ 72 + pgcnt = 1UL << order; 73 + mod_node_page_state(folio_pgdat(folio), NR_IOMMU_PAGES, pgcnt); 74 + lruvec_stat_mod_folio(folio, NR_SECONDARY_PAGETABLE, pgcnt); 75 + 76 + return folio_address(folio); 77 + } 78 + EXPORT_SYMBOL_GPL(iommu_alloc_pages_node_sz); 79 + 80 + static void __iommu_free_desc(struct ioptdesc *iopt) 81 + { 82 + struct folio *folio = ioptdesc_folio(iopt); 83 + const unsigned long pgcnt = 1UL << folio_order(folio); 84 + 85 + mod_node_page_state(folio_pgdat(folio), NR_IOMMU_PAGES, -pgcnt); 86 + lruvec_stat_mod_folio(folio, NR_SECONDARY_PAGETABLE, -pgcnt); 87 + folio_put(folio); 88 + } 89 + 90 + /** 91 + * iommu_free_pages - free pages 92 + * @virt: virtual address of the page to be freed. 93 + * 94 + * The page must have have been allocated by iommu_alloc_pages_node_sz() 95 + */ 96 + void iommu_free_pages(void *virt) 97 + { 98 + if (!virt) 99 + return; 100 + __iommu_free_desc(virt_to_ioptdesc(virt)); 101 + } 102 + EXPORT_SYMBOL_GPL(iommu_free_pages); 103 + 104 + /** 105 + * iommu_put_pages_list - free a list of pages. 106 + * @list: The list of pages to be freed 107 + * 108 + * Frees a list of pages allocated by iommu_alloc_pages_node_sz(). On return the 109 + * passed list is invalid, the caller must use IOMMU_PAGES_LIST_INIT to reinit 110 + * the list if it expects to use it again. 111 + */ 112 + void iommu_put_pages_list(struct iommu_pages_list *list) 113 + { 114 + struct ioptdesc *iopt, *tmp; 115 + 116 + list_for_each_entry_safe(iopt, tmp, &list->pages, iopt_freelist_elm) 117 + __iommu_free_desc(iopt); 118 + } 119 + EXPORT_SYMBOL_GPL(iommu_put_pages_list);

+63 -148

drivers/iommu/iommu-pages.h

··· 7 7 #ifndef __IOMMU_PAGES_H 8 8 #define __IOMMU_PAGES_H 9 9 10 - #include <linux/vmstat.h> 11 - #include <linux/gfp.h> 12 - #include <linux/mm.h> 10 + #include <linux/iommu.h> 13 11 14 - /* 15 - * All page allocations that should be reported to as "iommu-pagetables" to 16 - * userspace must use one of the functions below. This includes allocations of 17 - * page-tables and other per-iommu_domain configuration structures. 12 + /** 13 + * struct ioptdesc - Memory descriptor for IOMMU page tables 14 + * @iopt_freelist_elm: List element for a struct iommu_pages_list 18 15 * 19 - * This is necessary for the proper accounting as IOMMU state can be rather 20 - * large, i.e. multiple gigabytes in size. 16 + * This struct overlays struct page for now. Do not modify without a good 17 + * understanding of the issues. 21 18 */ 19 + struct ioptdesc { 20 + unsigned long __page_flags; 21 + 22 + struct list_head iopt_freelist_elm; 23 + unsigned long __page_mapping; 24 + pgoff_t __index; 25 + void *_private; 26 + 27 + unsigned int __page_type; 28 + atomic_t __page_refcount; 29 + #ifdef CONFIG_MEMCG 30 + unsigned long memcg_data; 31 + #endif 32 + }; 33 + 34 + static inline struct ioptdesc *folio_ioptdesc(struct folio *folio) 35 + { 36 + return (struct ioptdesc *)folio; 37 + } 38 + 39 + static inline struct folio *ioptdesc_folio(struct ioptdesc *iopt) 40 + { 41 + return (struct folio *)iopt; 42 + } 43 + 44 + static inline struct ioptdesc *virt_to_ioptdesc(void *virt) 45 + { 46 + return folio_ioptdesc(virt_to_folio(virt)); 47 + } 48 + 49 + void *iommu_alloc_pages_node_sz(int nid, gfp_t gfp, size_t size); 50 + void iommu_free_pages(void *virt); 51 + void iommu_put_pages_list(struct iommu_pages_list *list); 22 52 23 53 /** 24 - * __iommu_alloc_account - account for newly allocated page. 25 - * @page: head struct page of the page. 26 - * @order: order of the page 54 + * iommu_pages_list_add - add the page to a iommu_pages_list 55 + * @list: List to add the page to 56 + * @virt: Address returned from iommu_alloc_pages_node_sz() 27 57 */ 28 - static inline void __iommu_alloc_account(struct page *page, int order) 58 + static inline void iommu_pages_list_add(struct iommu_pages_list *list, 59 + void *virt) 29 60 { 30 - const long pgcnt = 1l << order; 31 - 32 - mod_node_page_state(page_pgdat(page), NR_IOMMU_PAGES, pgcnt); 33 - mod_lruvec_page_state(page, NR_SECONDARY_PAGETABLE, pgcnt); 61 + list_add_tail(&virt_to_ioptdesc(virt)->iopt_freelist_elm, &list->pages); 34 62 } 35 63 36 64 /** 37 - * __iommu_free_account - account a page that is about to be freed. 38 - * @page: head struct page of the page. 39 - * @order: order of the page 40 - */ 41 - static inline void __iommu_free_account(struct page *page, int order) 42 - { 43 - const long pgcnt = 1l << order; 44 - 45 - mod_node_page_state(page_pgdat(page), NR_IOMMU_PAGES, -pgcnt); 46 - mod_lruvec_page_state(page, NR_SECONDARY_PAGETABLE, -pgcnt); 47 - } 48 - 49 - /** 50 - * __iommu_alloc_pages - allocate a zeroed page of a given order. 51 - * @gfp: buddy allocator flags 52 - * @order: page order 65 + * iommu_pages_list_splice - Put all the pages in list from into list to 66 + * @from: Source list of pages 67 + * @to: Destination list of pages 53 68 * 54 - * returns the head struct page of the allocated page. 69 + * from must be re-initialized after calling this function if it is to be 70 + * used again. 55 71 */ 56 - static inline struct page *__iommu_alloc_pages(gfp_t gfp, int order) 72 + static inline void iommu_pages_list_splice(struct iommu_pages_list *from, 73 + struct iommu_pages_list *to) 57 74 { 58 - struct page *page; 59 - 60 - page = alloc_pages(gfp | __GFP_ZERO, order); 61 - if (unlikely(!page)) 62 - return NULL; 63 - 64 - __iommu_alloc_account(page, order); 65 - 66 - return page; 75 + list_splice(&from->pages, &to->pages); 67 76 } 68 77 69 78 /** 70 - * __iommu_free_pages - free page of a given order 71 - * @page: head struct page of the page 72 - * @order: page order 79 + * iommu_pages_list_empty - True if the list is empty 80 + * @list: List to check 73 81 */ 74 - static inline void __iommu_free_pages(struct page *page, int order) 82 + static inline bool iommu_pages_list_empty(struct iommu_pages_list *list) 75 83 { 76 - if (!page) 77 - return; 78 - 79 - __iommu_free_account(page, order); 80 - __free_pages(page, order); 84 + return list_empty(&list->pages); 81 85 } 82 86 83 87 /** 84 - * iommu_alloc_pages_node - allocate a zeroed page of a given order from 85 - * specific NUMA node. 88 + * iommu_alloc_pages_sz - Allocate a zeroed page of a given size from 89 + * specific NUMA node 86 90 * @nid: memory NUMA node id 87 91 * @gfp: buddy allocator flags 88 - * @order: page order 92 + * @size: Memory size to allocate, this is rounded up to a power of 2 89 93 * 90 - * returns the virtual address of the allocated page 94 + * Returns the virtual address of the allocated page. 91 95 */ 92 - static inline void *iommu_alloc_pages_node(int nid, gfp_t gfp, int order) 96 + static inline void *iommu_alloc_pages_sz(gfp_t gfp, size_t size) 93 97 { 94 - struct page *page = alloc_pages_node(nid, gfp | __GFP_ZERO, order); 95 - 96 - if (unlikely(!page)) 97 - return NULL; 98 - 99 - __iommu_alloc_account(page, order); 100 - 101 - return page_address(page); 102 - } 103 - 104 - /** 105 - * iommu_alloc_pages - allocate a zeroed page of a given order 106 - * @gfp: buddy allocator flags 107 - * @order: page order 108 - * 109 - * returns the virtual address of the allocated page 110 - */ 111 - static inline void *iommu_alloc_pages(gfp_t gfp, int order) 112 - { 113 - struct page *page = __iommu_alloc_pages(gfp, order); 114 - 115 - if (unlikely(!page)) 116 - return NULL; 117 - 118 - return page_address(page); 119 - } 120 - 121 - /** 122 - * iommu_alloc_page_node - allocate a zeroed page at specific NUMA node. 123 - * @nid: memory NUMA node id 124 - * @gfp: buddy allocator flags 125 - * 126 - * returns the virtual address of the allocated page 127 - */ 128 - static inline void *iommu_alloc_page_node(int nid, gfp_t gfp) 129 - { 130 - return iommu_alloc_pages_node(nid, gfp, 0); 131 - } 132 - 133 - /** 134 - * iommu_alloc_page - allocate a zeroed page 135 - * @gfp: buddy allocator flags 136 - * 137 - * returns the virtual address of the allocated page 138 - */ 139 - static inline void *iommu_alloc_page(gfp_t gfp) 140 - { 141 - return iommu_alloc_pages(gfp, 0); 142 - } 143 - 144 - /** 145 - * iommu_free_pages - free page of a given order 146 - * @virt: virtual address of the page to be freed. 147 - * @order: page order 148 - */ 149 - static inline void iommu_free_pages(void *virt, int order) 150 - { 151 - if (!virt) 152 - return; 153 - 154 - __iommu_free_pages(virt_to_page(virt), order); 155 - } 156 - 157 - /** 158 - * iommu_free_page - free page 159 - * @virt: virtual address of the page to be freed. 160 - */ 161 - static inline void iommu_free_page(void *virt) 162 - { 163 - iommu_free_pages(virt, 0); 164 - } 165 - 166 - /** 167 - * iommu_put_pages_list - free a list of pages. 168 - * @page: the head of the lru list to be freed. 169 - * 170 - * There are no locking requirement for these pages, as they are going to be 171 - * put on a free list as soon as refcount reaches 0. Pages are put on this LRU 172 - * list once they are removed from the IOMMU page tables. However, they can 173 - * still be access through debugfs. 174 - */ 175 - static inline void iommu_put_pages_list(struct list_head *page) 176 - { 177 - while (!list_empty(page)) { 178 - struct page *p = list_entry(page->prev, struct page, lru); 179 - 180 - list_del(&p->lru); 181 - __iommu_free_account(p, 0); 182 - put_page(p); 183 - } 98 + return iommu_alloc_pages_node_sz(NUMA_NO_NODE, gfp, size); 184 99 } 185 100 186 101 #endif /* __IOMMU_PAGES_H */

+6 -12

drivers/iommu/iommu-sva.c

··· 63 63 * reference is taken. Caller must call iommu_sva_unbind_device() 64 64 * to release each reference. 65 65 * 66 - * iommu_dev_enable_feature(dev, IOMMU_DEV_FEAT_SVA) must be called first, to 67 - * initialize the required SVA features. 68 - * 69 66 * On error, returns an ERR_PTR value. 70 67 */ 71 68 struct iommu_sva *iommu_sva_bind_device(struct device *dev, struct mm_struct *mm) ··· 296 299 const struct iommu_ops *ops = dev_iommu_ops(dev); 297 300 struct iommu_domain *domain; 298 301 299 - if (ops->domain_alloc_sva) { 300 - domain = ops->domain_alloc_sva(dev, mm); 301 - if (IS_ERR(domain)) 302 - return domain; 303 - } else { 304 - domain = ops->domain_alloc(IOMMU_DOMAIN_SVA); 305 - if (!domain) 306 - return ERR_PTR(-ENOMEM); 307 - } 302 + if (!ops->domain_alloc_sva) 303 + return ERR_PTR(-EOPNOTSUPP); 304 + 305 + domain = ops->domain_alloc_sva(dev, mm); 306 + if (IS_ERR(domain)) 307 + return domain; 308 308 309 309 domain->type = IOMMU_DOMAIN_SVA; 310 310 domain->cookie_type = IOMMU_COOKIE_SVA;

+52 -55

drivers/iommu/iommu.c

··· 277 277 err = bus_iommu_probe(iommu_buses[i]); 278 278 if (err) 279 279 iommu_device_unregister(iommu); 280 + else 281 + WRITE_ONCE(iommu->ready, true); 280 282 return err; 281 283 } 282 284 EXPORT_SYMBOL_GPL(iommu_device_register); ··· 424 422 * is buried in the bus dma_configure path. Properly unpicking that is 425 423 * still a big job, so for now just invoke the whole thing. The device 426 424 * already having a driver bound means dma_configure has already run and 427 - * either found no IOMMU to wait for, or we're in its replay call right 428 - * now, so either way there's no point calling it again. 425 + * found no IOMMU to wait for, so there's no point calling it again. 429 426 */ 430 - if (!dev->driver && dev->bus->dma_configure) { 427 + if (!dev->iommu->fwspec && !dev->driver && dev->bus->dma_configure) { 431 428 mutex_unlock(&iommu_probe_device_lock); 432 429 dev->bus->dma_configure(dev); 433 430 mutex_lock(&iommu_probe_device_lock); 431 + /* If another instance finished the job for us, skip it */ 432 + if (!dev->iommu || dev->iommu_group) 433 + return -ENODEV; 434 434 } 435 435 /* 436 436 * At this point, relevant devices either now have a fwspec which will ··· 1633 1629 if (ops->identity_domain) 1634 1630 return ops->identity_domain; 1635 1631 1636 - /* Older drivers create the identity domain via ops->domain_alloc() */ 1637 - if (!ops->domain_alloc) 1632 + if (ops->domain_alloc_identity) { 1633 + domain = ops->domain_alloc_identity(dev); 1634 + if (IS_ERR(domain)) 1635 + return domain; 1636 + } else { 1638 1637 return ERR_PTR(-EOPNOTSUPP); 1639 - 1640 - domain = ops->domain_alloc(IOMMU_DOMAIN_IDENTITY); 1641 - if (IS_ERR(domain)) 1642 - return domain; 1643 - if (!domain) 1644 - return ERR_PTR(-ENOMEM); 1638 + } 1645 1639 1646 1640 iommu_domain_init(domain, IOMMU_DOMAIN_IDENTITY, ops); 1647 1641 return domain; ··· 2027 2025 domain = ops->domain_alloc_paging(dev); 2028 2026 else if (ops->domain_alloc_paging_flags) 2029 2027 domain = ops->domain_alloc_paging_flags(dev, flags, NULL); 2028 + #if IS_ENABLED(CONFIG_FSL_PAMU) 2030 2029 else if (ops->domain_alloc && !flags) 2031 2030 domain = ops->domain_alloc(IOMMU_DOMAIN_UNMANAGED); 2031 + #endif 2032 2032 else 2033 2033 return ERR_PTR(-EOPNOTSUPP); 2034 2034 ··· 2208 2204 return xa_tag_pointer(domain, IOMMU_PASID_ARRAY_DOMAIN); 2209 2205 } 2210 2206 2207 + static bool domain_iommu_ops_compatible(const struct iommu_ops *ops, 2208 + struct iommu_domain *domain) 2209 + { 2210 + if (domain->owner == ops) 2211 + return true; 2212 + 2213 + /* For static domains, owner isn't set. */ 2214 + if (domain == ops->blocked_domain || domain == ops->identity_domain) 2215 + return true; 2216 + 2217 + return false; 2218 + } 2219 + 2211 2220 static int __iommu_attach_group(struct iommu_domain *domain, 2212 2221 struct iommu_group *group) 2213 2222 { ··· 2231 2214 return -EBUSY; 2232 2215 2233 2216 dev = iommu_group_first_dev(group); 2234 - if (!dev_has_iommu(dev) || dev_iommu_ops(dev) != domain->owner) 2217 + if (!dev_has_iommu(dev) || 2218 + !domain_iommu_ops_compatible(dev_iommu_ops(dev), domain)) 2235 2219 return -EINVAL; 2236 2220 2237 2221 return __iommu_group_set_domain(group, domain); ··· 2413 2395 unsigned int pgsize_idx, pgsize_idx_next; 2414 2396 unsigned long pgsizes; 2415 2397 size_t offset, pgsize, pgsize_next; 2398 + size_t offset_end; 2416 2399 unsigned long addr_merge = paddr | iova; 2417 2400 2418 2401 /* Page sizes supported by the hardware and small enough for @size */ ··· 2454 2435 * If size is big enough to accommodate the larger page, reduce 2455 2436 * the number of smaller pages. 2456 2437 */ 2457 - if (offset + pgsize_next <= size) 2438 + if (!check_add_overflow(offset, pgsize_next, &offset_end) && 2439 + offset_end <= size) 2458 2440 size = offset; 2459 2441 2460 2442 out_set_count: ··· 2862 2842 } 2863 2843 EXPORT_SYMBOL_GPL(iommu_default_passthrough); 2864 2844 2865 - const struct iommu_ops *iommu_ops_from_fwnode(const struct fwnode_handle *fwnode) 2845 + static const struct iommu_device *iommu_from_fwnode(const struct fwnode_handle *fwnode) 2866 2846 { 2867 - const struct iommu_ops *ops = NULL; 2868 - struct iommu_device *iommu; 2847 + const struct iommu_device *iommu, *ret = NULL; 2869 2848 2870 2849 spin_lock(&iommu_device_lock); 2871 2850 list_for_each_entry(iommu, &iommu_device_list, list) 2872 2851 if (iommu->fwnode == fwnode) { 2873 - ops = iommu->ops; 2852 + ret = iommu; 2874 2853 break; 2875 2854 } 2876 2855 spin_unlock(&iommu_device_lock); 2877 - return ops; 2856 + return ret; 2857 + } 2858 + 2859 + const struct iommu_ops *iommu_ops_from_fwnode(const struct fwnode_handle *fwnode) 2860 + { 2861 + const struct iommu_device *iommu = iommu_from_fwnode(fwnode); 2862 + 2863 + return iommu ? iommu->ops : NULL; 2878 2864 } 2879 2865 2880 2866 int iommu_fwspec_init(struct device *dev, struct fwnode_handle *iommu_fwnode) 2881 2867 { 2882 - const struct iommu_ops *ops = iommu_ops_from_fwnode(iommu_fwnode); 2868 + const struct iommu_device *iommu = iommu_from_fwnode(iommu_fwnode); 2883 2869 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); 2884 2870 2885 - if (!ops) 2871 + if (!iommu) 2886 2872 return driver_deferred_probe_check_state(dev); 2873 + if (!dev->iommu && !READ_ONCE(iommu->ready)) 2874 + return -EPROBE_DEFER; 2887 2875 2888 2876 if (fwspec) 2889 - return ops == iommu_fwspec_ops(fwspec) ? 0 : -EINVAL; 2877 + return iommu->ops == iommu_fwspec_ops(fwspec) ? 0 : -EINVAL; 2890 2878 2891 2879 if (!dev_iommu_get(dev)) 2892 2880 return -ENOMEM; ··· 2947 2919 return 0; 2948 2920 } 2949 2921 EXPORT_SYMBOL_GPL(iommu_fwspec_add_ids); 2950 - 2951 - /* 2952 - * Per device IOMMU features. 2953 - */ 2954 - int iommu_dev_enable_feature(struct device *dev, enum iommu_dev_features feat) 2955 - { 2956 - if (dev_has_iommu(dev)) { 2957 - const struct iommu_ops *ops = dev_iommu_ops(dev); 2958 - 2959 - if (ops->dev_enable_feat) 2960 - return ops->dev_enable_feat(dev, feat); 2961 - } 2962 - 2963 - return -ENODEV; 2964 - } 2965 - EXPORT_SYMBOL_GPL(iommu_dev_enable_feature); 2966 - 2967 - /* 2968 - * The device drivers should do the necessary cleanups before calling this. 2969 - */ 2970 - int iommu_dev_disable_feature(struct device *dev, enum iommu_dev_features feat) 2971 - { 2972 - if (dev_has_iommu(dev)) { 2973 - const struct iommu_ops *ops = dev_iommu_ops(dev); 2974 - 2975 - if (ops->dev_disable_feat) 2976 - return ops->dev_disable_feat(dev, feat); 2977 - } 2978 - 2979 - return -EBUSY; 2980 - } 2981 - EXPORT_SYMBOL_GPL(iommu_dev_disable_feature); 2982 2922 2983 2923 /** 2984 2924 * iommu_setup_default_domain - Set the default_domain for the group ··· 3450 3454 !ops->blocked_domain->ops->set_dev_pasid) 3451 3455 return -EOPNOTSUPP; 3452 3456 3453 - if (ops != domain->owner || pasid == IOMMU_NO_PASID) 3457 + if (!domain_iommu_ops_compatible(ops, domain) || 3458 + pasid == IOMMU_NO_PASID) 3454 3459 return -EINVAL; 3455 3460 3456 3461 mutex_lock(&group->mutex); ··· 3533 3536 if (!domain->ops->set_dev_pasid) 3534 3537 return -EOPNOTSUPP; 3535 3538 3536 - if (dev_iommu_ops(dev) != domain->owner || 3539 + if (!domain_iommu_ops_compatible(dev_iommu_ops(dev), domain) || 3537 3540 pasid == IOMMU_NO_PASID || !handle) 3538 3541 return -EINVAL; 3539 3542

+29 -30

drivers/iommu/iommufd/device.c

··· 221 221 refcount_inc(&idev->obj.users); 222 222 /* igroup refcount moves into iommufd_device */ 223 223 idev->igroup = igroup; 224 - mutex_init(&idev->iopf_lock); 225 224 226 225 /* 227 226 * If the caller fails after this success it must call ··· 424 425 return 0; 425 426 } 426 427 428 + static bool iommufd_hwpt_compatible_device(struct iommufd_hw_pagetable *hwpt, 429 + struct iommufd_device *idev) 430 + { 431 + struct pci_dev *pdev; 432 + 433 + if (!hwpt->fault || !dev_is_pci(idev->dev)) 434 + return true; 435 + 436 + /* 437 + * Once we turn on PCI/PRI support for VF, the response failure code 438 + * should not be forwarded to the hardware due to PRI being a shared 439 + * resource between PF and VFs. There is no coordination for this 440 + * shared capability. This waits for a vPRI reset to recover. 441 + */ 442 + pdev = to_pci_dev(idev->dev); 443 + 444 + return (!pdev->is_virtfn || !pci_pri_supported(pdev)); 445 + } 446 + 427 447 static int iommufd_hwpt_attach_device(struct iommufd_hw_pagetable *hwpt, 428 448 struct iommufd_device *idev, 429 449 ioasid_t pasid) 430 450 { 431 451 struct iommufd_attach_handle *handle; 432 452 int rc; 453 + 454 + if (!iommufd_hwpt_compatible_device(hwpt, idev)) 455 + return -EINVAL; 433 456 434 457 rc = iommufd_hwpt_pasid_compat(hwpt, idev, pasid); 435 458 if (rc) ··· 461 440 if (!handle) 462 441 return -ENOMEM; 463 442 464 - if (hwpt->fault) { 465 - rc = iommufd_fault_iopf_enable(idev); 466 - if (rc) 467 - goto out_free_handle; 468 - } 469 - 470 443 handle->idev = idev; 471 444 if (pasid == IOMMU_NO_PASID) 472 445 rc = iommu_attach_group_handle(hwpt->domain, idev->igroup->group, ··· 469 454 rc = iommu_attach_device_pasid(hwpt->domain, idev->dev, pasid, 470 455 &handle->handle); 471 456 if (rc) 472 - goto out_disable_iopf; 457 + goto out_free_handle; 473 458 474 459 return 0; 475 460 476 - out_disable_iopf: 477 - if (hwpt->fault) 478 - iommufd_fault_iopf_disable(idev); 479 461 out_free_handle: 480 462 kfree(handle); 481 463 return rc; ··· 504 492 else 505 493 iommu_detach_device_pasid(hwpt->domain, idev->dev, pasid); 506 494 507 - if (hwpt->fault) { 508 - iommufd_auto_response_faults(hwpt, handle); 509 - iommufd_fault_iopf_disable(idev); 510 - } 495 + iommufd_auto_response_faults(hwpt, handle); 511 496 kfree(handle); 512 497 } 513 498 ··· 516 507 struct iommufd_attach_handle *handle, *old_handle; 517 508 int rc; 518 509 510 + if (!iommufd_hwpt_compatible_device(hwpt, idev)) 511 + return -EINVAL; 512 + 519 513 rc = iommufd_hwpt_pasid_compat(hwpt, idev, pasid); 520 514 if (rc) 521 515 return rc; ··· 529 517 if (!handle) 530 518 return -ENOMEM; 531 519 532 - if (hwpt->fault && !old->fault) { 533 - rc = iommufd_fault_iopf_enable(idev); 534 - if (rc) 535 - goto out_free_handle; 536 - } 537 - 538 520 handle->idev = idev; 539 521 if (pasid == IOMMU_NO_PASID) 540 522 rc = iommu_replace_group_handle(idev->igroup->group, ··· 537 531 rc = iommu_replace_device_pasid(hwpt->domain, idev->dev, 538 532 pasid, &handle->handle); 539 533 if (rc) 540 - goto out_disable_iopf; 534 + goto out_free_handle; 541 535 542 - if (old->fault) { 543 - iommufd_auto_response_faults(hwpt, old_handle); 544 - if (!hwpt->fault) 545 - iommufd_fault_iopf_disable(idev); 546 - } 536 + iommufd_auto_response_faults(hwpt, old_handle); 547 537 kfree(old_handle); 548 538 549 539 return 0; 550 540 551 - out_disable_iopf: 552 - if (hwpt->fault && !old->fault) 553 - iommufd_fault_iopf_disable(idev); 554 541 out_free_handle: 555 542 kfree(handle); 556 543 return rc;

+1 -47

drivers/iommu/iommufd/eventq.c

··· 9 9 #include <linux/iommufd.h> 10 10 #include <linux/module.h> 11 11 #include <linux/mutex.h> 12 - #include <linux/pci.h> 13 - #include <linux/pci-ats.h> 14 12 #include <linux/poll.h> 15 13 #include <uapi/linux/iommufd.h> 16 14 ··· 16 18 #include "iommufd_private.h" 17 19 18 20 /* IOMMUFD_OBJ_FAULT Functions */ 19 - 20 - int iommufd_fault_iopf_enable(struct iommufd_device *idev) 21 - { 22 - struct device *dev = idev->dev; 23 - int ret; 24 - 25 - /* 26 - * Once we turn on PCI/PRI support for VF, the response failure code 27 - * should not be forwarded to the hardware due to PRI being a shared 28 - * resource between PF and VFs. There is no coordination for this 29 - * shared capability. This waits for a vPRI reset to recover. 30 - */ 31 - if (dev_is_pci(dev)) { 32 - struct pci_dev *pdev = to_pci_dev(dev); 33 - 34 - if (pdev->is_virtfn && pci_pri_supported(pdev)) 35 - return -EINVAL; 36 - } 37 - 38 - mutex_lock(&idev->iopf_lock); 39 - /* Device iopf has already been on. */ 40 - if (++idev->iopf_enabled > 1) { 41 - mutex_unlock(&idev->iopf_lock); 42 - return 0; 43 - } 44 - 45 - ret = iommu_dev_enable_feature(dev, IOMMU_DEV_FEAT_IOPF); 46 - if (ret) 47 - --idev->iopf_enabled; 48 - mutex_unlock(&idev->iopf_lock); 49 - 50 - return ret; 51 - } 52 - 53 - void iommufd_fault_iopf_disable(struct iommufd_device *idev) 54 - { 55 - mutex_lock(&idev->iopf_lock); 56 - if (!WARN_ON(idev->iopf_enabled == 0)) { 57 - if (--idev->iopf_enabled == 0) 58 - iommu_dev_disable_feature(idev->dev, IOMMU_DEV_FEAT_IOPF); 59 - } 60 - mutex_unlock(&idev->iopf_lock); 61 - } 62 - 63 21 void iommufd_auto_response_faults(struct iommufd_hw_pagetable *hwpt, 64 22 struct iommufd_attach_handle *handle) 65 23 { ··· 24 70 struct list_head free_list; 25 71 unsigned long index; 26 72 27 - if (!fault) 73 + if (!fault || !handle) 28 74 return; 29 75 INIT_LIST_HEAD(&free_list); 30 76

-6

drivers/iommu/iommufd/iommufd_private.h

··· 425 425 /* always the physical device */ 426 426 struct device *dev; 427 427 bool enforce_cache_coherency; 428 - /* protect iopf_enabled counter */ 429 - struct mutex iopf_lock; 430 - unsigned int iopf_enabled; 431 428 }; 432 429 433 430 static inline struct iommufd_device * ··· 503 506 int iommufd_fault_alloc(struct iommufd_ucmd *ucmd); 504 507 void iommufd_fault_destroy(struct iommufd_object *obj); 505 508 int iommufd_fault_iopf_handler(struct iopf_group *group); 506 - 507 - int iommufd_fault_iopf_enable(struct iommufd_device *idev); 508 - void iommufd_fault_iopf_disable(struct iommufd_device *idev); 509 509 void iommufd_auto_response_faults(struct iommufd_hw_pagetable *hwpt, 510 510 struct iommufd_attach_handle *handle); 511 511

+48 -11

drivers/iommu/iommufd/selftest.c

··· 58 58 MOCK_PFN_HUGE_IOVA = _MOCK_PFN_START << 2, 59 59 }; 60 60 61 + static int mock_dev_enable_iopf(struct device *dev, struct iommu_domain *domain); 62 + static void mock_dev_disable_iopf(struct device *dev, struct iommu_domain *domain); 63 + 61 64 /* 62 65 * Syzkaller has trouble randomizing the correct iova to use since it is linked 63 66 * to the map ioctl's output, and it has no ide about that. So, simplify things. ··· 171 168 int id; 172 169 u32 cache[MOCK_DEV_CACHE_NUM]; 173 170 atomic_t pasid_1024_fake_error; 171 + unsigned int iopf_refcount; 172 + struct iommu_domain *domain; 174 173 }; 175 174 176 175 static inline struct mock_dev *to_mock_dev(struct device *dev) ··· 226 221 up_write(&mdev->viommu_rwsem); 227 222 } 228 223 224 + rc = mock_dev_enable_iopf(dev, domain); 225 + if (rc) 226 + return rc; 227 + 228 + mock_dev_disable_iopf(dev, mdev->domain); 229 + mdev->domain = domain; 230 + 229 231 return 0; 230 232 } 231 233 ··· 241 229 struct iommu_domain *old) 242 230 { 243 231 struct mock_dev *mdev = to_mock_dev(dev); 232 + int rc; 244 233 245 234 /* 246 235 * Per the first attach with pasid 1024, set the ··· 268 255 atomic_set(&mdev->pasid_1024_fake_error, 1); 269 256 } 270 257 } 258 + 259 + rc = mock_dev_enable_iopf(dev, domain); 260 + if (rc) 261 + return rc; 262 + 263 + mock_dev_disable_iopf(dev, old); 271 264 272 265 return 0; 273 266 } ··· 629 610 { 630 611 } 631 612 632 - static int mock_dev_enable_feat(struct device *dev, enum iommu_dev_features feat) 613 + static int mock_dev_enable_iopf(struct device *dev, struct iommu_domain *domain) 633 614 { 634 - if (feat != IOMMU_DEV_FEAT_IOPF || !mock_iommu_iopf_queue) 615 + struct mock_dev *mdev = to_mock_dev(dev); 616 + int ret; 617 + 618 + if (!domain || !domain->iopf_handler) 619 + return 0; 620 + 621 + if (!mock_iommu_iopf_queue) 635 622 return -ENODEV; 636 623 637 - return iopf_queue_add_device(mock_iommu_iopf_queue, dev); 638 - } 624 + if (mdev->iopf_refcount) { 625 + mdev->iopf_refcount++; 626 + return 0; 627 + } 639 628 640 - static int mock_dev_disable_feat(struct device *dev, enum iommu_dev_features feat) 641 - { 642 - if (feat != IOMMU_DEV_FEAT_IOPF || !mock_iommu_iopf_queue) 643 - return -ENODEV; 629 + ret = iopf_queue_add_device(mock_iommu_iopf_queue, dev); 630 + if (ret) 631 + return ret; 644 632 645 - iopf_queue_remove_device(mock_iommu_iopf_queue, dev); 633 + mdev->iopf_refcount = 1; 646 634 647 635 return 0; 636 + } 637 + 638 + static void mock_dev_disable_iopf(struct device *dev, struct iommu_domain *domain) 639 + { 640 + struct mock_dev *mdev = to_mock_dev(dev); 641 + 642 + if (!domain || !domain->iopf_handler) 643 + return; 644 + 645 + if (--mdev->iopf_refcount) 646 + return; 647 + 648 + iopf_queue_remove_device(mock_iommu_iopf_queue, dev); 648 649 } 649 650 650 651 static void mock_viommu_destroy(struct iommufd_viommu *viommu) ··· 809 770 .device_group = generic_device_group, 810 771 .probe_device = mock_probe_device, 811 772 .page_response = mock_domain_page_response, 812 - .dev_enable_feat = mock_dev_enable_feat, 813 - .dev_disable_feat = mock_dev_disable_feat, 814 773 .user_pasid_table = true, 815 774 .viommu_alloc = mock_viommu_alloc, 816 775 .default_domain_ops =

+2 -1

drivers/iommu/ipmmu-vmsa.c

··· 1090 1090 if (mmu->features->has_cache_leaf_nodes && ipmmu_is_root(mmu)) 1091 1091 return 0; 1092 1092 1093 - ret = iommu_device_sysfs_add(&mmu->iommu, &pdev->dev, NULL, dev_name(&pdev->dev)); 1093 + ret = iommu_device_sysfs_add(&mmu->iommu, &pdev->dev, NULL, "%s", 1094 + dev_name(&pdev->dev)); 1094 1095 if (ret) 1095 1096 return ret; 1096 1097

+26 -11

drivers/iommu/mtk_iommu.c

··· 1550 1550 .larbid_remap = {{0}, {1}, {2}, {3}, {4}}, /* Linear mapping. */ 1551 1551 }; 1552 1552 1553 + static const unsigned int mt8192_larb_region_msk[MT8192_MULTI_REGION_NR_MAX][MTK_LARB_NR_MAX] = { 1554 + [0] = {~0, ~0}, /* Region0: larb0/1 */ 1555 + [1] = {0, 0, 0, 0, ~0, ~0, 0, ~0}, /* Region1: larb4/5/7 */ 1556 + [2] = {0, 0, ~0, 0, 0, 0, 0, 0, /* Region2: larb2/9/11/13/14/16/17/18/19/20 */ 1557 + 0, ~0, 0, ~0, 0, ~(u32)(BIT(9) | BIT(10)), ~(u32)(BIT(4) | BIT(5)), 0, 1558 + ~0, ~0, ~0, ~0, ~0}, 1559 + [3] = {0}, 1560 + [4] = {[13] = BIT(9) | BIT(10)}, /* larb13 port9/10 */ 1561 + [5] = {[14] = BIT(4) | BIT(5)}, /* larb14 port4/5 */ 1562 + }; 1563 + 1564 + static const struct mtk_iommu_plat_data mt6893_data = { 1565 + .m4u_plat = M4U_MT8192, 1566 + .flags = HAS_BCLK | OUT_ORDER_WR_EN | HAS_SUB_COMM_2BITS | 1567 + WR_THROT_EN | IOVA_34_EN | SHARE_PGTABLE | MTK_IOMMU_TYPE_MM, 1568 + .inv_sel_reg = REG_MMU_INV_SEL_GEN2, 1569 + .banks_num = 1, 1570 + .banks_enable = {true}, 1571 + .iova_region = mt8192_multi_dom, 1572 + .iova_region_nr = ARRAY_SIZE(mt8192_multi_dom), 1573 + .iova_region_larb_msk = mt8192_larb_region_msk, 1574 + .larbid_remap = {{0}, {1}, {4, 5}, {7}, {2}, {9, 11, 19, 20}, 1575 + {0, 14, 16}, {0, 13, 18, 17}}, 1576 + }; 1577 + 1553 1578 static const struct mtk_iommu_plat_data mt8167_data = { 1554 1579 .m4u_plat = M4U_MT8167, 1555 1580 .flags = RESET_AXI | HAS_LEGACY_IVRP_PADDR | MTK_IOMMU_TYPE_MM, ··· 1698 1673 27, 28 /* ccu0 */, MTK_INVALID_LARBID}, {4, 6}}, 1699 1674 }; 1700 1675 1701 - static const unsigned int mt8192_larb_region_msk[MT8192_MULTI_REGION_NR_MAX][MTK_LARB_NR_MAX] = { 1702 - [0] = {~0, ~0}, /* Region0: larb0/1 */ 1703 - [1] = {0, 0, 0, 0, ~0, ~0, 0, ~0}, /* Region1: larb4/5/7 */ 1704 - [2] = {0, 0, ~0, 0, 0, 0, 0, 0, /* Region2: larb2/9/11/13/14/16/17/18/19/20 */ 1705 - 0, ~0, 0, ~0, 0, ~(u32)(BIT(9) | BIT(10)), ~(u32)(BIT(4) | BIT(5)), 0, 1706 - ~0, ~0, ~0, ~0, ~0}, 1707 - [3] = {0}, 1708 - [4] = {[13] = BIT(9) | BIT(10)}, /* larb13 port9/10 */ 1709 - [5] = {[14] = BIT(4) | BIT(5)}, /* larb14 port4/5 */ 1710 - }; 1711 - 1712 1676 static const struct mtk_iommu_plat_data mt8192_data = { 1713 1677 .m4u_plat = M4U_MT8192, 1714 1678 .flags = HAS_BCLK | HAS_SUB_COMM_2BITS | OUT_ORDER_WR_EN | ··· 1791 1777 { .compatible = "mediatek,mt2712-m4u", .data = &mt2712_data}, 1792 1778 { .compatible = "mediatek,mt6779-m4u", .data = &mt6779_data}, 1793 1779 { .compatible = "mediatek,mt6795-m4u", .data = &mt6795_data}, 1780 + { .compatible = "mediatek,mt6893-iommu-mm", .data = &mt6893_data}, 1794 1781 { .compatible = "mediatek,mt8167-m4u", .data = &mt8167_data}, 1795 1782 { .compatible = "mediatek,mt8173-m4u", .data = &mt8173_data}, 1796 1783 { .compatible = "mediatek,mt8183-m4u", .data = &mt8183_data},

+1 -1

drivers/iommu/riscv/Makefile

··· 1 1 # SPDX-License-Identifier: GPL-2.0-only 2 - obj-$(CONFIG_RISCV_IOMMU) += iommu.o iommu-platform.o 2 + obj-y += iommu.o iommu-platform.o 3 3 obj-$(CONFIG_RISCV_IOMMU_PCI) += iommu-pci.o

+22 -21

drivers/iommu/riscv/iommu.c

··· 48 48 /* Device resource-managed allocations */ 49 49 struct riscv_iommu_devres { 50 50 void *addr; 51 - int order; 52 51 }; 53 52 54 53 static void riscv_iommu_devres_pages_release(struct device *dev, void *res) 55 54 { 56 55 struct riscv_iommu_devres *devres = res; 57 56 58 - iommu_free_pages(devres->addr, devres->order); 57 + iommu_free_pages(devres->addr); 59 58 } 60 59 61 60 static int riscv_iommu_devres_pages_match(struct device *dev, void *res, void *p) ··· 65 66 return devres->addr == target->addr; 66 67 } 67 68 68 - static void *riscv_iommu_get_pages(struct riscv_iommu_device *iommu, int order) 69 + static void *riscv_iommu_get_pages(struct riscv_iommu_device *iommu, 70 + unsigned int size) 69 71 { 70 72 struct riscv_iommu_devres *devres; 71 73 void *addr; 72 74 73 - addr = iommu_alloc_pages_node(dev_to_node(iommu->dev), 74 - GFP_KERNEL_ACCOUNT, order); 75 + addr = iommu_alloc_pages_node_sz(dev_to_node(iommu->dev), 76 + GFP_KERNEL_ACCOUNT, size); 75 77 if (unlikely(!addr)) 76 78 return NULL; 77 79 ··· 80 80 sizeof(struct riscv_iommu_devres), GFP_KERNEL); 81 81 82 82 if (unlikely(!devres)) { 83 - iommu_free_pages(addr, order); 83 + iommu_free_pages(addr); 84 84 return NULL; 85 85 } 86 86 87 87 devres->addr = addr; 88 - devres->order = order; 89 88 90 89 devres_add(iommu->dev, devres); 91 90 ··· 162 163 } else { 163 164 do { 164 165 const size_t queue_size = entry_size << (logsz + 1); 165 - const int order = get_order(queue_size); 166 166 167 - queue->base = riscv_iommu_get_pages(iommu, order); 167 + queue->base = riscv_iommu_get_pages( 168 + iommu, max(queue_size, SZ_4K)); 168 169 queue->phys = __pa(queue->base); 169 170 } while (!queue->base && logsz-- > 0); 170 171 } ··· 619 620 break; 620 621 } 621 622 622 - ptr = riscv_iommu_get_pages(iommu, 0); 623 + ptr = riscv_iommu_get_pages(iommu, SZ_4K); 623 624 if (!ptr) 624 625 return NULL; 625 626 ··· 699 700 } 700 701 701 702 if (!iommu->ddt_root) { 702 - iommu->ddt_root = riscv_iommu_get_pages(iommu, 0); 703 + iommu->ddt_root = riscv_iommu_get_pages(iommu, SZ_4K); 703 704 iommu->ddt_phys = __pa(iommu->ddt_root); 704 705 } 705 706 ··· 1086 1087 #define _io_pte_entry(pn, prot) ((_PAGE_PFN_MASK & ((pn) << _PAGE_PFN_SHIFT)) | (prot)) 1087 1088 1088 1089 static void riscv_iommu_pte_free(struct riscv_iommu_domain *domain, 1089 - unsigned long pte, struct list_head *freelist) 1090 + unsigned long pte, 1091 + struct iommu_pages_list *freelist) 1090 1092 { 1091 1093 unsigned long *ptr; 1092 1094 int i; ··· 1105 1105 } 1106 1106 1107 1107 if (freelist) 1108 - list_add_tail(&virt_to_page(ptr)->lru, freelist); 1108 + iommu_pages_list_add(freelist, ptr); 1109 1109 else 1110 - iommu_free_page(ptr); 1110 + iommu_free_pages(ptr); 1111 1111 } 1112 1112 1113 1113 static unsigned long *riscv_iommu_pte_alloc(struct riscv_iommu_domain *domain, ··· 1144 1144 * page table. This might race with other mappings, retry. 1145 1145 */ 1146 1146 if (_io_pte_none(pte)) { 1147 - addr = iommu_alloc_page_node(domain->numa_node, gfp); 1147 + addr = iommu_alloc_pages_node_sz(domain->numa_node, gfp, 1148 + SZ_4K); 1148 1149 if (!addr) 1149 1150 return NULL; 1150 1151 old = pte; 1151 1152 pte = _io_pte_entry(virt_to_pfn(addr), _PAGE_TABLE); 1152 1153 if (cmpxchg_relaxed(ptr, old, pte) != old) { 1153 - iommu_free_page(addr); 1154 + iommu_free_pages(addr); 1154 1155 goto pte_retry; 1155 1156 } 1156 1157 } ··· 1195 1194 unsigned long *ptr; 1196 1195 unsigned long pte, old, pte_prot; 1197 1196 int rc = 0; 1198 - LIST_HEAD(freelist); 1197 + struct iommu_pages_list freelist = IOMMU_PAGES_LIST_INIT(freelist); 1199 1198 1200 1199 if (!(prot & IOMMU_WRITE)) 1201 1200 pte_prot = _PAGE_BASE | _PAGE_READ; ··· 1226 1225 1227 1226 *mapped = size; 1228 1227 1229 - if (!list_empty(&freelist)) { 1228 + if (!iommu_pages_list_empty(&freelist)) { 1230 1229 /* 1231 1230 * In 1.0 spec version, the smallest scope we can use to 1232 1231 * invalidate all levels of page table (i.e. leaf and non-leaf) ··· 1386 1385 domain->numa_node = dev_to_node(iommu->dev); 1387 1386 domain->amo_enabled = !!(iommu->caps & RISCV_IOMMU_CAPABILITIES_AMO_HWAD); 1388 1387 domain->pgd_mode = pgd_mode; 1389 - domain->pgd_root = iommu_alloc_page_node(domain->numa_node, 1390 - GFP_KERNEL_ACCOUNT); 1388 + domain->pgd_root = iommu_alloc_pages_node_sz(domain->numa_node, 1389 + GFP_KERNEL_ACCOUNT, SZ_4K); 1391 1390 if (!domain->pgd_root) { 1392 1391 kfree(domain); 1393 1392 return ERR_PTR(-ENOMEM); ··· 1396 1395 domain->pscid = ida_alloc_range(&riscv_iommu_pscids, 1, 1397 1396 RISCV_IOMMU_MAX_PSCID, GFP_KERNEL); 1398 1397 if (domain->pscid < 0) { 1399 - iommu_free_page(domain->pgd_root); 1398 + iommu_free_pages(domain->pgd_root); 1400 1399 kfree(domain); 1401 1400 return ERR_PTR(-ENOMEM); 1402 1401 }

+8 -6

drivers/iommu/rockchip-iommu.c

··· 730 730 if (rk_dte_is_pt_valid(dte)) 731 731 goto done; 732 732 733 - page_table = iommu_alloc_page(GFP_ATOMIC | rk_ops->gfp_flags); 733 + page_table = iommu_alloc_pages_sz(GFP_ATOMIC | rk_ops->gfp_flags, 734 + SPAGE_SIZE); 734 735 if (!page_table) 735 736 return ERR_PTR(-ENOMEM); 736 737 737 738 pt_dma = dma_map_single(rk_domain->dma_dev, page_table, SPAGE_SIZE, DMA_TO_DEVICE); 738 739 if (dma_mapping_error(rk_domain->dma_dev, pt_dma)) { 739 740 dev_err(rk_domain->dma_dev, "DMA mapping error while allocating page table\n"); 740 - iommu_free_page(page_table); 741 + iommu_free_pages(page_table); 741 742 return ERR_PTR(-ENOMEM); 742 743 } 743 744 ··· 1063 1062 * Each level1 (dt) and level2 (pt) table has 1024 4-byte entries. 1064 1063 * Allocate one 4 KiB page for each table. 1065 1064 */ 1066 - rk_domain->dt = iommu_alloc_page(GFP_KERNEL | rk_ops->gfp_flags); 1065 + rk_domain->dt = iommu_alloc_pages_sz(GFP_KERNEL | rk_ops->gfp_flags, 1066 + SPAGE_SIZE); 1067 1067 if (!rk_domain->dt) 1068 1068 goto err_free_domain; 1069 1069 ··· 1088 1086 return &rk_domain->domain; 1089 1087 1090 1088 err_free_dt: 1091 - iommu_free_page(rk_domain->dt); 1089 + iommu_free_pages(rk_domain->dt); 1092 1090 err_free_domain: 1093 1091 kfree(rk_domain); 1094 1092 ··· 1109 1107 u32 *page_table = phys_to_virt(pt_phys); 1110 1108 dma_unmap_single(rk_domain->dma_dev, pt_phys, 1111 1109 SPAGE_SIZE, DMA_TO_DEVICE); 1112 - iommu_free_page(page_table); 1110 + iommu_free_pages(page_table); 1113 1111 } 1114 1112 } 1115 1113 1116 1114 dma_unmap_single(rk_domain->dma_dev, rk_domain->dt_dma, 1117 1115 SPAGE_SIZE, DMA_TO_DEVICE); 1118 - iommu_free_page(rk_domain->dt); 1116 + iommu_free_pages(rk_domain->dt); 1119 1117 1120 1118 kfree(rk_domain); 1121 1119 }

+311 -34

drivers/iommu/s390-iommu.c

··· 31 31 unsigned long *dma_table; 32 32 spinlock_t list_lock; 33 33 struct rcu_head rcu; 34 + u8 origin_type; 34 35 }; 35 36 36 37 static struct iommu_domain blocking_domain; 38 + 39 + static inline unsigned int calc_rfx(dma_addr_t ptr) 40 + { 41 + return ((unsigned long)ptr >> ZPCI_RF_SHIFT) & ZPCI_INDEX_MASK; 42 + } 43 + 44 + static inline unsigned int calc_rsx(dma_addr_t ptr) 45 + { 46 + return ((unsigned long)ptr >> ZPCI_RS_SHIFT) & ZPCI_INDEX_MASK; 47 + } 37 48 38 49 static inline unsigned int calc_rtx(dma_addr_t ptr) 39 50 { ··· 67 56 *entry |= (pfaa & ZPCI_PTE_ADDR_MASK); 68 57 } 69 58 59 + static inline void set_rf_rso(unsigned long *entry, phys_addr_t rso) 60 + { 61 + *entry &= ZPCI_RTE_FLAG_MASK; 62 + *entry |= (rso & ZPCI_RTE_ADDR_MASK); 63 + *entry |= ZPCI_TABLE_TYPE_RFX; 64 + } 65 + 66 + static inline void set_rs_rto(unsigned long *entry, phys_addr_t rto) 67 + { 68 + *entry &= ZPCI_RTE_FLAG_MASK; 69 + *entry |= (rto & ZPCI_RTE_ADDR_MASK); 70 + *entry |= ZPCI_TABLE_TYPE_RSX; 71 + } 72 + 70 73 static inline void set_rt_sto(unsigned long *entry, phys_addr_t sto) 71 74 { 72 75 *entry &= ZPCI_RTE_FLAG_MASK; ··· 93 68 *entry &= ZPCI_STE_FLAG_MASK; 94 69 *entry |= (pto & ZPCI_STE_ADDR_MASK); 95 70 *entry |= ZPCI_TABLE_TYPE_SX; 71 + } 72 + 73 + static inline void validate_rf_entry(unsigned long *entry) 74 + { 75 + *entry &= ~ZPCI_TABLE_VALID_MASK; 76 + *entry &= ~ZPCI_TABLE_OFFSET_MASK; 77 + *entry |= ZPCI_TABLE_VALID; 78 + *entry |= ZPCI_TABLE_LEN_RFX; 79 + } 80 + 81 + static inline void validate_rs_entry(unsigned long *entry) 82 + { 83 + *entry &= ~ZPCI_TABLE_VALID_MASK; 84 + *entry &= ~ZPCI_TABLE_OFFSET_MASK; 85 + *entry |= ZPCI_TABLE_VALID; 86 + *entry |= ZPCI_TABLE_LEN_RSX; 96 87 } 97 88 98 89 static inline void validate_rt_entry(unsigned long *entry) ··· 159 118 static inline int pt_entry_isvalid(unsigned long entry) 160 119 { 161 120 return (entry & ZPCI_PTE_VALID_MASK) == ZPCI_PTE_VALID; 121 + } 122 + 123 + static inline unsigned long *get_rf_rso(unsigned long entry) 124 + { 125 + if ((entry & ZPCI_TABLE_TYPE_MASK) == ZPCI_TABLE_TYPE_RFX) 126 + return phys_to_virt(entry & ZPCI_RTE_ADDR_MASK); 127 + else 128 + return NULL; 129 + } 130 + 131 + static inline unsigned long *get_rs_rto(unsigned long entry) 132 + { 133 + if ((entry & ZPCI_TABLE_TYPE_MASK) == ZPCI_TABLE_TYPE_RSX) 134 + return phys_to_virt(entry & ZPCI_RTE_ADDR_MASK); 135 + else 136 + return NULL; 162 137 } 163 138 164 139 static inline unsigned long *get_rt_sto(unsigned long entry) ··· 248 191 dma_free_cpu_table(sto); 249 192 } 250 193 251 - static void dma_cleanup_tables(unsigned long *table) 194 + static void dma_free_rt_table(unsigned long entry) 252 195 { 196 + unsigned long *rto = get_rs_rto(entry); 253 197 int rtx; 254 198 255 - if (!table) 199 + for (rtx = 0; rtx < ZPCI_TABLE_ENTRIES; rtx++) 200 + if (reg_entry_isvalid(rto[rtx])) 201 + dma_free_seg_table(rto[rtx]); 202 + 203 + dma_free_cpu_table(rto); 204 + } 205 + 206 + static void dma_free_rs_table(unsigned long entry) 207 + { 208 + unsigned long *rso = get_rf_rso(entry); 209 + int rsx; 210 + 211 + for (rsx = 0; rsx < ZPCI_TABLE_ENTRIES; rsx++) 212 + if (reg_entry_isvalid(rso[rsx])) 213 + dma_free_rt_table(rso[rsx]); 214 + 215 + dma_free_cpu_table(rso); 216 + } 217 + 218 + static void dma_cleanup_tables(struct s390_domain *domain) 219 + { 220 + int rtx, rsx, rfx; 221 + 222 + if (!domain->dma_table) 256 223 return; 257 224 258 - for (rtx = 0; rtx < ZPCI_TABLE_ENTRIES; rtx++) 259 - if (reg_entry_isvalid(table[rtx])) 260 - dma_free_seg_table(table[rtx]); 225 + switch (domain->origin_type) { 226 + case ZPCI_TABLE_TYPE_RFX: 227 + for (rfx = 0; rfx < ZPCI_TABLE_ENTRIES; rfx++) 228 + if (reg_entry_isvalid(domain->dma_table[rfx])) 229 + dma_free_rs_table(domain->dma_table[rfx]); 230 + break; 231 + case ZPCI_TABLE_TYPE_RSX: 232 + for (rsx = 0; rsx < ZPCI_TABLE_ENTRIES; rsx++) 233 + if (reg_entry_isvalid(domain->dma_table[rsx])) 234 + dma_free_rt_table(domain->dma_table[rsx]); 235 + break; 236 + case ZPCI_TABLE_TYPE_RTX: 237 + for (rtx = 0; rtx < ZPCI_TABLE_ENTRIES; rtx++) 238 + if (reg_entry_isvalid(domain->dma_table[rtx])) 239 + dma_free_seg_table(domain->dma_table[rtx]); 240 + break; 241 + default: 242 + WARN_ONCE(1, "Invalid IOMMU table (%x)\n", domain->origin_type); 243 + return; 244 + } 261 245 262 - dma_free_cpu_table(table); 246 + dma_free_cpu_table(domain->dma_table); 263 247 } 264 248 265 249 static unsigned long *dma_alloc_page_table(gfp_t gfp) ··· 314 216 for (entry = table; entry < table + ZPCI_PT_ENTRIES; entry++) 315 217 *entry = ZPCI_PTE_INVALID; 316 218 return table; 219 + } 220 + 221 + static unsigned long *dma_walk_rs_table(unsigned long *rso, 222 + dma_addr_t dma_addr, gfp_t gfp) 223 + { 224 + unsigned int rsx = calc_rsx(dma_addr); 225 + unsigned long old_rse, rse; 226 + unsigned long *rsep, *rto; 227 + 228 + rsep = &rso[rsx]; 229 + rse = READ_ONCE(*rsep); 230 + if (reg_entry_isvalid(rse)) { 231 + rto = get_rs_rto(rse); 232 + } else { 233 + rto = dma_alloc_cpu_table(gfp); 234 + if (!rto) 235 + return NULL; 236 + 237 + set_rs_rto(&rse, virt_to_phys(rto)); 238 + validate_rs_entry(&rse); 239 + entry_clr_protected(&rse); 240 + 241 + old_rse = cmpxchg(rsep, ZPCI_TABLE_INVALID, rse); 242 + if (old_rse != ZPCI_TABLE_INVALID) { 243 + /* Somone else was faster, use theirs */ 244 + dma_free_cpu_table(rto); 245 + rto = get_rs_rto(old_rse); 246 + } 247 + } 248 + return rto; 249 + } 250 + 251 + static unsigned long *dma_walk_rf_table(unsigned long *rfo, 252 + dma_addr_t dma_addr, gfp_t gfp) 253 + { 254 + unsigned int rfx = calc_rfx(dma_addr); 255 + unsigned long old_rfe, rfe; 256 + unsigned long *rfep, *rso; 257 + 258 + rfep = &rfo[rfx]; 259 + rfe = READ_ONCE(*rfep); 260 + if (reg_entry_isvalid(rfe)) { 261 + rso = get_rf_rso(rfe); 262 + } else { 263 + rso = dma_alloc_cpu_table(gfp); 264 + if (!rso) 265 + return NULL; 266 + 267 + set_rf_rso(&rfe, virt_to_phys(rso)); 268 + validate_rf_entry(&rfe); 269 + entry_clr_protected(&rfe); 270 + 271 + old_rfe = cmpxchg(rfep, ZPCI_TABLE_INVALID, rfe); 272 + if (old_rfe != ZPCI_TABLE_INVALID) { 273 + /* Somone else was faster, use theirs */ 274 + dma_free_cpu_table(rso); 275 + rso = get_rf_rso(old_rfe); 276 + } 277 + } 278 + 279 + if (!rso) 280 + return NULL; 281 + 282 + return dma_walk_rs_table(rso, dma_addr, gfp); 317 283 } 318 284 319 285 static unsigned long *dma_get_seg_table_origin(unsigned long *rtep, gfp_t gfp) ··· 433 271 return pto; 434 272 } 435 273 436 - static unsigned long *dma_walk_cpu_trans(unsigned long *rto, dma_addr_t dma_addr, gfp_t gfp) 274 + static unsigned long *dma_walk_region_tables(struct s390_domain *domain, 275 + dma_addr_t dma_addr, gfp_t gfp) 437 276 { 438 - unsigned long *sto, *pto; 277 + switch (domain->origin_type) { 278 + case ZPCI_TABLE_TYPE_RFX: 279 + return dma_walk_rf_table(domain->dma_table, dma_addr, gfp); 280 + case ZPCI_TABLE_TYPE_RSX: 281 + return dma_walk_rs_table(domain->dma_table, dma_addr, gfp); 282 + case ZPCI_TABLE_TYPE_RTX: 283 + return domain->dma_table; 284 + default: 285 + return NULL; 286 + } 287 + } 288 + 289 + static unsigned long *dma_walk_cpu_trans(struct s390_domain *domain, 290 + dma_addr_t dma_addr, gfp_t gfp) 291 + { 292 + unsigned long *rto, *sto, *pto; 439 293 unsigned int rtx, sx, px; 294 + 295 + rto = dma_walk_region_tables(domain, dma_addr, gfp); 296 + if (!rto) 297 + return NULL; 440 298 441 299 rtx = calc_rtx(dma_addr); 442 300 sto = dma_get_seg_table_origin(&rto[rtx], gfp); ··· 511 329 } 512 330 } 513 331 332 + static inline u64 max_tbl_size(struct s390_domain *domain) 333 + { 334 + switch (domain->origin_type) { 335 + case ZPCI_TABLE_TYPE_RTX: 336 + return ZPCI_TABLE_SIZE_RT - 1; 337 + case ZPCI_TABLE_TYPE_RSX: 338 + return ZPCI_TABLE_SIZE_RS - 1; 339 + case ZPCI_TABLE_TYPE_RFX: 340 + return U64_MAX; 341 + default: 342 + return 0; 343 + } 344 + } 345 + 514 346 static struct iommu_domain *s390_domain_alloc_paging(struct device *dev) 515 347 { 348 + struct zpci_dev *zdev = to_zpci_dev(dev); 516 349 struct s390_domain *s390_domain; 350 + u64 aperture_size; 517 351 518 352 s390_domain = kzalloc(sizeof(*s390_domain), GFP_KERNEL); 519 353 if (!s390_domain) ··· 540 342 kfree(s390_domain); 541 343 return NULL; 542 344 } 345 + 346 + aperture_size = min(s390_iommu_aperture, 347 + zdev->end_dma - zdev->start_dma + 1); 348 + if (aperture_size <= (ZPCI_TABLE_SIZE_RT - zdev->start_dma)) { 349 + s390_domain->origin_type = ZPCI_TABLE_TYPE_RTX; 350 + } else if (aperture_size <= (ZPCI_TABLE_SIZE_RS - zdev->start_dma) && 351 + (zdev->dtsm & ZPCI_IOTA_DT_RS)) { 352 + s390_domain->origin_type = ZPCI_TABLE_TYPE_RSX; 353 + } else if (zdev->dtsm & ZPCI_IOTA_DT_RF) { 354 + s390_domain->origin_type = ZPCI_TABLE_TYPE_RFX; 355 + } else { 356 + /* Assume RTX available */ 357 + s390_domain->origin_type = ZPCI_TABLE_TYPE_RTX; 358 + aperture_size = ZPCI_TABLE_SIZE_RT - zdev->start_dma; 359 + } 360 + zdev->end_dma = zdev->start_dma + aperture_size - 1; 361 + 543 362 s390_domain->domain.geometry.force_aperture = true; 544 363 s390_domain->domain.geometry.aperture_start = 0; 545 - s390_domain->domain.geometry.aperture_end = ZPCI_TABLE_SIZE_RT - 1; 364 + s390_domain->domain.geometry.aperture_end = max_tbl_size(s390_domain); 546 365 547 366 spin_lock_init(&s390_domain->list_lock); 548 367 INIT_LIST_HEAD_RCU(&s390_domain->devices); ··· 571 356 { 572 357 struct s390_domain *s390_domain = container_of(head, struct s390_domain, rcu); 573 358 574 - dma_cleanup_tables(s390_domain->dma_table); 359 + dma_cleanup_tables(s390_domain); 575 360 kfree(s390_domain); 576 361 } 577 362 ··· 596 381 spin_unlock_irqrestore(&zdev->dom_lock, flags); 597 382 } 598 383 384 + static u64 get_iota_region_flag(struct s390_domain *domain) 385 + { 386 + switch (domain->origin_type) { 387 + case ZPCI_TABLE_TYPE_RTX: 388 + return ZPCI_IOTA_RTTO_FLAG; 389 + case ZPCI_TABLE_TYPE_RSX: 390 + return ZPCI_IOTA_RSTO_FLAG; 391 + case ZPCI_TABLE_TYPE_RFX: 392 + return ZPCI_IOTA_RFTO_FLAG; 393 + default: 394 + WARN_ONCE(1, "Invalid IOMMU table (%x)\n", domain->origin_type); 395 + return 0; 396 + } 397 + } 398 + 599 399 static int s390_iommu_domain_reg_ioat(struct zpci_dev *zdev, 600 400 struct iommu_domain *domain, u8 *status) 601 401 { ··· 629 399 default: 630 400 s390_domain = to_s390_domain(domain); 631 401 iota = virt_to_phys(s390_domain->dma_table) | 632 - ZPCI_IOTA_RTTO_FLAG; 402 + get_iota_region_flag(s390_domain); 633 403 rc = zpci_register_ioat(zdev, 0, zdev->start_dma, 634 404 zdev->end_dma, iota, status); 635 405 } ··· 712 482 { 713 483 struct zpci_dev *zdev = to_zpci_dev(dev); 714 484 struct iommu_resv_region *region; 485 + u64 max_size, end_resv; 486 + unsigned long flags; 715 487 716 488 if (zdev->start_dma) { 717 489 region = iommu_alloc_resv_region(0, zdev->start_dma, 0, ··· 723 491 list_add_tail(&region->list, list); 724 492 } 725 493 726 - if (zdev->end_dma < ZPCI_TABLE_SIZE_RT - 1) { 727 - region = iommu_alloc_resv_region(zdev->end_dma + 1, 728 - ZPCI_TABLE_SIZE_RT - zdev->end_dma - 1, 729 - 0, IOMMU_RESV_RESERVED, GFP_KERNEL); 494 + spin_lock_irqsave(&zdev->dom_lock, flags); 495 + if (zdev->s390_domain->type == IOMMU_DOMAIN_BLOCKED || 496 + zdev->s390_domain->type == IOMMU_DOMAIN_IDENTITY) { 497 + spin_unlock_irqrestore(&zdev->dom_lock, flags); 498 + return; 499 + } 500 + 501 + max_size = max_tbl_size(to_s390_domain(zdev->s390_domain)); 502 + spin_unlock_irqrestore(&zdev->dom_lock, flags); 503 + 504 + if (zdev->end_dma < max_size) { 505 + end_resv = max_size - zdev->end_dma; 506 + region = iommu_alloc_resv_region(zdev->end_dma + 1, end_resv, 507 + 0, IOMMU_RESV_RESERVED, 508 + GFP_KERNEL); 730 509 if (!region) 731 510 return; 732 511 list_add_tail(&region->list, list); ··· 753 510 754 511 zdev = to_zpci_dev(dev); 755 512 756 - if (zdev->start_dma > zdev->end_dma || 757 - zdev->start_dma > ZPCI_TABLE_SIZE_RT - 1) 513 + if (zdev->start_dma > zdev->end_dma) 758 514 return ERR_PTR(-EINVAL); 759 - 760 - if (zdev->end_dma > ZPCI_TABLE_SIZE_RT - 1) 761 - zdev->end_dma = ZPCI_TABLE_SIZE_RT - 1; 762 515 763 516 if (zdev->tlb_refresh) 764 517 dev->iommu->shadow_on_flush = 1; ··· 845 606 int rc; 846 607 847 608 for (i = 0; i < nr_pages; i++) { 848 - entry = dma_walk_cpu_trans(s390_domain->dma_table, dma_addr, 849 - gfp); 609 + entry = dma_walk_cpu_trans(s390_domain, dma_addr, gfp); 850 610 if (unlikely(!entry)) { 851 611 rc = -ENOMEM; 852 612 goto undo_cpu_trans; ··· 860 622 undo_cpu_trans: 861 623 while (i-- > 0) { 862 624 dma_addr -= PAGE_SIZE; 863 - entry = dma_walk_cpu_trans(s390_domain->dma_table, 864 - dma_addr, gfp); 625 + entry = dma_walk_cpu_trans(s390_domain, dma_addr, gfp); 865 626 if (!entry) 866 627 break; 867 628 dma_update_cpu_trans(entry, 0, ZPCI_PTE_INVALID); ··· 877 640 int rc = 0; 878 641 879 642 for (i = 0; i < nr_pages; i++) { 880 - entry = dma_walk_cpu_trans(s390_domain->dma_table, dma_addr, 881 - GFP_ATOMIC); 643 + entry = dma_walk_cpu_trans(s390_domain, dma_addr, GFP_ATOMIC); 882 644 if (unlikely(!entry)) { 883 645 rc = -EINVAL; 884 646 break; ··· 921 685 return rc; 922 686 } 923 687 688 + static unsigned long *get_rso_from_iova(struct s390_domain *domain, 689 + dma_addr_t iova) 690 + { 691 + unsigned long *rfo; 692 + unsigned long rfe; 693 + unsigned int rfx; 694 + 695 + switch (domain->origin_type) { 696 + case ZPCI_TABLE_TYPE_RFX: 697 + rfo = domain->dma_table; 698 + rfx = calc_rfx(iova); 699 + rfe = READ_ONCE(rfo[rfx]); 700 + if (!reg_entry_isvalid(rfe)) 701 + return NULL; 702 + return get_rf_rso(rfe); 703 + case ZPCI_TABLE_TYPE_RSX: 704 + return domain->dma_table; 705 + default: 706 + return NULL; 707 + } 708 + } 709 + 710 + static unsigned long *get_rto_from_iova(struct s390_domain *domain, 711 + dma_addr_t iova) 712 + { 713 + unsigned long *rso; 714 + unsigned long rse; 715 + unsigned int rsx; 716 + 717 + switch (domain->origin_type) { 718 + case ZPCI_TABLE_TYPE_RFX: 719 + case ZPCI_TABLE_TYPE_RSX: 720 + rso = get_rso_from_iova(domain, iova); 721 + rsx = calc_rsx(iova); 722 + rse = READ_ONCE(rso[rsx]); 723 + if (!reg_entry_isvalid(rse)) 724 + return NULL; 725 + return get_rs_rto(rse); 726 + case ZPCI_TABLE_TYPE_RTX: 727 + return domain->dma_table; 728 + default: 729 + return NULL; 730 + } 731 + } 732 + 924 733 static phys_addr_t s390_iommu_iova_to_phys(struct iommu_domain *domain, 925 734 dma_addr_t iova) 926 735 { ··· 979 698 iova > domain->geometry.aperture_end) 980 699 return 0; 981 700 701 + rto = get_rto_from_iova(s390_domain, iova); 702 + if (!rto) 703 + return 0; 704 + 982 705 rtx = calc_rtx(iova); 983 706 sx = calc_sx(iova); 984 707 px = calc_px(iova); 985 - rto = s390_domain->dma_table; 986 708 987 709 rte = READ_ONCE(rto[rtx]); 988 710 if (reg_entry_isvalid(rte)) { ··· 1040 756 1041 757 int zpci_init_iommu(struct zpci_dev *zdev) 1042 758 { 1043 - u64 aperture_size; 1044 759 int rc = 0; 1045 760 1046 761 rc = iommu_device_sysfs_add(&zdev->iommu_dev, NULL, NULL, ··· 1056 773 } 1057 774 if (rc) 1058 775 goto out_sysfs; 1059 - 1060 - zdev->start_dma = PAGE_ALIGN(zdev->start_dma); 1061 - aperture_size = min3(s390_iommu_aperture, 1062 - ZPCI_TABLE_SIZE_RT - zdev->start_dma, 1063 - zdev->end_dma - zdev->start_dma + 1); 1064 - zdev->end_dma = zdev->start_dma + aperture_size - 1; 1065 776 1066 777 return 0; 1067 778

+3 -3

drivers/iommu/sun50i-iommu.c

··· 690 690 if (!sun50i_domain) 691 691 return NULL; 692 692 693 - sun50i_domain->dt = iommu_alloc_pages(GFP_KERNEL | GFP_DMA32, 694 - get_order(DT_SIZE)); 693 + sun50i_domain->dt = 694 + iommu_alloc_pages_sz(GFP_KERNEL | GFP_DMA32, DT_SIZE); 695 695 if (!sun50i_domain->dt) 696 696 goto err_free_domain; 697 697 ··· 713 713 { 714 714 struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain); 715 715 716 - iommu_free_pages(sun50i_domain->dt, get_order(DT_SIZE)); 716 + iommu_free_pages(sun50i_domain->dt); 717 717 sun50i_domain->dt = NULL; 718 718 719 719 kfree(sun50i_domain);

+58 -53

drivers/iommu/tegra-smmu.c

··· 51 51 struct iommu_device iommu; /* IOMMU Core code handle */ 52 52 }; 53 53 54 + struct tegra_pd; 55 + struct tegra_pt; 56 + 54 57 struct tegra_smmu_as { 55 58 struct iommu_domain domain; 56 59 struct tegra_smmu *smmu; 57 60 unsigned int use_count; 58 61 spinlock_t lock; 59 62 u32 *count; 60 - struct page **pts; 61 - struct page *pd; 63 + struct tegra_pt **pts; 64 + struct tegra_pd *pd; 62 65 dma_addr_t pd_dma; 63 66 unsigned id; 64 67 u32 attr; ··· 157 154 158 155 #define SMMU_PDE_ATTR (SMMU_PDE_READABLE | SMMU_PDE_WRITABLE | \ 159 156 SMMU_PDE_NONSECURE) 157 + 158 + struct tegra_pd { 159 + u32 val[SMMU_NUM_PDE]; 160 + }; 161 + 162 + struct tegra_pt { 163 + u32 val[SMMU_NUM_PTE]; 164 + }; 160 165 161 166 static unsigned int iova_pd_index(unsigned long iova) 162 167 { ··· 295 284 296 285 as->attr = SMMU_PD_READABLE | SMMU_PD_WRITABLE | SMMU_PD_NONSECURE; 297 286 298 - as->pd = __iommu_alloc_pages(GFP_KERNEL | __GFP_DMA, 0); 287 + as->pd = iommu_alloc_pages_sz(GFP_KERNEL | __GFP_DMA, SMMU_SIZE_PD); 299 288 if (!as->pd) { 300 289 kfree(as); 301 290 return NULL; ··· 303 292 304 293 as->count = kcalloc(SMMU_NUM_PDE, sizeof(u32), GFP_KERNEL); 305 294 if (!as->count) { 306 - __iommu_free_pages(as->pd, 0); 295 + iommu_free_pages(as->pd); 307 296 kfree(as); 308 297 return NULL; 309 298 } ··· 311 300 as->pts = kcalloc(SMMU_NUM_PDE, sizeof(*as->pts), GFP_KERNEL); 312 301 if (!as->pts) { 313 302 kfree(as->count); 314 - __iommu_free_pages(as->pd, 0); 303 + iommu_free_pages(as->pd); 315 304 kfree(as); 316 305 return NULL; 317 306 } ··· 428 417 goto unlock; 429 418 } 430 419 431 - as->pd_dma = dma_map_page(smmu->dev, as->pd, 0, SMMU_SIZE_PD, 432 - DMA_TO_DEVICE); 420 + as->pd_dma = 421 + dma_map_single(smmu->dev, as->pd, SMMU_SIZE_PD, DMA_TO_DEVICE); 433 422 if (dma_mapping_error(smmu->dev, as->pd_dma)) { 434 423 err = -ENOMEM; 435 424 goto unlock; ··· 461 450 return 0; 462 451 463 452 err_unmap: 464 - dma_unmap_page(smmu->dev, as->pd_dma, SMMU_SIZE_PD, DMA_TO_DEVICE); 453 + dma_unmap_single(smmu->dev, as->pd_dma, SMMU_SIZE_PD, DMA_TO_DEVICE); 465 454 unlock: 466 455 mutex_unlock(&smmu->lock); 467 456 ··· 480 469 481 470 tegra_smmu_free_asid(smmu, as->id); 482 471 483 - dma_unmap_page(smmu->dev, as->pd_dma, SMMU_SIZE_PD, DMA_TO_DEVICE); 472 + dma_unmap_single(smmu->dev, as->pd_dma, SMMU_SIZE_PD, DMA_TO_DEVICE); 484 473 485 474 as->smmu = NULL; 486 475 ··· 559 548 { 560 549 unsigned int pd_index = iova_pd_index(iova); 561 550 struct tegra_smmu *smmu = as->smmu; 562 - u32 *pd = page_address(as->pd); 551 + struct tegra_pd *pd = as->pd; 563 552 unsigned long offset = pd_index * sizeof(*pd); 564 553 565 554 /* Set the page directory entry first */ 566 - pd[pd_index] = value; 555 + pd->val[pd_index] = value; 567 556 568 557 /* The flush the page directory entry from caches */ 569 558 dma_sync_single_range_for_device(smmu->dev, as->pd_dma, offset, ··· 575 564 smmu_flush(smmu); 576 565 } 577 566 578 - static u32 *tegra_smmu_pte_offset(struct page *pt_page, unsigned long iova) 567 + static u32 *tegra_smmu_pte_offset(struct tegra_pt *pt, unsigned long iova) 579 568 { 580 - u32 *pt = page_address(pt_page); 581 - 582 - return pt + iova_pt_index(iova); 569 + return &pt->val[iova_pt_index(iova)]; 583 570 } 584 571 585 572 static u32 *tegra_smmu_pte_lookup(struct tegra_smmu_as *as, unsigned long iova, ··· 585 576 { 586 577 unsigned int pd_index = iova_pd_index(iova); 587 578 struct tegra_smmu *smmu = as->smmu; 588 - struct page *pt_page; 589 - u32 *pd; 579 + struct tegra_pt *pt; 590 580 591 - pt_page = as->pts[pd_index]; 592 - if (!pt_page) 581 + pt = as->pts[pd_index]; 582 + if (!pt) 593 583 return NULL; 594 584 595 - pd = page_address(as->pd); 596 - *dmap = smmu_pde_to_dma(smmu, pd[pd_index]); 585 + *dmap = smmu_pde_to_dma(smmu, as->pd->val[pd_index]); 597 586 598 - return tegra_smmu_pte_offset(pt_page, iova); 587 + return tegra_smmu_pte_offset(pt, iova); 599 588 } 600 589 601 590 static u32 *as_get_pte(struct tegra_smmu_as *as, dma_addr_t iova, 602 - dma_addr_t *dmap, struct page *page) 591 + dma_addr_t *dmap, struct tegra_pt *pt) 603 592 { 604 593 unsigned int pde = iova_pd_index(iova); 605 594 struct tegra_smmu *smmu = as->smmu; ··· 605 598 if (!as->pts[pde]) { 606 599 dma_addr_t dma; 607 600 608 - dma = dma_map_page(smmu->dev, page, 0, SMMU_SIZE_PT, 609 - DMA_TO_DEVICE); 601 + dma = dma_map_single(smmu->dev, pt, SMMU_SIZE_PT, 602 + DMA_TO_DEVICE); 610 603 if (dma_mapping_error(smmu->dev, dma)) { 611 - __iommu_free_pages(page, 0); 604 + iommu_free_pages(pt); 612 605 return NULL; 613 606 } 614 607 615 608 if (!smmu_dma_addr_valid(smmu, dma)) { 616 - dma_unmap_page(smmu->dev, dma, SMMU_SIZE_PT, 617 - DMA_TO_DEVICE); 618 - __iommu_free_pages(page, 0); 609 + dma_unmap_single(smmu->dev, dma, SMMU_SIZE_PT, 610 + DMA_TO_DEVICE); 611 + iommu_free_pages(pt); 619 612 return NULL; 620 613 } 621 614 622 - as->pts[pde] = page; 615 + as->pts[pde] = pt; 623 616 624 617 tegra_smmu_set_pde(as, iova, SMMU_MK_PDE(dma, SMMU_PDE_ATTR | 625 618 SMMU_PDE_NEXT)); 626 619 627 620 *dmap = dma; 628 621 } else { 629 - u32 *pd = page_address(as->pd); 630 - 631 - *dmap = smmu_pde_to_dma(smmu, pd[pde]); 622 + *dmap = smmu_pde_to_dma(smmu, as->pd->val[pde]); 632 623 } 633 624 634 625 return tegra_smmu_pte_offset(as->pts[pde], iova); ··· 642 637 static void tegra_smmu_pte_put_use(struct tegra_smmu_as *as, unsigned long iova) 643 638 { 644 639 unsigned int pde = iova_pd_index(iova); 645 - struct page *page = as->pts[pde]; 640 + struct tegra_pt *pt = as->pts[pde]; 646 641 647 642 /* 648 643 * When no entries in this page table are used anymore, return the ··· 650 645 */ 651 646 if (--as->count[pde] == 0) { 652 647 struct tegra_smmu *smmu = as->smmu; 653 - u32 *pd = page_address(as->pd); 654 - dma_addr_t pte_dma = smmu_pde_to_dma(smmu, pd[pde]); 648 + dma_addr_t pte_dma = smmu_pde_to_dma(smmu, as->pd->val[pde]); 655 649 656 650 tegra_smmu_set_pde(as, iova, 0); 657 651 658 - dma_unmap_page(smmu->dev, pte_dma, SMMU_SIZE_PT, DMA_TO_DEVICE); 659 - __iommu_free_pages(page, 0); 652 + dma_unmap_single(smmu->dev, pte_dma, SMMU_SIZE_PT, 653 + DMA_TO_DEVICE); 654 + iommu_free_pages(pt); 660 655 as->pts[pde] = NULL; 661 656 } 662 657 } ··· 676 671 smmu_flush(smmu); 677 672 } 678 673 679 - static struct page *as_get_pde_page(struct tegra_smmu_as *as, 680 - unsigned long iova, gfp_t gfp, 681 - unsigned long *flags) 674 + static struct tegra_pt *as_get_pde_page(struct tegra_smmu_as *as, 675 + unsigned long iova, gfp_t gfp, 676 + unsigned long *flags) 682 677 { 683 678 unsigned int pde = iova_pd_index(iova); 684 - struct page *page = as->pts[pde]; 679 + struct tegra_pt *pt = as->pts[pde]; 685 680 686 681 /* at first check whether allocation needs to be done at all */ 687 - if (page) 688 - return page; 682 + if (pt) 683 + return pt; 689 684 690 685 /* 691 686 * In order to prevent exhaustion of the atomic memory pool, we ··· 695 690 if (gfpflags_allow_blocking(gfp)) 696 691 spin_unlock_irqrestore(&as->lock, *flags); 697 692 698 - page = __iommu_alloc_pages(gfp | __GFP_DMA, 0); 693 + pt = iommu_alloc_pages_sz(gfp | __GFP_DMA, SMMU_SIZE_PT); 699 694 700 695 if (gfpflags_allow_blocking(gfp)) 701 696 spin_lock_irqsave(&as->lock, *flags); ··· 706 701 * if allocation succeeded and the allocation failure isn't fatal. 707 702 */ 708 703 if (as->pts[pde]) { 709 - if (page) 710 - __iommu_free_pages(page, 0); 704 + if (pt) 705 + iommu_free_pages(pt); 711 706 712 - page = as->pts[pde]; 707 + pt = as->pts[pde]; 713 708 } 714 709 715 - return page; 710 + return pt; 716 711 } 717 712 718 713 static int ··· 722 717 { 723 718 struct tegra_smmu_as *as = to_smmu_as(domain); 724 719 dma_addr_t pte_dma; 725 - struct page *page; 720 + struct tegra_pt *pt; 726 721 u32 pte_attrs; 727 722 u32 *pte; 728 723 729 - page = as_get_pde_page(as, iova, gfp, flags); 730 - if (!page) 724 + pt = as_get_pde_page(as, iova, gfp, flags); 725 + if (!pt) 731 726 return -ENOMEM; 732 727 733 - pte = as_get_pte(as, iova, &pte_dma, page); 728 + pte = as_get_pte(as, iova, &pte_dma, pt); 734 729 if (!pte) 735 730 return -ENOMEM; 736 731

+113 -80

drivers/iommu/virtio-iommu.c

··· 48 48 u64 pgsize_bitmap; 49 49 u32 first_domain; 50 50 u32 last_domain; 51 + u32 identity_domain_id; 51 52 /* Supported MAP flags */ 52 53 u32 map_flags; 53 54 u32 probe_size; ··· 63 62 struct viommu_domain { 64 63 struct iommu_domain domain; 65 64 struct viommu_dev *viommu; 66 - struct mutex mutex; /* protects viommu pointer */ 67 65 unsigned int id; 68 66 u32 map_flags; 69 67 ··· 70 70 struct rb_root_cached mappings; 71 71 72 72 unsigned long nr_endpoints; 73 - bool bypass; 74 73 }; 75 74 76 75 struct viommu_endpoint { ··· 95 96 struct virtio_iommu_fault fault; 96 97 }; 97 98 }; 99 + 100 + static struct viommu_domain viommu_identity_domain; 98 101 99 102 #define to_viommu_domain(domain) \ 100 103 container_of(domain, struct viommu_domain, domain) ··· 304 303 out_unlock: 305 304 spin_unlock_irqrestore(&viommu->request_lock, flags); 306 305 return ret; 306 + } 307 + 308 + static int viommu_send_attach_req(struct viommu_dev *viommu, struct device *dev, 309 + struct virtio_iommu_req_attach *req) 310 + { 311 + int ret; 312 + unsigned int i; 313 + struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); 314 + 315 + for (i = 0; i < fwspec->num_ids; i++) { 316 + req->endpoint = cpu_to_le32(fwspec->ids[i]); 317 + ret = viommu_send_req_sync(viommu, req, sizeof(*req)); 318 + if (ret) 319 + return ret; 320 + } 321 + return 0; 307 322 } 308 323 309 324 /* ··· 654 637 655 638 /* IOMMU API */ 656 639 657 - static struct iommu_domain *viommu_domain_alloc(unsigned type) 640 + static struct iommu_domain *viommu_domain_alloc_paging(struct device *dev) 658 641 { 659 - struct viommu_domain *vdomain; 660 - 661 - if (type != IOMMU_DOMAIN_UNMANAGED && 662 - type != IOMMU_DOMAIN_DMA && 663 - type != IOMMU_DOMAIN_IDENTITY) 664 - return NULL; 665 - 666 - vdomain = kzalloc(sizeof(*vdomain), GFP_KERNEL); 667 - if (!vdomain) 668 - return NULL; 669 - 670 - mutex_init(&vdomain->mutex); 671 - spin_lock_init(&vdomain->mappings_lock); 672 - vdomain->mappings = RB_ROOT_CACHED; 673 - 674 - return &vdomain->domain; 675 - } 676 - 677 - static int viommu_domain_finalise(struct viommu_endpoint *vdev, 678 - struct iommu_domain *domain) 679 - { 680 - int ret; 681 - unsigned long viommu_page_size; 642 + struct viommu_endpoint *vdev = dev_iommu_priv_get(dev); 682 643 struct viommu_dev *viommu = vdev->viommu; 683 - struct viommu_domain *vdomain = to_viommu_domain(domain); 644 + unsigned long viommu_page_size; 645 + struct viommu_domain *vdomain; 646 + int ret; 684 647 685 648 viommu_page_size = 1UL << __ffs(viommu->pgsize_bitmap); 686 649 if (viommu_page_size > PAGE_SIZE) { 687 650 dev_err(vdev->dev, 688 651 "granule 0x%lx larger than system page size 0x%lx\n", 689 652 viommu_page_size, PAGE_SIZE); 690 - return -ENODEV; 653 + return ERR_PTR(-ENODEV); 691 654 } 655 + 656 + vdomain = kzalloc(sizeof(*vdomain), GFP_KERNEL); 657 + if (!vdomain) 658 + return ERR_PTR(-ENOMEM); 659 + 660 + spin_lock_init(&vdomain->mappings_lock); 661 + vdomain->mappings = RB_ROOT_CACHED; 692 662 693 663 ret = ida_alloc_range(&viommu->domain_ids, viommu->first_domain, 694 664 viommu->last_domain, GFP_KERNEL); 695 - if (ret < 0) 696 - return ret; 697 - 698 - vdomain->id = (unsigned int)ret; 699 - 700 - domain->pgsize_bitmap = viommu->pgsize_bitmap; 701 - domain->geometry = viommu->geometry; 702 - 703 - vdomain->map_flags = viommu->map_flags; 704 - vdomain->viommu = viommu; 705 - 706 - if (domain->type == IOMMU_DOMAIN_IDENTITY) { 707 - if (virtio_has_feature(viommu->vdev, 708 - VIRTIO_IOMMU_F_BYPASS_CONFIG)) { 709 - vdomain->bypass = true; 710 - return 0; 711 - } 712 - 713 - ret = viommu_domain_map_identity(vdev, vdomain); 714 - if (ret) { 715 - ida_free(&viommu->domain_ids, vdomain->id); 716 - vdomain->viommu = NULL; 717 - return ret; 718 - } 665 + if (ret < 0) { 666 + kfree(vdomain); 667 + return ERR_PTR(ret); 719 668 } 720 669 721 - return 0; 670 + vdomain->id = (unsigned int)ret; 671 + 672 + vdomain->domain.pgsize_bitmap = viommu->pgsize_bitmap; 673 + vdomain->domain.geometry = viommu->geometry; 674 + 675 + vdomain->map_flags = viommu->map_flags; 676 + vdomain->viommu = viommu; 677 + 678 + return &vdomain->domain; 722 679 } 723 680 724 681 static void viommu_domain_free(struct iommu_domain *domain) ··· 708 717 kfree(vdomain); 709 718 } 710 719 720 + static struct iommu_domain *viommu_domain_alloc_identity(struct device *dev) 721 + { 722 + struct viommu_endpoint *vdev = dev_iommu_priv_get(dev); 723 + struct iommu_domain *domain; 724 + int ret; 725 + 726 + if (virtio_has_feature(vdev->viommu->vdev, 727 + VIRTIO_IOMMU_F_BYPASS_CONFIG)) 728 + return &viommu_identity_domain.domain; 729 + 730 + domain = viommu_domain_alloc_paging(dev); 731 + if (IS_ERR(domain)) 732 + return domain; 733 + 734 + ret = viommu_domain_map_identity(vdev, to_viommu_domain(domain)); 735 + if (ret) { 736 + viommu_domain_free(domain); 737 + return ERR_PTR(ret); 738 + } 739 + return domain; 740 + } 741 + 711 742 static int viommu_attach_dev(struct iommu_domain *domain, struct device *dev) 712 743 { 713 - int i; 714 744 int ret = 0; 715 745 struct virtio_iommu_req_attach req; 716 - struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); 717 746 struct viommu_endpoint *vdev = dev_iommu_priv_get(dev); 718 747 struct viommu_domain *vdomain = to_viommu_domain(domain); 719 748 720 - mutex_lock(&vdomain->mutex); 721 - if (!vdomain->viommu) { 722 - /* 723 - * Properly initialize the domain now that we know which viommu 724 - * owns it. 725 - */ 726 - ret = viommu_domain_finalise(vdev, domain); 727 - } else if (vdomain->viommu != vdev->viommu) { 728 - ret = -EINVAL; 729 - } 730 - mutex_unlock(&vdomain->mutex); 731 - 732 - if (ret) 733 - return ret; 749 + if (vdomain->viommu != vdev->viommu) 750 + return -EINVAL; 734 751 735 752 /* 736 753 * In the virtio-iommu device, when attaching the endpoint to a new ··· 760 761 .domain = cpu_to_le32(vdomain->id), 761 762 }; 762 763 763 - if (vdomain->bypass) 764 - req.flags |= cpu_to_le32(VIRTIO_IOMMU_ATTACH_F_BYPASS); 765 - 766 - for (i = 0; i < fwspec->num_ids; i++) { 767 - req.endpoint = cpu_to_le32(fwspec->ids[i]); 768 - 769 - ret = viommu_send_req_sync(vdomain->viommu, &req, sizeof(req)); 770 - if (ret) 771 - return ret; 772 - } 764 + ret = viommu_send_attach_req(vdomain->viommu, dev, &req); 765 + if (ret) 766 + return ret; 773 767 774 768 if (!vdomain->nr_endpoints) { 775 769 /* ··· 779 787 780 788 return 0; 781 789 } 790 + 791 + static int viommu_attach_identity_domain(struct iommu_domain *domain, 792 + struct device *dev) 793 + { 794 + int ret = 0; 795 + struct virtio_iommu_req_attach req; 796 + struct viommu_endpoint *vdev = dev_iommu_priv_get(dev); 797 + struct viommu_domain *vdomain = to_viommu_domain(domain); 798 + 799 + req = (struct virtio_iommu_req_attach) { 800 + .head.type = VIRTIO_IOMMU_T_ATTACH, 801 + .domain = cpu_to_le32(vdev->viommu->identity_domain_id), 802 + .flags = cpu_to_le32(VIRTIO_IOMMU_ATTACH_F_BYPASS), 803 + }; 804 + 805 + ret = viommu_send_attach_req(vdev->viommu, dev, &req); 806 + if (ret) 807 + return ret; 808 + 809 + if (vdev->vdomain) 810 + vdev->vdomain->nr_endpoints--; 811 + vdomain->nr_endpoints++; 812 + vdev->vdomain = vdomain; 813 + return 0; 814 + } 815 + 816 + static struct viommu_domain viommu_identity_domain = { 817 + .domain = { 818 + .type = IOMMU_DOMAIN_IDENTITY, 819 + .ops = &(const struct iommu_domain_ops) { 820 + .attach_dev = viommu_attach_identity_domain, 821 + }, 822 + }, 823 + }; 782 824 783 825 static void viommu_detach_dev(struct viommu_endpoint *vdev) 784 826 { ··· 1088 1062 1089 1063 static struct iommu_ops viommu_ops = { 1090 1064 .capable = viommu_capable, 1091 - .domain_alloc = viommu_domain_alloc, 1065 + .domain_alloc_identity = viommu_domain_alloc_identity, 1066 + .domain_alloc_paging = viommu_domain_alloc_paging, 1092 1067 .probe_device = viommu_probe_device, 1093 1068 .release_device = viommu_release_device, 1094 1069 .device_group = viommu_device_group, ··· 1210 1183 1211 1184 if (virtio_has_feature(vdev, VIRTIO_IOMMU_F_MMIO)) 1212 1185 viommu->map_flags |= VIRTIO_IOMMU_MAP_F_MMIO; 1186 + 1187 + /* Reserve an ID to use as the bypass domain */ 1188 + if (virtio_has_feature(viommu->vdev, VIRTIO_IOMMU_F_BYPASS_CONFIG)) { 1189 + viommu->identity_domain_id = viommu->first_domain; 1190 + viommu->first_domain++; 1191 + } 1213 1192 1214 1193 viommu_ops.pgsize_bitmap = viommu->pgsize_bitmap; 1215 1194

-40

drivers/misc/uacce/uacce.c

··· 465 465 kfree(uacce); 466 466 } 467 467 468 - static unsigned int uacce_enable_sva(struct device *parent, unsigned int flags) 469 - { 470 - int ret; 471 - 472 - if (!(flags & UACCE_DEV_SVA)) 473 - return flags; 474 - 475 - flags &= ~UACCE_DEV_SVA; 476 - 477 - ret = iommu_dev_enable_feature(parent, IOMMU_DEV_FEAT_IOPF); 478 - if (ret) { 479 - dev_err(parent, "failed to enable IOPF feature! ret = %pe\n", ERR_PTR(ret)); 480 - return flags; 481 - } 482 - 483 - ret = iommu_dev_enable_feature(parent, IOMMU_DEV_FEAT_SVA); 484 - if (ret) { 485 - dev_err(parent, "failed to enable SVA feature! ret = %pe\n", ERR_PTR(ret)); 486 - iommu_dev_disable_feature(parent, IOMMU_DEV_FEAT_IOPF); 487 - return flags; 488 - } 489 - 490 - return flags | UACCE_DEV_SVA; 491 - } 492 - 493 - static void uacce_disable_sva(struct uacce_device *uacce) 494 - { 495 - if (!(uacce->flags & UACCE_DEV_SVA)) 496 - return; 497 - 498 - iommu_dev_disable_feature(uacce->parent, IOMMU_DEV_FEAT_SVA); 499 - iommu_dev_disable_feature(uacce->parent, IOMMU_DEV_FEAT_IOPF); 500 - } 501 - 502 468 /** 503 469 * uacce_alloc() - alloc an accelerator 504 470 * @parent: pointer of uacce parent device ··· 483 517 uacce = kzalloc(sizeof(struct uacce_device), GFP_KERNEL); 484 518 if (!uacce) 485 519 return ERR_PTR(-ENOMEM); 486 - 487 - flags = uacce_enable_sva(parent, flags); 488 520 489 521 uacce->parent = parent; 490 522 uacce->flags = flags; ··· 506 542 return uacce; 507 543 508 544 err_with_uacce: 509 - uacce_disable_sva(uacce); 510 545 kfree(uacce); 511 546 return ERR_PTR(ret); 512 547 } ··· 567 604 */ 568 605 unmap_mapping_range(q->mapping, 0, 0, 1); 569 606 } 570 - 571 - /* disable sva now since no opened queues */ 572 - uacce_disable_sva(uacce); 573 607 574 608 if (uacce->cdev) 575 609 cdev_device_del(uacce->cdev, &uacce->dev);

+288

include/dt-bindings/memory/mediatek,mt6893-memory-port.h

··· 1 + /* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */ 2 + /* 3 + * Copyright (c) 2020 MediaTek Inc. 4 + * Copyright (c) 2025 Collabora Ltd 5 + * AngeloGioacchino Del Regno <angelogioacchino.delregno@collabora.com> 6 + */ 7 + #ifndef _DT_BINDINGS_MEMORY_MT6893_LARB_PORT_H_ 8 + #define _DT_BINDINGS_MEMORY_MT6893_LARB_PORT_H_ 9 + 10 + #include <dt-bindings/memory/mtk-memory-port.h> 11 + 12 + /* 13 + * MM IOMMU supports 16GB dma address. 14 + * 15 + * The address will preassign like this: 16 + * 17 + * modules dma-address-region larbs-ports 18 + * disp 0 ~ 4G larb0/2 19 + * vcodec 4G ~ 8G larb4/5/7 20 + * cam/mdp 8G ~ 12G larb9/11/13/14/16/17/18/19/20 21 + * CCU0 0x4000_0000 ~ 0x43ff_ffff larb13: port 9/10 22 + * CCU1 0x4400_0000 ~ 0x47ff_ffff larb14: port 4/5 23 + * 24 + * larb3/6/8/10/12/15 are null. 25 + */ 26 + 27 + /* larb0 */ 28 + #define M4U_PORT_L0_DISP_POSTMASK0 MTK_M4U_DOM_ID(0, 0) 29 + #define M4U_PORT_L0_MDP_RDMA4 MTK_M4U_DOM_ID(0, 1) 30 + #define M4U_PORT_L0_OVL_RDMA0_HDR MTK_M4U_DOM_ID(0, 2) 31 + #define M4U_PORT_L0_OVL_2L_RDMA1_HDR MTK_M4U_DOM_ID(0, 3) 32 + #define M4U_PORT_L0_OVL_2L_RDMA3_HDR MTK_M4U_DOM_ID(0, 4) 33 + #define M4U_PORT_L0_OVL_RDMA0 MTK_M4U_DOM_ID(0, 5) 34 + #define M4U_PORT_L0_OVL_2L_RDMA1 MTK_M4U_DOM_ID(0, 6) 35 + #define M4U_PORT_L0_OVL_2L_RDMA3 MTK_M4U_DOM_ID(0, 7) 36 + #define M4U_PORT_L0_OVL_RDMA1_SYSRAM MTK_M4U_DOM_ID(0, 8) 37 + #define M4U_PORT_L0_OVL_2L_RDMA0_SYSRAM MTK_M4U_DOM_ID(0, 9) 38 + #define M4U_PORT_L0_OVL_2L_RDMA2_SYSRAM MTK_M4U_DOM_ID(0, 10) 39 + #define M4U_PORT_L0_DISP_WDMA0 MTK_M4U_DOM_ID(0, 11) 40 + #define M4U_PORT_L0_DISP_RDMA0 MTK_M4U_DOM_ID(0, 12) 41 + #define M4U_PORT_L0_DISP_UFBC_WDMA0 MTK_M4U_DOM_ID(0, 13) 42 + #define M4U_PORT_L0_DISP_FAKE0 MTK_M4U_DOM_ID(0, 14) 43 + 44 + /* larb1 */ 45 + #define M4U_PORT_L1_DISP_POSTMASK1 MTK_M4U_DOM_ID(1, 0) 46 + #define M4U_PORT_L1_MDP_RDMA5 MTK_M4U_DOM_ID(1, 1) 47 + #define M4U_PORT_L1_OVL_RDMA1_HDR MTK_M4U_DOM_ID(1, 2) 48 + #define M4U_PORT_L1_OVL_2L_RDMA0_HDR MTK_M4U_DOM_ID(1, 3) 49 + #define M4U_PORT_L1_OVL_2L_RDMA2_HDR MTK_M4U_DOM_ID(1, 4) 50 + #define M4U_PORT_L1_OVL_RDMA1 MTK_M4U_DOM_ID(1, 5) 51 + #define M4U_PORT_L1_OVL_2L_RDMA0 MTK_M4U_DOM_ID(1, 6) 52 + #define M4U_PORT_L1_OVL_2L_RDMA2 MTK_M4U_DOM_ID(1, 7) 53 + #define M4U_PORT_L1_OVL_RDMA0_SYSRAM MTK_M4U_DOM_ID(1, 8) 54 + #define M4U_PORT_L1_OVL_2L_RDMA1_SYSRAM MTK_M4U_DOM_ID(1, 9) 55 + #define M4U_PORT_L1_OVL_2L_RDMA3_SYSRAM MTK_M4U_DOM_ID(1, 10) 56 + #define M4U_PORT_L1_DISP_WDMA1 MTK_M4U_DOM_ID(1, 11) 57 + #define M4U_PORT_L1_DISP_RDMA1 MTK_M4U_DOM_ID(1, 12) 58 + #define M4U_PORT_L1_DISP_UFBC_WDMA1 MTK_M4U_DOM_ID(1, 13) 59 + #define M4U_PORT_L1_DISP_FAKE1 MTK_M4U_DOM_ID(1, 14) 60 + 61 + /* larb2 */ 62 + #define M4U_PORT_L2_MDP_RDMA0 MTK_M4U_DOM_ID(2, 0) 63 + #define M4U_PORT_L2_MDP_RDMA2 MTK_M4U_DOM_ID(2, 1) 64 + #define M4U_PORT_L2_MDP_WROT0 MTK_M4U_DOM_ID(2, 2) 65 + #define M4U_PORT_L2_MDP_WROT2 MTK_M4U_DOM_ID(2, 3) 66 + #define M4U_PORT_L2_MDP_FILMGRAIN0 MTK_M4U_DOM_ID(2, 4) 67 + #define M4U_PORT_L2_MDP_FAKE0 MTK_M4U_DOM_ID(2, 5) 68 + 69 + /* larb3: null */ 70 + 71 + /* larb4 */ 72 + #define M4U_PORT_L4_VDEC_MC_EXT_MDP MTK_M4U_DOM_ID(4, 0) 73 + #define M4U_PORT_L4_VDEC_UFO_EXT_MDP MTK_M4U_DOM_ID(4, 1) 74 + #define M4U_PORT_L4_VDEC_PP_EXT_MDP MTK_M4U_DOM_ID(4, 2) 75 + #define M4U_PORT_L4_VDEC_PRED_RD_EXT_MDP MTK_M4U_DOM_ID(4, 3) 76 + #define M4U_PORT_L4_VDEC_PRED_WR_EXT_MDP MTK_M4U_DOM_ID(4, 4) 77 + #define M4U_PORT_L4_VDEC_PPWRAP_EXT_MDP MTK_M4U_DOM_ID(4, 5) 78 + #define M4U_PORT_L4_VDEC_TILE_EXT_MDP MTK_M4U_DOM_ID(4, 6) 79 + #define M4U_PORT_L4_VDEC_VLD_EXT_MDP MTK_M4U_DOM_ID(4, 7) 80 + #define M4U_PORT_L4_VDEC_VLD2_EXT_MDP MTK_M4U_DOM_ID(4, 8) 81 + #define M4U_PORT_L4_VDEC_AVC_MV_EXT_MDP MTK_M4U_DOM_ID(4, 9) 82 + #define M4U_PORT_L4_VDEC_RG_CTRL_DMA_EXT_MDP MTK_M4U_DOM_ID(4, 10) 83 + 84 + /* larb5 */ 85 + #define M4U_PORT_L5_VDEC_LAT0_VLD_EXT_DISP MTK_M4U_DOM_ID(5, 0) 86 + #define M4U_PORT_L5_VDEC_LAT0_VLD2_EXT_DISP MTK_M4U_DOM_ID(5, 1) 87 + #define M4U_PORT_L5_VDEC_LAT0_AVC_MV_EXT_DISP MTK_M4U_DOM_ID(5, 2) 88 + #define M4U_PORT_L5_VDEC_LAT0_PRED_RD_EXT_DISP MTK_M4U_DOM_ID(5, 3) 89 + #define M4U_PORT_L5_VDEC_LAT0_TILE_EXT_DISP MTK_M4U_DOM_ID(5, 4) 90 + #define M4U_PORT_L5_VDEC_LAT0_WDMA_EXT_DISP MTK_M4U_DOM_ID(5, 5) 91 + #define M4U_PORT_L5_VDEC_LAT0_RG_CTRL_DMA_EXT_DISP MTK_M4U_DOM_ID(5, 6) 92 + #define M4U_PORT_L5_VDEC_UFO_ENC_EXT_DISP MTK_M4U_DOM_ID(5, 7) 93 + 94 + /* larb6: null */ 95 + 96 + /* larb7 */ 97 + #define M4U_PORT_L7_VENC_RCPU_DISP MTK_M4U_DOM_ID(7, 0) 98 + #define M4U_PORT_L7_VENC_REC_DISP MTK_M4U_DOM_ID(7, 1) 99 + #define M4U_PORT_L7_VENC_BSDMA_DISP MTK_M4U_DOM_ID(7, 2) 100 + #define M4U_PORT_L7_VENC_SV_COMV_DISP MTK_M4U_DOM_ID(7, 3) 101 + #define M4U_PORT_L7_VENC_RD_COMV_DISP MTK_M4U_DOM_ID(7, 4) 102 + #define M4U_PORT_L7_VENC_NBM_RDMA_DISP MTK_M4U_DOM_ID(7, 5) 103 + #define M4U_PORT_L7_VENC_NBM_RDMA_LITE_DISP MTK_M4U_DOM_ID(7, 6) 104 + #define M4U_PORT_L7_JPGENC_Y_RDMA_DISP MTK_M4U_DOM_ID(7, 7) 105 + #define M4U_PORT_L7_JPGENC_C_RDMA_DISP MTK_M4U_DOM_ID(7, 8) 106 + #define M4U_PORT_L7_JPGENC_Q_TABLE_DISP MTK_M4U_DOM_ID(7, 9) 107 + #define M4U_PORT_L7_JPGENC_BSDMA_DISP MTK_M4U_DOM_ID(7, 10) 108 + #define M4U_PORT_L7_JPGENC_WDMA0_DISP MTK_M4U_DOM_ID(7, 11) 109 + #define M4U_PORT_L7_JPGENC_BSDMA0_DISP MTK_M4U_DOM_ID(7, 12) 110 + #define M4U_PORT_L7_VENC_NBM_WDMA_DISP MTK_M4U_DOM_ID(7, 13) 111 + #define M4U_PORT_L7_VENC_NBM_WDMA_LITE_DISP MTK_M4U_DOM_ID(7, 14) 112 + #define M4U_PORT_L7_VENC_CUR_LUMA_DISP MTK_M4U_DOM_ID(7, 15) 113 + #define M4U_PORT_L7_VENC_CUR_CHROMA_DISP MTK_M4U_DOM_ID(7, 16) 114 + #define M4U_PORT_L7_VENC_REF_LUMA_DISP MTK_M4U_DOM_ID(7, 17) 115 + #define M4U_PORT_L7_VENC_REF_CHROMA_DISP MTK_M4U_DOM_ID(7, 18) 116 + #define M4U_PORT_L7_VENC_SUB_R_LUMA_DISP MTK_M4U_DOM_ID(7, 19) 117 + #define M4U_PORT_L7_VENC_SUB_W_LUMA_DISP MTK_M4U_DOM_ID(7, 20) 118 + #define M4U_PORT_L7_VENC_FCS_NBM_RDMA_DISP MTK_M4U_DOM_ID(7, 21) 119 + #define M4U_PORT_L7_VENC_FCS_NBM_WDMA_DISP MTK_M4U_DOM_ID(7, 22) 120 + #define M4U_PORT_L7_JPGENC_WDMA1_DISP MTK_M4U_DOM_ID(7, 23) 121 + #define M4U_PORT_L7_JPGENC_BSDMA1_DISP MTK_M4U_DOM_ID(7, 24) 122 + #define M4U_PORT_L7_JPGENC_HUFF_OFFSET1_DISP MTK_M4U_DOM_ID(7, 25) 123 + #define M4U_PORT_L7_JPGENC_HUFF_OFFSET0_DISP MTK_M4U_DOM_ID(7, 26) 124 + 125 + /* larb8: null */ 126 + 127 + /* larb9 */ 128 + #define M4U_PORT_L9_IMG_IMGI_D1_MDP MTK_M4U_DOM_ID(9, 0) 129 + #define M4U_PORT_L9_IMG_IMGBI_D1_MDP MTK_M4U_DOM_ID(9, 1) 130 + #define M4U_PORT_L9_IMG_DMGI_D1_MDP MTK_M4U_DOM_ID(9, 2) 131 + #define M4U_PORT_L9_IMG_DEPI_D1_MDP MTK_M4U_DOM_ID(9, 3) 132 + #define M4U_PORT_L9_IMG_ICE_D1_MDP MTK_M4U_DOM_ID(9, 4) 133 + #define M4U_PORT_L9_IMG_SMTI_D1_MDP MTK_M4U_DOM_ID(9, 5) 134 + #define M4U_PORT_L9_IMG_SMTO_D2_MDP MTK_M4U_DOM_ID(9, 6) 135 + #define M4U_PORT_L9_IMG_SMTO_D1_MDP MTK_M4U_DOM_ID(9, 7) 136 + #define M4U_PORT_L9_IMG_CRZO_D1_MDP MTK_M4U_DOM_ID(9, 8) 137 + #define M4U_PORT_L9_IMG_IMG3O_D1_MDP MTK_M4U_DOM_ID(9, 9) 138 + #define M4U_PORT_L9_IMG_VIPI_D1_MDP MTK_M4U_DOM_ID(9, 10) 139 + #define M4U_PORT_L9_IMG_SMTI_D5_MDP MTK_M4U_DOM_ID(9, 11) 140 + #define M4U_PORT_L9_IMG_TIMGO_D1_MDP MTK_M4U_DOM_ID(9, 12) 141 + #define M4U_PORT_L9_IMG_UFBC_W0_MDP MTK_M4U_DOM_ID(9, 13) 142 + #define M4U_PORT_L9_IMG_UFBC_R0_MDP MTK_M4U_DOM_ID(9, 14) 143 + #define M4U_PORT_L9_IMG_WPE_RDMA1_MDP MTK_M4U_DOM_ID(9, 15) 144 + #define M4U_PORT_L9_IMG_WPE_RDMA0_MDP MTK_M4U_DOM_ID(9, 16) 145 + #define M4U_PORT_L9_IMG_WPE_WDMA_MDP MTK_M4U_DOM_ID(9, 17) 146 + #define M4U_PORT_L9_IMG_MFB_RDMA0_MDP MTK_M4U_DOM_ID(9, 18) 147 + #define M4U_PORT_L9_IMG_MFB_RDMA1_MDP MTK_M4U_DOM_ID(9, 19) 148 + #define M4U_PORT_L9_IMG_MFB_RDMA2_MDP MTK_M4U_DOM_ID(9, 20) 149 + #define M4U_PORT_L9_IMG_MFB_RDMA3_MDP MTK_M4U_DOM_ID(9, 21) 150 + #define M4U_PORT_L9_IMG_MFB_RDMA4_MDP MTK_M4U_DOM_ID(9, 22) 151 + #define M4U_PORT_L9_IMG_MFB_RDMA5_MDP MTK_M4U_DOM_ID(9, 23) 152 + #define M4U_PORT_L9_IMG_MFB_WDMA0_MDP MTK_M4U_DOM_ID(9, 24) 153 + #define M4U_PORT_L9_IMG_MFB_WDMA1_MDP MTK_M4U_DOM_ID(9, 25) 154 + #define M4U_PORT_L9_IMG_RESERVE6_MDP MTK_M4U_DOM_ID(9, 26) 155 + #define M4U_PORT_L9_IMG_RESERVE7_MDP MTK_M4U_DOM_ID(9, 27) 156 + #define M4U_PORT_L9_IMG_RESERVE8_MDP MTK_M4U_DOM_ID(9, 28) 157 + 158 + /* larb10: null */ 159 + 160 + /* larb11 */ 161 + #define M4U_PORT_L11_IMG_IMGI_D1_DISP MTK_M4U_DOM_ID(11, 0) 162 + #define M4U_PORT_L11_IMG_IMGBI_D1_DISP MTK_M4U_DOM_ID(11, 1) 163 + #define M4U_PORT_L11_IMG_DMGI_D1_DISP MTK_M4U_DOM_ID(11, 2) 164 + #define M4U_PORT_L11_IMG_DEPI_D1_DISP MTK_M4U_DOM_ID(11, 3) 165 + #define M4U_PORT_L11_IMG_ICE_D1_DISP MTK_M4U_DOM_ID(11, 4) 166 + #define M4U_PORT_L11_IMG_SMTI_D1_DISP MTK_M4U_DOM_ID(11, 5) 167 + #define M4U_PORT_L11_IMG_SMTO_D2_DISP MTK_M4U_DOM_ID(11, 6) 168 + #define M4U_PORT_L11_IMG_SMTO_D1_DISP MTK_M4U_DOM_ID(11, 7) 169 + #define M4U_PORT_L11_IMG_CRZO_D1_DISP MTK_M4U_DOM_ID(11, 8) 170 + #define M4U_PORT_L11_IMG_IMG3O_D1_DISP MTK_M4U_DOM_ID(11, 9) 171 + #define M4U_PORT_L11_IMG_VIPI_D1_DISP MTK_M4U_DOM_ID(11, 10) 172 + #define M4U_PORT_L11_IMG_SMTI_D5_DISP MTK_M4U_DOM_ID(11, 11) 173 + #define M4U_PORT_L11_IMG_TIMGO_D1_DISP MTK_M4U_DOM_ID(11, 12) 174 + #define M4U_PORT_L11_IMG_UFBC_W0_DISP MTK_M4U_DOM_ID(11, 13) 175 + #define M4U_PORT_L11_IMG_UFBC_R0_DISP MTK_M4U_DOM_ID(11, 14) 176 + #define M4U_PORT_L11_IMG_WPE_RDMA1_DISP MTK_M4U_DOM_ID(11, 15) 177 + #define M4U_PORT_L11_IMG_WPE_RDMA0_DISP MTK_M4U_DOM_ID(11, 16) 178 + #define M4U_PORT_L11_IMG_WPE_WDMA_DISP MTK_M4U_DOM_ID(11, 17) 179 + #define M4U_PORT_L11_IMG_MFB_RDMA0_DISP MTK_M4U_DOM_ID(11, 18) 180 + #define M4U_PORT_L11_IMG_MFB_RDMA1_DISP MTK_M4U_DOM_ID(11, 19) 181 + #define M4U_PORT_L11_IMG_MFB_RDMA2_DISP MTK_M4U_DOM_ID(11, 20) 182 + #define M4U_PORT_L11_IMG_MFB_RDMA3_DISP MTK_M4U_DOM_ID(11, 21) 183 + #define M4U_PORT_L11_IMG_MFB_RDMA4_DISP MTK_M4U_DOM_ID(11, 22) 184 + #define M4U_PORT_L11_IMG_MFB_RDMA5_DISP MTK_M4U_DOM_ID(11, 23) 185 + #define M4U_PORT_L11_IMG_MFB_WDMA0_DISP MTK_M4U_DOM_ID(11, 24) 186 + #define M4U_PORT_L11_IMG_MFB_WDMA1_DISP MTK_M4U_DOM_ID(11, 25) 187 + #define M4U_PORT_L11_IMG_RESERVE6_DISP MTK_M4U_DOM_ID(11, 26) 188 + #define M4U_PORT_L11_IMG_RESERVE7_DISP MTK_M4U_DOM_ID(11, 27) 189 + #define M4U_PORT_L11_IMG_RESERVE8_DISP MTK_M4U_DOM_ID(11, 28) 190 + 191 + /* larb12: null */ 192 + 193 + /* larb13 */ 194 + #define M4U_PORT_L13_CAM_MRAWI_MDP MTK_M4U_DOM_ID(13, 0) 195 + #define M4U_PORT_L13_CAM_MRAWO0_MDP MTK_M4U_DOM_ID(13, 1) 196 + #define M4U_PORT_L13_CAM_MRAWO1_MDP MTK_M4U_DOM_ID(13, 2) 197 + #define M4U_PORT_L13_CAM_CAMSV1_MDP MTK_M4U_DOM_ID(13, 3) 198 + #define M4U_PORT_L13_CAM_CAMSV2_MDP MTK_M4U_DOM_ID(13, 4) 199 + #define M4U_PORT_L13_CAM_CAMSV3_MDP MTK_M4U_DOM_ID(13, 5) 200 + #define M4U_PORT_L13_CAM_CAMSV4_MDP MTK_M4U_DOM_ID(13, 6) 201 + #define M4U_PORT_L13_CAM_CAMSV5_MDP MTK_M4U_DOM_ID(13, 7) 202 + #define M4U_PORT_L13_CAM_CAMSV6_MDP MTK_M4U_DOM_ID(13, 8) 203 + #define M4U_PORT_L13_CAM_CCUI_MDP MTK_M4U_DOM_ID(13, 9) 204 + #define M4U_PORT_L13_CAM_CCUO_MDP MTK_M4U_DOM_ID(13, 10) 205 + #define M4U_PORT_L13_CAM_FAKE_MDP MTK_M4U_DOM_ID(13, 11) 206 + 207 + /* larb14 */ 208 + #define M4U_PORT_L14_CAM_MRAWI_DISP MTK_M4U_DOM_ID(14, 0) 209 + #define M4U_PORT_L14_CAM_MRAWO0_DISP MTK_M4U_DOM_ID(14, 1) 210 + #define M4U_PORT_L14_CAM_MRAWO1_DISP MTK_M4U_DOM_ID(14, 2) 211 + #define M4U_PORT_L14_CAM_CAMSV0_DISP MTK_M4U_DOM_ID(14, 3) 212 + #define M4U_PORT_L14_CAM_CCUI_DISP MTK_M4U_DOM_ID(14, 4) 213 + #define M4U_PORT_L14_CAM_CCUO_DISP MTK_M4U_DOM_ID(14, 5) 214 + 215 + /* larb15: null */ 216 + 217 + /* larb16 */ 218 + #define M4U_PORT_L16_CAM_IMGO_R1_A_MDP MTK_M4U_DOM_ID(16, 0) 219 + #define M4U_PORT_L16_CAM_RRZO_R1_A_MDP MTK_M4U_DOM_ID(16, 1) 220 + #define M4U_PORT_L16_CAM_CQI_R1_A_MDP MTK_M4U_DOM_ID(16, 2) 221 + #define M4U_PORT_L16_CAM_BPCI_R1_A_MDP MTK_M4U_DOM_ID(16, 3) 222 + #define M4U_PORT_L16_CAM_YUVO_R1_A_MDP MTK_M4U_DOM_ID(16, 4) 223 + #define M4U_PORT_L16_CAM_UFDI_R2_A_MDP MTK_M4U_DOM_ID(16, 5) 224 + #define M4U_PORT_L16_CAM_RAWI_R2_A_MDP MTK_M4U_DOM_ID(16, 6) 225 + #define M4U_PORT_L16_CAM_RAWI_R3_A_MDP MTK_M4U_DOM_ID(16, 7) 226 + #define M4U_PORT_L16_CAM_AAO_R1_A_MDP MTK_M4U_DOM_ID(16, 8) 227 + #define M4U_PORT_L16_CAM_AFO_R1_A_MDP MTK_M4U_DOM_ID(16, 9) 228 + #define M4U_PORT_L16_CAM_FLKO_R1_A_MDP MTK_M4U_DOM_ID(16, 10) 229 + #define M4U_PORT_L16_CAM_LCESO_R1_A_MDP MTK_M4U_DOM_ID(16, 11) 230 + #define M4U_PORT_L16_CAM_CRZO_R1_A_MDP MTK_M4U_DOM_ID(16, 12) 231 + #define M4U_PORT_L16_CAM_LTMSO_R1_A_MDP MTK_M4U_DOM_ID(16, 13) 232 + #define M4U_PORT_L16_CAM_RSSO_R1_A_MDP MTK_M4U_DOM_ID(16, 14) 233 + #define M4U_PORT_L16_CAM_AAHO_R1_A_MDP MTK_M4U_DOM_ID(16, 15) 234 + #define M4U_PORT_L16_CAM_LSCI_R1_A_MDP MTK_M4U_DOM_ID(16, 16) 235 + 236 + /* larb17 */ 237 + #define M4U_PORT_L17_CAM_IMGO_R1_B_DISP MTK_M4U_DOM_ID(17, 0) 238 + #define M4U_PORT_L17_CAM_RRZO_R1_B_DISP MTK_M4U_DOM_ID(17, 1) 239 + #define M4U_PORT_L17_CAM_CQI_R1_B_DISP MTK_M4U_DOM_ID(17, 2) 240 + #define M4U_PORT_L17_CAM_BPCI_R1_B_DISP MTK_M4U_DOM_ID(17, 3) 241 + #define M4U_PORT_L17_CAM_YUVO_R1_B_DISP MTK_M4U_DOM_ID(17, 4) 242 + #define M4U_PORT_L17_CAM_UFDI_R2_B_DISP MTK_M4U_DOM_ID(17, 5) 243 + #define M4U_PORT_L17_CAM_RAWI_R2_B_DISP MTK_M4U_DOM_ID(17, 6) 244 + #define M4U_PORT_L17_CAM_RAWI_R3_B_DISP MTK_M4U_DOM_ID(17, 7) 245 + #define M4U_PORT_L17_CAM_AAO_R1_B_DISP MTK_M4U_DOM_ID(17, 8) 246 + #define M4U_PORT_L17_CAM_AFO_R1_B_DISP MTK_M4U_DOM_ID(17, 9) 247 + #define M4U_PORT_L17_CAM_FLKO_R1_B_DISP MTK_M4U_DOM_ID(17, 10) 248 + #define M4U_PORT_L17_CAM_LCESO_R1_B_DISP MTK_M4U_DOM_ID(17, 11) 249 + #define M4U_PORT_L17_CAM_CRZO_R1_B_DISP MTK_M4U_DOM_ID(17, 12) 250 + #define M4U_PORT_L17_CAM_LTMSO_R1_B_DISP MTK_M4U_DOM_ID(17, 13) 251 + #define M4U_PORT_L17_CAM_RSSO_R1_B_DISP MTK_M4U_DOM_ID(17, 14) 252 + #define M4U_PORT_L17_CAM_AAHO_R1_B_DISP MTK_M4U_DOM_ID(17, 15) 253 + #define M4U_PORT_L17_CAM_LSCI_R1_B_DISP MTK_M4U_DOM_ID(17, 16) 254 + 255 + /* larb18 */ 256 + #define M4U_PORT_L18_CAM_IMGO_R1_C_MDP MTK_M4U_DOM_ID(18, 0) 257 + #define M4U_PORT_L18_CAM_RRZO_R1_C_MDP MTK_M4U_DOM_ID(18, 1) 258 + #define M4U_PORT_L18_CAM_CQI_R1_C_MDP MTK_M4U_DOM_ID(18, 2) 259 + #define M4U_PORT_L18_CAM_BPCI_R1_C_MDP MTK_M4U_DOM_ID(18, 3) 260 + #define M4U_PORT_L18_CAM_YUVO_R1_C_MDP MTK_M4U_DOM_ID(18, 4) 261 + #define M4U_PORT_L18_CAM_UFDI_R2_C_MDP MTK_M4U_DOM_ID(18, 5) 262 + #define M4U_PORT_L18_CAM_RAWI_R2_C_MDP MTK_M4U_DOM_ID(18, 6) 263 + #define M4U_PORT_L18_CAM_RAWI_R3_C_MDP MTK_M4U_DOM_ID(18, 7) 264 + #define M4U_PORT_L18_CAM_AAO_R1_C_MDP MTK_M4U_DOM_ID(18, 8) 265 + #define M4U_PORT_L18_CAM_AFO_R1_C_MDP MTK_M4U_DOM_ID(18, 9) 266 + #define M4U_PORT_L18_CAM_FLKO_R1_C_MDP MTK_M4U_DOM_ID(18, 10) 267 + #define M4U_PORT_L18_CAM_LCESO_R1_C_MDP MTK_M4U_DOM_ID(18, 11) 268 + #define M4U_PORT_L18_CAM_CRZO_R1_C_MDP MTK_M4U_DOM_ID(18, 12) 269 + #define M4U_PORT_L18_CAM_LTMSO_R1_C_MDP MTK_M4U_DOM_ID(18, 13) 270 + #define M4U_PORT_L18_CAM_RSSO_R1_C_MDP MTK_M4U_DOM_ID(18, 14) 271 + #define M4U_PORT_L18_CAM_AAHO_R1_C_MDP MTK_M4U_DOM_ID(18, 15) 272 + #define M4U_PORT_L18_CAM_LSCI_R1_C_MDP MTK_M4U_DOM_ID(18, 16) 273 + 274 + /* larb19 */ 275 + #define M4U_PORT_L19_IPE_DVS_RDMA_DISP MTK_M4U_DOM_ID(19, 0) 276 + #define M4U_PORT_L19_IPE_DVS_WDMA_DISP MTK_M4U_DOM_ID(19, 1) 277 + #define M4U_PORT_L19_IPE_DVP_RDMA_DISP MTK_M4U_DOM_ID(19, 2) 278 + #define M4U_PORT_L19_IPE_DVP_WDMA_DISP MTK_M4U_DOM_ID(19, 3) 279 + 280 + /* larb20 */ 281 + #define M4U_PORT_L20_IPE_FDVT_RDA_DISP MTK_M4U_DOM_ID(20, 0) 282 + #define M4U_PORT_L20_IPE_FDVT_RDB_DISP MTK_M4U_DOM_ID(20, 1) 283 + #define M4U_PORT_L20_IPE_FDVT_WRA_DISP MTK_M4U_DOM_ID(20, 2) 284 + #define M4U_PORT_L20_IPE_FDVT_WRB_DISP MTK_M4U_DOM_ID(20, 3) 285 + #define M4U_PORT_L20_IPE_RSC_RDMA0_DISP MTK_M4U_DOM_ID(20, 4) 286 + #define M4U_PORT_L20_IPE_RSC_WDMA_DISP MTK_M4U_DOM_ID(20, 5) 287 + 288 + #endif

+3 -3

include/linux/adreno-smmu-priv.h

··· 45 45 * TTBR0 translation is enabled with the specified cfg 46 46 * @get_fault_info: Called by the GPU fault handler to get information about 47 47 * the fault 48 - * @set_stall: Configure whether stall on fault (CFCFG) is enabled. Call 49 - * before set_ttbr0_cfg(). If stalling on fault is enabled, 50 - * the GPU driver must call resume_translation() 48 + * @set_stall: Configure whether stall on fault (CFCFG) is enabled. If 49 + * stalling on fault is enabled, the GPU driver must call 50 + * resume_translation() 51 51 * @resume_translation: Resume translation after a fault 52 52 * 53 53 * @set_prr_bit: [optional] Configure the GPU's Partially Resident

+8

include/linux/io-pgtable.h

··· 88 88 * 89 89 * IO_PGTABLE_QUIRK_ARM_HD: Enables dirty tracking in stage 1 pagetable. 90 90 * IO_PGTABLE_QUIRK_ARM_S2FWB: Use the FWB format for the MemAttrs bits 91 + * 92 + * IO_PGTABLE_QUIRK_NO_WARN: Do not WARN_ON() on conflicting 93 + * mappings, but silently return -EEXISTS. Normally an attempt 94 + * to map over an existing mapping would indicate some sort of 95 + * kernel bug, which would justify the WARN_ON(). But for GPU 96 + * drivers, this could be under control of userspace. Which 97 + * deserves an error return, but not to spam dmesg. 91 98 */ 92 99 #define IO_PGTABLE_QUIRK_ARM_NS BIT(0) 93 100 #define IO_PGTABLE_QUIRK_NO_PERMS BIT(1) ··· 104 97 #define IO_PGTABLE_QUIRK_ARM_OUTER_WBWA BIT(6) 105 98 #define IO_PGTABLE_QUIRK_ARM_HD BIT(7) 106 99 #define IO_PGTABLE_QUIRK_ARM_S2FWB BIT(8) 100 + #define IO_PGTABLE_QUIRK_NO_WARN BIT(9) 107 101 unsigned long quirks; 108 102 unsigned long pgsize_bitmap; 109 103 unsigned int ias;

+23 -42

include/linux/iommu.h

··· 316 316 u32 num_sids; 317 317 }; 318 318 319 - /** 320 - * enum iommu_dev_features - Per device IOMMU features 321 - * @IOMMU_DEV_FEAT_SVA: Shared Virtual Addresses 322 - * @IOMMU_DEV_FEAT_IOPF: I/O Page Faults such as PRI or Stall. Generally 323 - * enabling %IOMMU_DEV_FEAT_SVA requires 324 - * %IOMMU_DEV_FEAT_IOPF, but some devices manage I/O Page 325 - * Faults themselves instead of relying on the IOMMU. When 326 - * supported, this feature must be enabled before and 327 - * disabled after %IOMMU_DEV_FEAT_SVA. 328 - * 329 - * Device drivers enable a feature using iommu_dev_enable_feature(). 330 - */ 331 - enum iommu_dev_features { 332 - IOMMU_DEV_FEAT_SVA, 333 - IOMMU_DEV_FEAT_IOPF, 334 - }; 335 - 336 319 #define IOMMU_NO_PASID (0U) /* Reserved for DMA w/o PASID */ 337 320 #define IOMMU_FIRST_GLOBAL_PASID (1U) /*starting range for allocation */ 338 321 #define IOMMU_PASID_INVALID (-1U) ··· 323 340 324 341 /* Read but do not clear any dirty bits */ 325 342 #define IOMMU_DIRTY_NO_CLEAR (1 << 0) 343 + 344 + /* 345 + * Pages allocated through iommu_alloc_pages_node_sz() can be placed on this 346 + * list using iommu_pages_list_add(). Note: ONLY pages from 347 + * iommu_alloc_pages_node_sz() can be used this way! 348 + */ 349 + struct iommu_pages_list { 350 + struct list_head pages; 351 + }; 352 + 353 + #define IOMMU_PAGES_LIST_INIT(name) \ 354 + ((struct iommu_pages_list){ .pages = LIST_HEAD_INIT(name.pages) }) 326 355 327 356 #ifdef CONFIG_IOMMU_API 328 357 ··· 358 363 unsigned long start; 359 364 unsigned long end; 360 365 size_t pgsize; 361 - struct list_head freelist; 366 + struct iommu_pages_list freelist; 362 367 bool queued; 363 368 }; 364 369 ··· 564 569 * op is allocated in the iommu driver and freed by the caller after 565 570 * use. The information type is one of enum iommu_hw_info_type defined 566 571 * in include/uapi/linux/iommufd.h. 567 - * @domain_alloc: allocate and return an iommu domain if success. Otherwise 568 - * NULL is returned. The domain is not fully initialized until 569 - * the caller iommu_domain_alloc() returns. 572 + * @domain_alloc: Do not use in new drivers 573 + * @domain_alloc_identity: allocate an IDENTITY domain. Drivers should prefer to 574 + * use identity_domain instead. This should only be used 575 + * if dynamic logic is necessary. 570 576 * @domain_alloc_paging_flags: Allocate an iommu domain corresponding to the 571 577 * input parameters as defined in 572 578 * include/uapi/linux/iommufd.h. The @user_data can be ··· 590 594 * @of_xlate: add OF master IDs to iommu grouping 591 595 * @is_attach_deferred: Check if domain attach should be deferred from iommu 592 596 * driver init to device driver init (default no) 593 - * @dev_enable/disable_feat: per device entries to enable/disable 594 - * iommu specific features. 595 597 * @page_response: handle page request response 596 598 * @def_domain_type: device default domain type, return value: 597 599 * - IOMMU_DOMAIN_IDENTITY: must use an identity domain ··· 623 629 void *(*hw_info)(struct device *dev, u32 *length, u32 *type); 624 630 625 631 /* Domain allocation and freeing by the iommu driver */ 632 + #if IS_ENABLED(CONFIG_FSL_PAMU) 626 633 struct iommu_domain *(*domain_alloc)(unsigned iommu_domain_type); 634 + #endif 635 + struct iommu_domain *(*domain_alloc_identity)(struct device *dev); 627 636 struct iommu_domain *(*domain_alloc_paging_flags)( 628 637 struct device *dev, u32 flags, 629 638 const struct iommu_user_data *user_data); ··· 649 652 bool (*is_attach_deferred)(struct device *dev); 650 653 651 654 /* Per device IOMMU features */ 652 - int (*dev_enable_feat)(struct device *dev, enum iommu_dev_features f); 653 - int (*dev_disable_feat)(struct device *dev, enum iommu_dev_features f); 654 - 655 655 void (*page_response)(struct device *dev, struct iopf_fault *evt, 656 656 struct iommu_page_response *msg); 657 657 ··· 744 750 * @dev: struct device for sysfs handling 745 751 * @singleton_group: Used internally for drivers that have only one group 746 752 * @max_pasids: number of supported PASIDs 753 + * @ready: set once iommu_device_register() has completed successfully 747 754 */ 748 755 struct iommu_device { 749 756 struct list_head list; ··· 753 758 struct device *dev; 754 759 struct iommu_group *singleton_group; 755 760 u32 max_pasids; 761 + bool ready; 756 762 }; 757 763 758 764 /** ··· 848 852 { 849 853 *gather = (struct iommu_iotlb_gather) { 850 854 .start = ULONG_MAX, 851 - .freelist = LIST_HEAD_INIT(gather->freelist), 855 + .freelist = IOMMU_PAGES_LIST_INIT(gather->freelist), 852 856 }; 853 857 } 854 858 ··· 1122 1126 1123 1127 extern struct mutex iommu_probe_device_lock; 1124 1128 int iommu_probe_device(struct device *dev); 1125 - 1126 - int iommu_dev_enable_feature(struct device *dev, enum iommu_dev_features f); 1127 - int iommu_dev_disable_feature(struct device *dev, enum iommu_dev_features f); 1128 1129 1129 1130 int iommu_device_use_default_domain(struct device *dev); 1130 1131 void iommu_device_unuse_default_domain(struct device *dev); ··· 1403 1410 1404 1411 static inline int iommu_fwspec_add_ids(struct device *dev, u32 *ids, 1405 1412 int num_ids) 1406 - { 1407 - return -ENODEV; 1408 - } 1409 - 1410 - static inline int 1411 - iommu_dev_enable_feature(struct device *dev, enum iommu_dev_features feat) 1412 - { 1413 - return -ENODEV; 1414 - } 1415 - 1416 - static inline int 1417 - iommu_dev_disable_feature(struct device *dev, enum iommu_dev_features feat) 1418 1413 { 1419 1414 return -ENODEV; 1420 1415 }

Configure Feed

Configure Feed