Merge tag 'iommu-updates-v4.19' of git://git.kernel.org/pub/scm/linux/kernel/git/joro/iommu

+2 -1

Documentation/admin-guide/kernel-parameters.txt

··· 1744 1744 merge 1745 1745 nomerge 1746 1746 soft 1747 - pt [x86, IA-64] 1747 + pt [x86] 1748 + nopt [x86] 1748 1749 nobypass [PPC/POWERNV] 1749 1750 Disable IOMMU bypass, using IOMMU for PCI devices. 1750 1751

+3

Documentation/devicetree/bindings/iommu/renesas,ipmmu-vmsa.txt

··· 19 19 - "renesas,ipmmu-r8a7794" for the R8A7794 (R-Car E2) IPMMU. 20 20 - "renesas,ipmmu-r8a7795" for the R8A7795 (R-Car H3) IPMMU. 21 21 - "renesas,ipmmu-r8a7796" for the R8A7796 (R-Car M3-W) IPMMU. 22 + - "renesas,ipmmu-r8a77965" for the R8A77965 (R-Car M3-N) IPMMU. 22 23 - "renesas,ipmmu-r8a77970" for the R8A77970 (R-Car V3M) IPMMU. 24 + - "renesas,ipmmu-r8a77980" for the R8A77980 (R-Car V3H) IPMMU. 25 + - "renesas,ipmmu-r8a77990" for the R8A77990 (R-Car E3) IPMMU. 23 26 - "renesas,ipmmu-r8a77995" for the R8A77995 (R-Car D3) IPMMU. 24 27 - "renesas,ipmmu-vmsa" for generic R-Car Gen2 or RZ/G1 VMSA-compatible 25 28 IPMMU.

+5

arch/x86/include/asm/irq_remapping.h

··· 33 33 IRQ_POSTING_CAP = 0, 34 34 }; 35 35 36 + enum { 37 + IRQ_REMAP_XAPIC_MODE, 38 + IRQ_REMAP_X2APIC_MODE, 39 + }; 40 + 36 41 struct vcpu_data { 37 42 u64 pi_desc_addr; /* Physical address of PI Descriptor */ 38 43 u32 vector; /* Guest vector of the interrupt */

+8

arch/x86/kernel/pci-dma.c

··· 40 40 * devices and allow every device to access to whole physical memory. This is 41 41 * useful if a user wants to use an IOMMU only for KVM device assignment to 42 42 * guests and not for driver dma translation. 43 + * It is also possible to disable by default in kernel config, and enable with 44 + * iommu=nopt at boot time. 43 45 */ 46 + #ifdef CONFIG_IOMMU_DEFAULT_PASSTHROUGH 47 + int iommu_pass_through __read_mostly = 1; 48 + #else 44 49 int iommu_pass_through __read_mostly; 50 + #endif 45 51 46 52 extern struct iommu_table_entry __iommu_table[], __iommu_table_end[]; 47 53 ··· 141 135 #endif 142 136 if (!strncmp(p, "pt", 2)) 143 137 iommu_pass_through = 1; 138 + if (!strncmp(p, "nopt", 4)) 139 + iommu_pass_through = 0; 144 140 145 141 gart_parse_options(p); 146 142

+35 -2

drivers/iommu/Kconfig

··· 60 60 61 61 endmenu 62 62 63 + config IOMMU_DEBUGFS 64 + bool "Export IOMMU internals in DebugFS" 65 + depends on DEBUG_FS 66 + help 67 + Allows exposure of IOMMU device internals. This option enables 68 + the use of debugfs by IOMMU drivers as required. Devices can, 69 + at initialization time, cause the IOMMU code to create a top-level 70 + debug/iommu directory, and then populate a subdirectory with 71 + entries as required. 72 + 73 + config IOMMU_DEFAULT_PASSTHROUGH 74 + bool "IOMMU passthrough by default" 75 + depends on IOMMU_API 76 + help 77 + Enable passthrough by default, removing the need to pass in 78 + iommu.passthrough=on or iommu=pt through command line. If this 79 + is enabled, you can still disable with iommu.passthrough=off 80 + or iommu=nopt depending on the architecture. 81 + 82 + If unsure, say N here. 83 + 63 84 config IOMMU_IOVA 64 85 tristate 65 86 ··· 155 134 This option enables support for the AMD IOMMUv2 features of the IOMMU 156 135 hardware. Select this option if you want to use devices that support 157 136 the PCI PRI and PASID interface. 137 + 138 + config AMD_IOMMU_DEBUGFS 139 + bool "Enable AMD IOMMU internals in DebugFS" 140 + depends on AMD_IOMMU && IOMMU_DEBUGFS 141 + ---help--- 142 + !!!WARNING!!! !!!WARNING!!! !!!WARNING!!! !!!WARNING!!! 143 + 144 + DO NOT ENABLE THIS OPTION UNLESS YOU REALLY, -REALLY- KNOW WHAT YOU ARE DOING!!! 145 + Exposes AMD IOMMU device internals in DebugFS. 146 + 147 + This option is -NOT- intended for production environments, and should 148 + not generally be enabled. 158 149 159 150 # Intel IOMMU support 160 151 config DMAR_TABLE ··· 317 284 select IOMMU_IO_PGTABLE_LPAE 318 285 select ARM_DMA_USE_IOMMU 319 286 help 320 - Support for the Renesas VMSA-compatible IPMMU Renesas found in the 321 - R-Mobile APE6 and R-Car H2/M2 SoCs. 287 + Support for the Renesas VMSA-compatible IPMMU found in the R-Mobile 288 + APE6, R-Car Gen2, and R-Car Gen3 SoCs. 322 289 323 290 If unsure, say N. 324 291

+3 -1

drivers/iommu/Makefile

··· 2 2 obj-$(CONFIG_IOMMU_API) += iommu.o 3 3 obj-$(CONFIG_IOMMU_API) += iommu-traces.o 4 4 obj-$(CONFIG_IOMMU_API) += iommu-sysfs.o 5 + obj-$(CONFIG_IOMMU_DEBUGFS) += iommu-debugfs.o 5 6 obj-$(CONFIG_IOMMU_DMA) += dma-iommu.o 6 7 obj-$(CONFIG_IOMMU_IO_PGTABLE) += io-pgtable.o 7 8 obj-$(CONFIG_IOMMU_IO_PGTABLE_ARMV7S) += io-pgtable-arm-v7s.o ··· 11 10 obj-$(CONFIG_OF_IOMMU) += of_iommu.o 12 11 obj-$(CONFIG_MSM_IOMMU) += msm_iommu.o 13 12 obj-$(CONFIG_AMD_IOMMU) += amd_iommu.o amd_iommu_init.o 13 + obj-$(CONFIG_AMD_IOMMU_DEBUGFS) += amd_iommu_debugfs.o 14 14 obj-$(CONFIG_AMD_IOMMU_V2) += amd_iommu_v2.o 15 15 obj-$(CONFIG_ARM_SMMU) += arm-smmu.o 16 16 obj-$(CONFIG_ARM_SMMU_V3) += arm-smmu-v3.o 17 17 obj-$(CONFIG_DMAR_TABLE) += dmar.o 18 - obj-$(CONFIG_INTEL_IOMMU) += intel-iommu.o 18 + obj-$(CONFIG_INTEL_IOMMU) += intel-iommu.o intel-pasid.o 19 19 obj-$(CONFIG_INTEL_IOMMU_SVM) += intel-svm.o 20 20 obj-$(CONFIG_IPMMU_VMSA) += ipmmu-vmsa.o 21 21 obj-$(CONFIG_IRQ_REMAP) += intel_irq_remapping.o irq_remapping.o

+19 -19

drivers/iommu/amd_iommu.c

··· 1404 1404 int level; 1405 1405 u64 *pte; 1406 1406 1407 + *page_size = 0; 1408 + 1407 1409 if (address > PM_LEVEL_SIZE(domain->mode)) 1408 1410 return NULL; 1409 1411 ··· 1946 1944 { 1947 1945 int ret; 1948 1946 1949 - /* 1950 - * Must be called with IRQs disabled. Warn here to detect early 1951 - * when its not. 1952 - */ 1953 - WARN_ON(!irqs_disabled()); 1954 - 1955 1947 /* lock domain */ 1956 1948 spin_lock(&domain->lock); 1957 1949 ··· 2110 2114 static void __detach_device(struct iommu_dev_data *dev_data) 2111 2115 { 2112 2116 struct protection_domain *domain; 2113 - 2114 - /* 2115 - * Must be called with IRQs disabled. Warn here to detect early 2116 - * when its not. 2117 - */ 2118 - WARN_ON(!irqs_disabled()); 2119 2117 2120 2118 domain = dev_data->domain; 2121 2119 ··· 2395 2405 } 2396 2406 2397 2407 if (amd_iommu_unmap_flush) { 2398 - dma_ops_free_iova(dma_dom, dma_addr, pages); 2399 2408 domain_flush_tlb(&dma_dom->domain); 2400 2409 domain_flush_complete(&dma_dom->domain); 2410 + dma_ops_free_iova(dma_dom, dma_addr, pages); 2401 2411 } else { 2402 2412 pages = __roundup_pow_of_two(pages); 2403 2413 queue_iova(&dma_dom->iovad, dma_addr >> PAGE_SHIFT, pages, 0); ··· 3182 3192 .detach_dev = amd_iommu_detach_device, 3183 3193 .map = amd_iommu_map, 3184 3194 .unmap = amd_iommu_unmap, 3185 - .map_sg = default_iommu_map_sg, 3186 3195 .iova_to_phys = amd_iommu_iova_to_phys, 3187 3196 .add_device = amd_iommu_add_device, 3188 3197 .remove_device = amd_iommu_remove_device, ··· 3863 3874 irte->lo.fields_remap.int_type = delivery_mode; 3864 3875 irte->lo.fields_remap.dm = dest_mode; 3865 3876 irte->hi.fields.vector = vector; 3866 - irte->lo.fields_remap.destination = dest_apicid; 3877 + irte->lo.fields_remap.destination = APICID_TO_IRTE_DEST_LO(dest_apicid); 3878 + irte->hi.fields.destination = APICID_TO_IRTE_DEST_HI(dest_apicid); 3867 3879 irte->lo.fields_remap.valid = 1; 3868 3880 } 3869 3881 ··· 3917 3927 3918 3928 if (!irte->lo.fields_remap.guest_mode) { 3919 3929 irte->hi.fields.vector = vector; 3920 - irte->lo.fields_remap.destination = dest_apicid; 3930 + irte->lo.fields_remap.destination = 3931 + APICID_TO_IRTE_DEST_LO(dest_apicid); 3932 + irte->hi.fields.destination = 3933 + APICID_TO_IRTE_DEST_HI(dest_apicid); 3921 3934 modify_irte_ga(devid, index, irte, NULL); 3922 3935 } 3923 3936 } ··· 4337 4344 irte->lo.val = 0; 4338 4345 irte->hi.fields.vector = cfg->vector; 4339 4346 irte->lo.fields_remap.guest_mode = 0; 4340 - irte->lo.fields_remap.destination = cfg->dest_apicid; 4347 + irte->lo.fields_remap.destination = 4348 + APICID_TO_IRTE_DEST_LO(cfg->dest_apicid); 4349 + irte->hi.fields.destination = 4350 + APICID_TO_IRTE_DEST_HI(cfg->dest_apicid); 4341 4351 irte->lo.fields_remap.int_type = apic->irq_delivery_mode; 4342 4352 irte->lo.fields_remap.dm = apic->irq_dest_mode; 4343 4353 ··· 4457 4461 raw_spin_lock_irqsave(&table->lock, flags); 4458 4462 4459 4463 if (ref->lo.fields_vapic.guest_mode) { 4460 - if (cpu >= 0) 4461 - ref->lo.fields_vapic.destination = cpu; 4464 + if (cpu >= 0) { 4465 + ref->lo.fields_vapic.destination = 4466 + APICID_TO_IRTE_DEST_LO(cpu); 4467 + ref->hi.fields.destination = 4468 + APICID_TO_IRTE_DEST_HI(cpu); 4469 + } 4462 4470 ref->lo.fields_vapic.is_run = is_run; 4463 4471 barrier(); 4464 4472 }

+33

drivers/iommu/amd_iommu_debugfs.c

··· 1 + // SPDX-License-Identifier: GPL-2.0 2 + /* 3 + * AMD IOMMU driver 4 + * 5 + * Copyright (C) 2018 Advanced Micro Devices, Inc. 6 + * 7 + * Author: Gary R Hook <gary.hook@amd.com> 8 + */ 9 + 10 + #include <linux/debugfs.h> 11 + #include <linux/iommu.h> 12 + #include <linux/pci.h> 13 + #include "amd_iommu_proto.h" 14 + #include "amd_iommu_types.h" 15 + 16 + static struct dentry *amd_iommu_debugfs; 17 + static DEFINE_MUTEX(amd_iommu_debugfs_lock); 18 + 19 + #define MAX_NAME_LEN 20 20 + 21 + void amd_iommu_debugfs_setup(struct amd_iommu *iommu) 22 + { 23 + char name[MAX_NAME_LEN + 1]; 24 + 25 + mutex_lock(&amd_iommu_debugfs_lock); 26 + if (!amd_iommu_debugfs) 27 + amd_iommu_debugfs = debugfs_create_dir("amd", 28 + iommu_debugfs_dir); 29 + mutex_unlock(&amd_iommu_debugfs_lock); 30 + 31 + snprintf(name, MAX_NAME_LEN, "iommu%02d", iommu->index); 32 + iommu->debugfs = debugfs_create_dir(name, amd_iommu_debugfs); 33 + }

+40 -17

drivers/iommu/amd_iommu_init.c

··· 153 153 bool amd_iommu_irq_remap __read_mostly; 154 154 155 155 int amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_VAPIC; 156 + static int amd_iommu_xt_mode = IRQ_REMAP_X2APIC_MODE; 156 157 157 158 static bool amd_iommu_detected; 158 159 static bool __initdata amd_iommu_disabled; ··· 281 280 282 281 static void init_translation_status(struct amd_iommu *iommu) 283 282 { 284 - u32 ctrl; 283 + u64 ctrl; 285 284 286 - ctrl = readl(iommu->mmio_base + MMIO_CONTROL_OFFSET); 285 + ctrl = readq(iommu->mmio_base + MMIO_CONTROL_OFFSET); 287 286 if (ctrl & (1<<CONTROL_IOMMU_EN)) 288 287 iommu->flags |= AMD_IOMMU_FLAG_TRANS_PRE_ENABLED; 289 288 } ··· 387 386 /* Generic functions to enable/disable certain features of the IOMMU. */ 388 387 static void iommu_feature_enable(struct amd_iommu *iommu, u8 bit) 389 388 { 390 - u32 ctrl; 389 + u64 ctrl; 391 390 392 - ctrl = readl(iommu->mmio_base + MMIO_CONTROL_OFFSET); 393 - ctrl |= (1 << bit); 394 - writel(ctrl, iommu->mmio_base + MMIO_CONTROL_OFFSET); 391 + ctrl = readq(iommu->mmio_base + MMIO_CONTROL_OFFSET); 392 + ctrl |= (1ULL << bit); 393 + writeq(ctrl, iommu->mmio_base + MMIO_CONTROL_OFFSET); 395 394 } 396 395 397 396 static void iommu_feature_disable(struct amd_iommu *iommu, u8 bit) 398 397 { 399 - u32 ctrl; 398 + u64 ctrl; 400 399 401 - ctrl = readl(iommu->mmio_base + MMIO_CONTROL_OFFSET); 402 - ctrl &= ~(1 << bit); 403 - writel(ctrl, iommu->mmio_base + MMIO_CONTROL_OFFSET); 400 + ctrl = readq(iommu->mmio_base + MMIO_CONTROL_OFFSET); 401 + ctrl &= ~(1ULL << bit); 402 + writeq(ctrl, iommu->mmio_base + MMIO_CONTROL_OFFSET); 404 403 } 405 404 406 405 static void iommu_set_inv_tlb_timeout(struct amd_iommu *iommu, int timeout) 407 406 { 408 - u32 ctrl; 407 + u64 ctrl; 409 408 410 - ctrl = readl(iommu->mmio_base + MMIO_CONTROL_OFFSET); 409 + ctrl = readq(iommu->mmio_base + MMIO_CONTROL_OFFSET); 411 410 ctrl &= ~CTRL_INV_TO_MASK; 412 411 ctrl |= (timeout << CONTROL_INV_TIMEOUT) & CTRL_INV_TO_MASK; 413 - writel(ctrl, iommu->mmio_base + MMIO_CONTROL_OFFSET); 412 + writeq(ctrl, iommu->mmio_base + MMIO_CONTROL_OFFSET); 414 413 } 415 414 416 415 /* Function to enable the hardware */ ··· 826 825 #endif /* CONFIG_IRQ_REMAP */ 827 826 828 827 return ret; 828 + } 829 + 830 + static void iommu_enable_xt(struct amd_iommu *iommu) 831 + { 832 + #ifdef CONFIG_IRQ_REMAP 833 + /* 834 + * XT mode (32-bit APIC destination ID) requires 835 + * GA mode (128-bit IRTE support) as a prerequisite. 836 + */ 837 + if (AMD_IOMMU_GUEST_IR_GA(amd_iommu_guest_ir) && 838 + amd_iommu_xt_mode == IRQ_REMAP_X2APIC_MODE) 839 + iommu_feature_enable(iommu, CONTROL_XT_EN); 840 + #endif /* CONFIG_IRQ_REMAP */ 829 841 } 830 842 831 843 static void iommu_enable_gt(struct amd_iommu *iommu) ··· 1521 1507 iommu->mmio_phys_end = MMIO_CNTR_CONF_OFFSET; 1522 1508 if (((h->efr_attr & (0x1 << IOMMU_FEAT_GASUP_SHIFT)) == 0)) 1523 1509 amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_LEGACY; 1510 + if (((h->efr_attr & (0x1 << IOMMU_FEAT_XTSUP_SHIFT)) == 0)) 1511 + amd_iommu_xt_mode = IRQ_REMAP_XAPIC_MODE; 1524 1512 break; 1525 1513 case 0x11: 1526 1514 case 0x40: ··· 1532 1516 iommu->mmio_phys_end = MMIO_CNTR_CONF_OFFSET; 1533 1517 if (((h->efr_reg & (0x1 << IOMMU_EFR_GASUP_SHIFT)) == 0)) 1534 1518 amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_LEGACY; 1519 + if (((h->efr_reg & (0x1 << IOMMU_EFR_XTSUP_SHIFT)) == 0)) 1520 + amd_iommu_xt_mode = IRQ_REMAP_XAPIC_MODE; 1535 1521 break; 1536 1522 default: 1537 1523 return -EINVAL; ··· 1850 1832 pr_info("AMD-Vi: Interrupt remapping enabled\n"); 1851 1833 if (AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir)) 1852 1834 pr_info("AMD-Vi: virtual APIC enabled\n"); 1835 + if (amd_iommu_xt_mode == IRQ_REMAP_X2APIC_MODE) 1836 + pr_info("AMD-Vi: X2APIC enabled\n"); 1853 1837 } 1854 1838 } 1855 1839 ··· 2188 2168 iommu_enable_event_buffer(iommu); 2189 2169 iommu_set_exclusion_range(iommu); 2190 2170 iommu_enable_ga(iommu); 2171 + iommu_enable_xt(iommu); 2191 2172 iommu_enable(iommu); 2192 2173 iommu_flush_all_caches(iommu); 2193 2174 } ··· 2233 2212 iommu_enable_command_buffer(iommu); 2234 2213 iommu_enable_event_buffer(iommu); 2235 2214 iommu_enable_ga(iommu); 2215 + iommu_enable_xt(iommu); 2236 2216 iommu_set_device_table(iommu); 2237 2217 iommu_flush_all_caches(iommu); 2238 2218 } ··· 2713 2691 return ret; 2714 2692 2715 2693 irq_remapping_enabled = 1; 2716 - 2717 - return 0; 2694 + return amd_iommu_xt_mode; 2718 2695 } 2719 2696 2720 2697 void amd_iommu_disable(void) ··· 2742 2721 */ 2743 2722 static int __init amd_iommu_init(void) 2744 2723 { 2724 + struct amd_iommu *iommu; 2745 2725 int ret; 2746 2726 2747 2727 ret = iommu_go_to_state(IOMMU_INITIALIZED); ··· 2752 2730 disable_iommus(); 2753 2731 free_iommu_resources(); 2754 2732 } else { 2755 - struct amd_iommu *iommu; 2756 - 2757 2733 uninit_device_table_dma(); 2758 2734 for_each_iommu(iommu) 2759 2735 iommu_flush_all_caches(iommu); 2760 2736 } 2761 2737 } 2738 + 2739 + for_each_iommu(iommu) 2740 + amd_iommu_debugfs_setup(iommu); 2762 2741 2763 2742 return ret; 2764 2743 }

+6

drivers/iommu/amd_iommu_proto.h

··· 33 33 extern void amd_iommu_init_notifier(void); 34 34 extern int amd_iommu_init_api(void); 35 35 36 + #ifdef CONFIG_AMD_IOMMU_DEBUGFS 37 + void amd_iommu_debugfs_setup(struct amd_iommu *iommu); 38 + #else 39 + static inline void amd_iommu_debugfs_setup(struct amd_iommu *iommu) {} 40 + #endif 41 + 36 42 /* Needed for interrupt remapping */ 37 43 extern int amd_iommu_prepare(void); 38 44 extern int amd_iommu_enable(void);

+16 -6

drivers/iommu/amd_iommu_types.h

··· 161 161 #define CONTROL_GAM_EN 0x19ULL 162 162 #define CONTROL_GALOG_EN 0x1CULL 163 163 #define CONTROL_GAINT_EN 0x1DULL 164 + #define CONTROL_XT_EN 0x32ULL 164 165 165 166 #define CTRL_INV_TO_MASK (7 << CONTROL_INV_TIMEOUT) 166 167 #define CTRL_INV_TO_NONE 0 ··· 379 378 #define IOMMU_CAP_EFR 27 380 379 381 380 /* IOMMU Feature Reporting Field (for IVHD type 10h */ 381 + #define IOMMU_FEAT_XTSUP_SHIFT 0 382 382 #define IOMMU_FEAT_GASUP_SHIFT 6 383 383 384 384 /* IOMMU Extended Feature Register (EFR) */ 385 + #define IOMMU_EFR_XTSUP_SHIFT 2 385 386 #define IOMMU_EFR_GASUP_SHIFT 7 386 387 387 388 #define MAX_DOMAIN_ID 65536 ··· 439 436 #define APERTURE_MAX_RANGES 32 /* allows 4GB of DMA address space */ 440 437 #define APERTURE_RANGE_INDEX(a) ((a) >> APERTURE_RANGE_SHIFT) 441 438 #define APERTURE_PAGE_INDEX(a) (((a) >> 21) & 0x3fULL) 442 - 443 439 444 440 /* 445 441 * This struct is used to pass information about ··· 596 594 597 595 u32 flags; 598 596 volatile u64 __aligned(8) cmd_sem; 597 + 598 + #ifdef CONFIG_AMD_IOMMU_DEBUGFS 599 + /* DebugFS Info */ 600 + struct dentry *debugfs; 601 + #endif 599 602 }; 600 603 601 604 static inline struct amd_iommu *dev_to_amd_iommu(struct device *dev) ··· 817 810 } fields; 818 811 }; 819 812 813 + #define APICID_TO_IRTE_DEST_LO(x) (x & 0xffffff) 814 + #define APICID_TO_IRTE_DEST_HI(x) ((x >> 24) & 0xff) 815 + 820 816 union irte_ga_lo { 821 817 u64 val; 822 818 ··· 833 823 dm : 1, 834 824 /* ------ */ 835 825 guest_mode : 1, 836 - destination : 8, 837 - rsvd : 48; 826 + destination : 24, 827 + ga_tag : 32; 838 828 } fields_remap; 839 829 840 830 /* For guest vAPIC */ ··· 847 837 is_run : 1, 848 838 /* ------ */ 849 839 guest_mode : 1, 850 - destination : 8, 851 - rsvd2 : 16, 840 + destination : 24, 852 841 ga_tag : 32; 853 842 } fields_vapic; 854 843 }; ··· 858 849 u64 vector : 8, 859 850 rsvd_1 : 4, 860 851 ga_root_ptr : 40, 861 - rsvd_2 : 12; 852 + rsvd_2 : 4, 853 + destination : 8; 862 854 } fields; 863 855 }; 864 856

+18 -8

drivers/iommu/arm-smmu-v3.c

··· 24 24 #include <linux/acpi_iort.h> 25 25 #include <linux/bitfield.h> 26 26 #include <linux/bitops.h> 27 + #include <linux/crash_dump.h> 27 28 #include <linux/delay.h> 28 29 #include <linux/dma-iommu.h> 29 30 #include <linux/err.h> ··· 367 366 #define MSI_IOVA_BASE 0x8000000 368 367 #define MSI_IOVA_LENGTH 0x100000 369 368 370 - static bool disable_bypass; 369 + static bool disable_bypass = 1; 371 370 module_param_named(disable_bypass, disable_bypass, bool, S_IRUGO); 372 371 MODULE_PARM_DESC(disable_bypass, 373 372 "Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU."); ··· 1302 1301 1303 1302 /* Sync our overflow flag, as we believe we're up to speed */ 1304 1303 q->cons = Q_OVF(q, q->prod) | Q_WRP(q, q->cons) | Q_IDX(q, q->cons); 1304 + writel(q->cons, q->cons_reg); 1305 1305 return IRQ_HANDLED; 1306 1306 } 1307 1307 ··· 1999 1997 .attach_dev = arm_smmu_attach_dev, 2000 1998 .map = arm_smmu_map, 2001 1999 .unmap = arm_smmu_unmap, 2002 - .map_sg = default_iommu_map_sg, 2003 2000 .flush_iotlb_all = arm_smmu_iotlb_sync, 2004 2001 .iotlb_sync = arm_smmu_iotlb_sync, 2005 2002 .iova_to_phys = arm_smmu_iova_to_phys, ··· 2212 2211 reg &= ~clr; 2213 2212 reg |= set; 2214 2213 writel_relaxed(reg | GBPA_UPDATE, gbpa); 2215 - return readl_relaxed_poll_timeout(gbpa, reg, !(reg & GBPA_UPDATE), 2216 - 1, ARM_SMMU_POLL_TIMEOUT_US); 2214 + ret = readl_relaxed_poll_timeout(gbpa, reg, !(reg & GBPA_UPDATE), 2215 + 1, ARM_SMMU_POLL_TIMEOUT_US); 2216 + 2217 + if (ret) 2218 + dev_err(smmu->dev, "GBPA not responding to update\n"); 2219 + return ret; 2217 2220 } 2218 2221 2219 2222 static void arm_smmu_free_msis(void *data) ··· 2397 2392 2398 2393 /* Clear CR0 and sync (disables SMMU and queue processing) */ 2399 2394 reg = readl_relaxed(smmu->base + ARM_SMMU_CR0); 2400 - if (reg & CR0_SMMUEN) 2395 + if (reg & CR0_SMMUEN) { 2396 + if (is_kdump_kernel()) { 2397 + arm_smmu_update_gbpa(smmu, GBPA_ABORT, 0); 2398 + arm_smmu_device_disable(smmu); 2399 + return -EBUSY; 2400 + } 2401 + 2401 2402 dev_warn(smmu->dev, "SMMU currently enabled! Resetting...\n"); 2403 + } 2402 2404 2403 2405 ret = arm_smmu_device_disable(smmu); 2404 2406 if (ret) ··· 2503 2491 enables |= CR0_SMMUEN; 2504 2492 } else { 2505 2493 ret = arm_smmu_update_gbpa(smmu, 0, GBPA_ABORT); 2506 - if (ret) { 2507 - dev_err(smmu->dev, "GBPA not responding to update\n"); 2494 + if (ret) 2508 2495 return ret; 2509 - } 2510 2496 } 2511 2497 ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0, 2512 2498 ARM_SMMU_CR0ACK);

+10 -7

drivers/iommu/arm-smmu.c

··· 1562 1562 .attach_dev = arm_smmu_attach_dev, 1563 1563 .map = arm_smmu_map, 1564 1564 .unmap = arm_smmu_unmap, 1565 - .map_sg = default_iommu_map_sg, 1566 1565 .flush_iotlb_all = arm_smmu_iotlb_sync, 1567 1566 .iotlb_sync = arm_smmu_iotlb_sync, 1568 1567 .iova_to_phys = arm_smmu_iova_to_phys, ··· 2102 2103 if (err) 2103 2104 return err; 2104 2105 2105 - if (smmu->version == ARM_SMMU_V2 && 2106 - smmu->num_context_banks != smmu->num_context_irqs) { 2107 - dev_err(dev, 2108 - "found only %d context interrupt(s) but %d required\n", 2109 - smmu->num_context_irqs, smmu->num_context_banks); 2110 - return -ENODEV; 2106 + if (smmu->version == ARM_SMMU_V2) { 2107 + if (smmu->num_context_banks > smmu->num_context_irqs) { 2108 + dev_err(dev, 2109 + "found only %d context irq(s) but %d required\n", 2110 + smmu->num_context_irqs, smmu->num_context_banks); 2111 + return -ENODEV; 2112 + } 2113 + 2114 + /* Ignore superfluous interrupts */ 2115 + smmu->num_context_irqs = smmu->num_context_banks; 2111 2116 } 2112 2117 2113 2118 for (i = 0; i < smmu->num_global_irqs; ++i) {

+3 -3

drivers/iommu/dmar.c

··· 1339 1339 qi_submit_sync(&desc, iommu); 1340 1340 } 1341 1341 1342 - void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 qdep, 1343 - u64 addr, unsigned mask) 1342 + void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 pfsid, 1343 + u16 qdep, u64 addr, unsigned mask) 1344 1344 { 1345 1345 struct qi_desc desc; 1346 1346 ··· 1355 1355 qdep = 0; 1356 1356 1357 1357 desc.low = QI_DEV_IOTLB_SID(sid) | QI_DEV_IOTLB_QDEP(qdep) | 1358 - QI_DIOTLB_TYPE; 1358 + QI_DIOTLB_TYPE | QI_DEV_IOTLB_PFSID(pfsid); 1359 1359 1360 1360 qi_submit_sync(&desc, iommu); 1361 1361 }

-1

drivers/iommu/exynos-iommu.c

··· 1332 1332 .detach_dev = exynos_iommu_detach_device, 1333 1333 .map = exynos_iommu_map, 1334 1334 .unmap = exynos_iommu_unmap, 1335 - .map_sg = default_iommu_map_sg, 1336 1335 .iova_to_phys = exynos_iommu_iova_to_phys, 1337 1336 .device_group = generic_device_group, 1338 1337 .add_device = exynos_iommu_add_device,

+77 -83

drivers/iommu/intel-iommu.c

··· 52 52 #include <asm/iommu.h> 53 53 54 54 #include "irq_remapping.h" 55 + #include "intel-pasid.h" 55 56 56 57 #define ROOT_SIZE VTD_PAGE_SIZE 57 58 #define CONTEXT_SIZE VTD_PAGE_SIZE ··· 380 379 for (idx = 0; idx < g_num_of_iommus; idx++) \ 381 380 if (domain->iommu_refcnt[idx]) 382 381 383 - struct dmar_domain { 384 - int nid; /* node id */ 385 - 386 - unsigned iommu_refcnt[DMAR_UNITS_SUPPORTED]; 387 - /* Refcount of devices per iommu */ 388 - 389 - 390 - u16 iommu_did[DMAR_UNITS_SUPPORTED]; 391 - /* Domain ids per IOMMU. Use u16 since 392 - * domain ids are 16 bit wide according 393 - * to VT-d spec, section 9.3 */ 394 - 395 - bool has_iotlb_device; 396 - struct list_head devices; /* all devices' list */ 397 - struct iova_domain iovad; /* iova's that belong to this domain */ 398 - 399 - struct dma_pte *pgd; /* virtual address */ 400 - int gaw; /* max guest address width */ 401 - 402 - /* adjusted guest address width, 0 is level 2 30-bit */ 403 - int agaw; 404 - 405 - int flags; /* flags to find out type of domain */ 406 - 407 - int iommu_coherency;/* indicate coherency of iommu access */ 408 - int iommu_snooping; /* indicate snooping control feature*/ 409 - int iommu_count; /* reference count of iommu */ 410 - int iommu_superpage;/* Level of superpages supported: 411 - 0 == 4KiB (no superpages), 1 == 2MiB, 412 - 2 == 1GiB, 3 == 512GiB, 4 == 1TiB */ 413 - u64 max_addr; /* maximum mapped address */ 414 - 415 - struct iommu_domain domain; /* generic domain data structure for 416 - iommu core */ 417 - }; 418 - 419 - /* PCI domain-device relationship */ 420 - struct device_domain_info { 421 - struct list_head link; /* link to domain siblings */ 422 - struct list_head global; /* link to global list */ 423 - u8 bus; /* PCI bus number */ 424 - u8 devfn; /* PCI devfn number */ 425 - u8 pasid_supported:3; 426 - u8 pasid_enabled:1; 427 - u8 pri_supported:1; 428 - u8 pri_enabled:1; 429 - u8 ats_supported:1; 430 - u8 ats_enabled:1; 431 - u8 ats_qdep; 432 - struct device *dev; /* it's NULL for PCIe-to-PCI bridge */ 433 - struct intel_iommu *iommu; /* IOMMU used by this device */ 434 - struct dmar_domain *domain; /* pointer to domain */ 435 - }; 436 - 437 382 struct dmar_rmrr_unit { 438 383 struct list_head list; /* list of rmrr units */ 439 384 struct acpi_dmar_header *hdr; /* ACPI header */ ··· 469 522 #define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1)) 470 523 static DEFINE_SPINLOCK(device_domain_lock); 471 524 static LIST_HEAD(device_domain_list); 525 + 526 + /* 527 + * Iterate over elements in device_domain_list and call the specified 528 + * callback @fn against each element. This helper should only be used 529 + * in the context where the device_domain_lock has already been holden. 530 + */ 531 + int for_each_device_domain(int (*fn)(struct device_domain_info *info, 532 + void *data), void *data) 533 + { 534 + int ret = 0; 535 + struct device_domain_info *info; 536 + 537 + assert_spin_locked(&device_domain_lock); 538 + list_for_each_entry(info, &device_domain_list, global) { 539 + ret = fn(info, data); 540 + if (ret) 541 + return ret; 542 + } 543 + 544 + return 0; 545 + } 472 546 473 547 const struct iommu_ops intel_iommu_ops; 474 548 ··· 597 629 domains[did & 0xff] = domain; 598 630 } 599 631 600 - static inline void *alloc_pgtable_page(int node) 632 + void *alloc_pgtable_page(int node) 601 633 { 602 634 struct page *page; 603 635 void *vaddr = NULL; ··· 608 640 return vaddr; 609 641 } 610 642 611 - static inline void free_pgtable_page(void *vaddr) 643 + void free_pgtable_page(void *vaddr) 612 644 { 613 645 free_page((unsigned long)vaddr); 614 646 } ··· 691 723 } 692 724 693 725 /* This functionin only returns single iommu in a domain */ 694 - static struct intel_iommu *domain_get_iommu(struct dmar_domain *domain) 726 + struct intel_iommu *domain_get_iommu(struct dmar_domain *domain) 695 727 { 696 728 int iommu_id; 697 729 ··· 1469 1501 return; 1470 1502 1471 1503 pdev = to_pci_dev(info->dev); 1504 + /* For IOMMU that supports device IOTLB throttling (DIT), we assign 1505 + * PFSID to the invalidation desc of a VF such that IOMMU HW can gauge 1506 + * queue depth at PF level. If DIT is not set, PFSID will be treated as 1507 + * reserved, which should be set to 0. 1508 + */ 1509 + if (!ecap_dit(info->iommu->ecap)) 1510 + info->pfsid = 0; 1511 + else { 1512 + struct pci_dev *pf_pdev; 1513 + 1514 + /* pdev will be returned if device is not a vf */ 1515 + pf_pdev = pci_physfn(pdev); 1516 + info->pfsid = PCI_DEVID(pf_pdev->bus->number, pf_pdev->devfn); 1517 + } 1472 1518 1473 1519 #ifdef CONFIG_INTEL_IOMMU_SVM 1474 1520 /* The PCIe spec, in its wisdom, declares that the behaviour of ··· 1548 1566 1549 1567 sid = info->bus << 8 | info->devfn; 1550 1568 qdep = info->ats_qdep; 1551 - qi_flush_dev_iotlb(info->iommu, sid, qdep, addr, mask); 1569 + qi_flush_dev_iotlb(info->iommu, sid, info->pfsid, 1570 + qdep, addr, mask); 1552 1571 } 1553 1572 spin_unlock_irqrestore(&device_domain_lock, flags); 1554 1573 } ··· 1783 1800 if (pasid_enabled(iommu)) { 1784 1801 if (ecap_prs(iommu->ecap)) 1785 1802 intel_svm_finish_prq(iommu); 1786 - intel_svm_free_pasid_tables(iommu); 1803 + intel_svm_exit(iommu); 1787 1804 } 1788 1805 #endif 1789 1806 } ··· 2478 2495 info->dev = dev; 2479 2496 info->domain = domain; 2480 2497 info->iommu = iommu; 2498 + info->pasid_table = NULL; 2481 2499 2482 2500 if (dev && dev_is_pci(dev)) { 2483 2501 struct pci_dev *pdev = to_pci_dev(info->dev); ··· 2536 2552 list_add(&info->global, &device_domain_list); 2537 2553 if (dev) 2538 2554 dev->archdata.iommu = info; 2555 + 2556 + if (dev && dev_is_pci(dev) && info->pasid_supported) { 2557 + ret = intel_pasid_alloc_table(dev); 2558 + if (ret) { 2559 + __dmar_remove_one_dev_info(info); 2560 + spin_unlock_irqrestore(&device_domain_lock, flags); 2561 + return NULL; 2562 + } 2563 + } 2539 2564 spin_unlock_irqrestore(&device_domain_lock, flags); 2540 2565 2541 2566 if (dev && domain_context_mapping(domain, dev)) { ··· 3297 3304 } 3298 3305 3299 3306 for_each_active_iommu(iommu, drhd) { 3307 + /* 3308 + * Find the max pasid size of all IOMMU's in the system. 3309 + * We need to ensure the system pasid table is no bigger 3310 + * than the smallest supported. 3311 + */ 3312 + if (pasid_enabled(iommu)) { 3313 + u32 temp = 2 << ecap_pss(iommu->ecap); 3314 + 3315 + intel_pasid_max_id = min_t(u32, temp, 3316 + intel_pasid_max_id); 3317 + } 3318 + 3300 3319 g_iommus[iommu->seq_id] = iommu; 3301 3320 3302 3321 intel_iommu_init_qi(iommu); ··· 3364 3359 hw_pass_through = 0; 3365 3360 #ifdef CONFIG_INTEL_IOMMU_SVM 3366 3361 if (pasid_enabled(iommu)) 3367 - intel_svm_alloc_pasid_tables(iommu); 3362 + intel_svm_init(iommu); 3368 3363 #endif 3369 3364 } 3370 3365 ··· 3531 3526 return iova_pfn; 3532 3527 } 3533 3528 3534 - static struct dmar_domain *get_valid_domain_for_dev(struct device *dev) 3529 + struct dmar_domain *get_valid_domain_for_dev(struct device *dev) 3535 3530 { 3536 3531 struct dmar_domain *domain, *tmp; 3537 3532 struct dmar_rmrr_unit *rmrr; ··· 4359 4354 4360 4355 #ifdef CONFIG_INTEL_IOMMU_SVM 4361 4356 if (pasid_enabled(iommu)) 4362 - intel_svm_alloc_pasid_tables(iommu); 4357 + intel_svm_init(iommu); 4363 4358 #endif 4364 4359 4365 4360 if (dmaru->ignored) { ··· 4911 4906 if (info->dev) { 4912 4907 iommu_disable_dev_iotlb(info); 4913 4908 domain_context_clear(iommu, info->dev); 4909 + intel_pasid_free_table(info->dev); 4914 4910 } 4915 4911 4916 4912 unlink_domain_info(info); ··· 5237 5231 5238 5232 #ifdef CONFIG_INTEL_IOMMU_SVM 5239 5233 #define MAX_NR_PASID_BITS (20) 5240 - static inline unsigned long intel_iommu_get_pts(struct intel_iommu *iommu) 5234 + static inline unsigned long intel_iommu_get_pts(struct device *dev) 5241 5235 { 5242 - /* 5243 - * Convert ecap_pss to extend context entry pts encoding, also 5244 - * respect the soft pasid_max value set by the iommu. 5245 - * - number of PASID bits = ecap_pss + 1 5246 - * - number of PASID table entries = 2^(pts + 5) 5247 - * Therefore, pts = ecap_pss - 4 5248 - * e.g. KBL ecap_pss = 0x13, PASID has 20 bits, pts = 15 5249 - */ 5250 - if (ecap_pss(iommu->ecap) < 5) 5236 + int pts, max_pasid; 5237 + 5238 + max_pasid = intel_pasid_get_dev_max_id(dev); 5239 + pts = find_first_bit((unsigned long *)&max_pasid, MAX_NR_PASID_BITS); 5240 + if (pts < 5) 5251 5241 return 0; 5252 5242 5253 - /* pasid_max is encoded as actual number of entries not the bits */ 5254 - return find_first_bit((unsigned long *)&iommu->pasid_max, 5255 - MAX_NR_PASID_BITS) - 5; 5243 + return pts - 5; 5256 5244 } 5257 5245 5258 5246 int intel_iommu_enable_pasid(struct intel_iommu *iommu, struct intel_svm_dev *sdev) ··· 5282 5282 if (!(ctx_lo & CONTEXT_PASIDE)) { 5283 5283 if (iommu->pasid_state_table) 5284 5284 context[1].hi = (u64)virt_to_phys(iommu->pasid_state_table); 5285 - context[1].lo = (u64)virt_to_phys(iommu->pasid_table) | 5286 - intel_iommu_get_pts(iommu); 5285 + context[1].lo = (u64)virt_to_phys(info->pasid_table->table) | 5286 + intel_iommu_get_pts(sdev->dev); 5287 5287 5288 5288 wmb(); 5289 5289 /* CONTEXT_TT_MULTI_LEVEL and CONTEXT_TT_DEV_IOTLB are both ··· 5350 5350 return NULL; 5351 5351 } 5352 5352 5353 - if (!iommu->pasid_table) { 5354 - dev_err(dev, "PASID not enabled on IOMMU; cannot enable SVM\n"); 5355 - return NULL; 5356 - } 5357 - 5358 5353 return iommu; 5359 5354 } 5360 5355 #endif /* CONFIG_INTEL_IOMMU_SVM */ ··· 5362 5367 .detach_dev = intel_iommu_detach_device, 5363 5368 .map = intel_iommu_map, 5364 5369 .unmap = intel_iommu_unmap, 5365 - .map_sg = default_iommu_map_sg, 5366 5370 .iova_to_phys = intel_iommu_iova_to_phys, 5367 5371 .add_device = intel_iommu_add_device, 5368 5372 .remove_device = intel_iommu_remove_device,

+239

drivers/iommu/intel-pasid.c

··· 1 + // SPDX-License-Identifier: GPL-2.0 2 + /** 3 + * intel-pasid.c - PASID idr, table and entry manipulation 4 + * 5 + * Copyright (C) 2018 Intel Corporation 6 + * 7 + * Author: Lu Baolu <baolu.lu@linux.intel.com> 8 + */ 9 + 10 + #define pr_fmt(fmt) "DMAR: " fmt 11 + 12 + #include <linux/dmar.h> 13 + #include <linux/intel-iommu.h> 14 + #include <linux/iommu.h> 15 + #include <linux/memory.h> 16 + #include <linux/pci.h> 17 + #include <linux/pci-ats.h> 18 + #include <linux/spinlock.h> 19 + 20 + #include "intel-pasid.h" 21 + 22 + /* 23 + * Intel IOMMU system wide PASID name space: 24 + */ 25 + static DEFINE_SPINLOCK(pasid_lock); 26 + u32 intel_pasid_max_id = PASID_MAX; 27 + static DEFINE_IDR(pasid_idr); 28 + 29 + int intel_pasid_alloc_id(void *ptr, int start, int end, gfp_t gfp) 30 + { 31 + int ret, min, max; 32 + 33 + min = max_t(int, start, PASID_MIN); 34 + max = min_t(int, end, intel_pasid_max_id); 35 + 36 + WARN_ON(in_interrupt()); 37 + idr_preload(gfp); 38 + spin_lock(&pasid_lock); 39 + ret = idr_alloc(&pasid_idr, ptr, min, max, GFP_ATOMIC); 40 + spin_unlock(&pasid_lock); 41 + idr_preload_end(); 42 + 43 + return ret; 44 + } 45 + 46 + void intel_pasid_free_id(int pasid) 47 + { 48 + spin_lock(&pasid_lock); 49 + idr_remove(&pasid_idr, pasid); 50 + spin_unlock(&pasid_lock); 51 + } 52 + 53 + void *intel_pasid_lookup_id(int pasid) 54 + { 55 + void *p; 56 + 57 + spin_lock(&pasid_lock); 58 + p = idr_find(&pasid_idr, pasid); 59 + spin_unlock(&pasid_lock); 60 + 61 + return p; 62 + } 63 + 64 + /* 65 + * Per device pasid table management: 66 + */ 67 + static inline void 68 + device_attach_pasid_table(struct device_domain_info *info, 69 + struct pasid_table *pasid_table) 70 + { 71 + info->pasid_table = pasid_table; 72 + list_add(&info->table, &pasid_table->dev); 73 + } 74 + 75 + static inline void 76 + device_detach_pasid_table(struct device_domain_info *info, 77 + struct pasid_table *pasid_table) 78 + { 79 + info->pasid_table = NULL; 80 + list_del(&info->table); 81 + } 82 + 83 + struct pasid_table_opaque { 84 + struct pasid_table **pasid_table; 85 + int segment; 86 + int bus; 87 + int devfn; 88 + }; 89 + 90 + static int search_pasid_table(struct device_domain_info *info, void *opaque) 91 + { 92 + struct pasid_table_opaque *data = opaque; 93 + 94 + if (info->iommu->segment == data->segment && 95 + info->bus == data->bus && 96 + info->devfn == data->devfn && 97 + info->pasid_table) { 98 + *data->pasid_table = info->pasid_table; 99 + return 1; 100 + } 101 + 102 + return 0; 103 + } 104 + 105 + static int get_alias_pasid_table(struct pci_dev *pdev, u16 alias, void *opaque) 106 + { 107 + struct pasid_table_opaque *data = opaque; 108 + 109 + data->segment = pci_domain_nr(pdev->bus); 110 + data->bus = PCI_BUS_NUM(alias); 111 + data->devfn = alias & 0xff; 112 + 113 + return for_each_device_domain(&search_pasid_table, data); 114 + } 115 + 116 + /* 117 + * Allocate a pasid table for @dev. It should be called in a 118 + * single-thread context. 119 + */ 120 + int intel_pasid_alloc_table(struct device *dev) 121 + { 122 + struct device_domain_info *info; 123 + struct pasid_table *pasid_table; 124 + struct pasid_table_opaque data; 125 + struct page *pages; 126 + size_t size, count; 127 + int ret, order; 128 + 129 + info = dev->archdata.iommu; 130 + if (WARN_ON(!info || !dev_is_pci(dev) || 131 + !info->pasid_supported || info->pasid_table)) 132 + return -EINVAL; 133 + 134 + /* DMA alias device already has a pasid table, use it: */ 135 + data.pasid_table = &pasid_table; 136 + ret = pci_for_each_dma_alias(to_pci_dev(dev), 137 + &get_alias_pasid_table, &data); 138 + if (ret) 139 + goto attach_out; 140 + 141 + pasid_table = kzalloc(sizeof(*pasid_table), GFP_ATOMIC); 142 + if (!pasid_table) 143 + return -ENOMEM; 144 + INIT_LIST_HEAD(&pasid_table->dev); 145 + 146 + size = sizeof(struct pasid_entry); 147 + count = min_t(int, pci_max_pasids(to_pci_dev(dev)), intel_pasid_max_id); 148 + order = get_order(size * count); 149 + pages = alloc_pages_node(info->iommu->node, 150 + GFP_ATOMIC | __GFP_ZERO, 151 + order); 152 + if (!pages) 153 + return -ENOMEM; 154 + 155 + pasid_table->table = page_address(pages); 156 + pasid_table->order = order; 157 + pasid_table->max_pasid = count; 158 + 159 + attach_out: 160 + device_attach_pasid_table(info, pasid_table); 161 + 162 + return 0; 163 + } 164 + 165 + void intel_pasid_free_table(struct device *dev) 166 + { 167 + struct device_domain_info *info; 168 + struct pasid_table *pasid_table; 169 + 170 + info = dev->archdata.iommu; 171 + if (!info || !dev_is_pci(dev) || 172 + !info->pasid_supported || !info->pasid_table) 173 + return; 174 + 175 + pasid_table = info->pasid_table; 176 + device_detach_pasid_table(info, pasid_table); 177 + 178 + if (!list_empty(&pasid_table->dev)) 179 + return; 180 + 181 + free_pages((unsigned long)pasid_table->table, pasid_table->order); 182 + kfree(pasid_table); 183 + } 184 + 185 + struct pasid_table *intel_pasid_get_table(struct device *dev) 186 + { 187 + struct device_domain_info *info; 188 + 189 + info = dev->archdata.iommu; 190 + if (!info) 191 + return NULL; 192 + 193 + return info->pasid_table; 194 + } 195 + 196 + int intel_pasid_get_dev_max_id(struct device *dev) 197 + { 198 + struct device_domain_info *info; 199 + 200 + info = dev->archdata.iommu; 201 + if (!info || !info->pasid_table) 202 + return 0; 203 + 204 + return info->pasid_table->max_pasid; 205 + } 206 + 207 + struct pasid_entry *intel_pasid_get_entry(struct device *dev, int pasid) 208 + { 209 + struct pasid_table *pasid_table; 210 + struct pasid_entry *entries; 211 + 212 + pasid_table = intel_pasid_get_table(dev); 213 + if (WARN_ON(!pasid_table || pasid < 0 || 214 + pasid >= intel_pasid_get_dev_max_id(dev))) 215 + return NULL; 216 + 217 + entries = pasid_table->table; 218 + 219 + return &entries[pasid]; 220 + } 221 + 222 + /* 223 + * Interfaces for PASID table entry manipulation: 224 + */ 225 + static inline void pasid_clear_entry(struct pasid_entry *pe) 226 + { 227 + WRITE_ONCE(pe->val, 0); 228 + } 229 + 230 + void intel_pasid_clear_entry(struct device *dev, int pasid) 231 + { 232 + struct pasid_entry *pe; 233 + 234 + pe = intel_pasid_get_entry(dev, pasid); 235 + if (WARN_ON(!pe)) 236 + return; 237 + 238 + pasid_clear_entry(pe); 239 + }

+39

drivers/iommu/intel-pasid.h

··· 1 + /* SPDX-License-Identifier: GPL-2.0 */ 2 + /* 3 + * intel-pasid.h - PASID idr, table and entry header 4 + * 5 + * Copyright (C) 2018 Intel Corporation 6 + * 7 + * Author: Lu Baolu <baolu.lu@linux.intel.com> 8 + */ 9 + 10 + #ifndef __INTEL_PASID_H 11 + #define __INTEL_PASID_H 12 + 13 + #define PASID_MIN 0x1 14 + #define PASID_MAX 0x100000 15 + 16 + struct pasid_entry { 17 + u64 val; 18 + }; 19 + 20 + /* The representative of a PASID table */ 21 + struct pasid_table { 22 + void *table; /* pasid table pointer */ 23 + int order; /* page order of pasid table */ 24 + int max_pasid; /* max pasid */ 25 + struct list_head dev; /* device list */ 26 + }; 27 + 28 + extern u32 intel_pasid_max_id; 29 + int intel_pasid_alloc_id(void *ptr, int start, int end, gfp_t gfp); 30 + void intel_pasid_free_id(int pasid); 31 + void *intel_pasid_lookup_id(int pasid); 32 + int intel_pasid_alloc_table(struct device *dev); 33 + void intel_pasid_free_table(struct device *dev); 34 + struct pasid_table *intel_pasid_get_table(struct device *dev); 35 + int intel_pasid_get_dev_max_id(struct device *dev); 36 + struct pasid_entry *intel_pasid_get_entry(struct device *dev, int pasid); 37 + void intel_pasid_clear_entry(struct device *dev, int pasid); 38 + 39 + #endif /* __INTEL_PASID_H */

+33 -46

drivers/iommu/intel-svm.c

··· 27 27 #include <linux/mm_types.h> 28 28 #include <asm/page.h> 29 29 30 + #include "intel-pasid.h" 31 + 30 32 #define PASID_ENTRY_P BIT_ULL(0) 31 33 #define PASID_ENTRY_FLPM_5LP BIT_ULL(9) 32 34 #define PASID_ENTRY_SRE BIT_ULL(11) 33 35 34 36 static irqreturn_t prq_event_thread(int irq, void *d); 35 37 36 - struct pasid_entry { 37 - u64 val; 38 - }; 39 - 40 38 struct pasid_state_entry { 41 39 u64 val; 42 40 }; 43 41 44 - int intel_svm_alloc_pasid_tables(struct intel_iommu *iommu) 42 + int intel_svm_init(struct intel_iommu *iommu) 45 43 { 46 44 struct page *pages; 47 45 int order; ··· 64 66 iommu->pasid_max = 0x20000; 65 67 66 68 order = get_order(sizeof(struct pasid_entry) * iommu->pasid_max); 67 - pages = alloc_pages(GFP_KERNEL | __GFP_ZERO, order); 68 - if (!pages) { 69 - pr_warn("IOMMU: %s: Failed to allocate PASID table\n", 70 - iommu->name); 71 - return -ENOMEM; 72 - } 73 - iommu->pasid_table = page_address(pages); 74 - pr_info("%s: Allocated order %d PASID table.\n", iommu->name, order); 75 - 76 69 if (ecap_dis(iommu->ecap)) { 77 70 /* Just making it explicit... */ 78 71 BUILD_BUG_ON(sizeof(struct pasid_entry) != sizeof(struct pasid_state_entry)); ··· 75 86 iommu->name); 76 87 } 77 88 78 - idr_init(&iommu->pasid_idr); 79 - 80 89 return 0; 81 90 } 82 91 83 - int intel_svm_free_pasid_tables(struct intel_iommu *iommu) 92 + int intel_svm_exit(struct intel_iommu *iommu) 84 93 { 85 94 int order = get_order(sizeof(struct pasid_entry) * iommu->pasid_max); 86 95 87 - if (iommu->pasid_table) { 88 - free_pages((unsigned long)iommu->pasid_table, order); 89 - iommu->pasid_table = NULL; 90 - } 91 96 if (iommu->pasid_state_table) { 92 97 free_pages((unsigned long)iommu->pasid_state_table, order); 93 98 iommu->pasid_state_table = NULL; 94 99 } 95 - idr_destroy(&iommu->pasid_idr); 100 + 96 101 return 0; 97 102 } 98 103 ··· 262 279 * page) so that we end up taking a fault that the hardware really 263 280 * *has* to handle gracefully without affecting other processes. 264 281 */ 265 - svm->iommu->pasid_table[svm->pasid].val = 0; 266 - wmb(); 267 - 268 282 rcu_read_lock(); 269 283 list_for_each_entry_rcu(sdev, &svm->devs, list) { 284 + intel_pasid_clear_entry(sdev->dev, svm->pasid); 270 285 intel_flush_pasid_dev(svm, sdev, svm->pasid); 271 286 intel_flush_svm_range_dev(svm, sdev, 0, -1, 0, !svm->mm); 272 287 } ··· 280 299 }; 281 300 282 301 static DEFINE_MUTEX(pasid_mutex); 302 + static LIST_HEAD(global_svm_list); 283 303 284 304 int intel_svm_bind_mm(struct device *dev, int *pasid, int flags, struct svm_dev_ops *ops) 285 305 { 286 306 struct intel_iommu *iommu = intel_svm_device_to_iommu(dev); 307 + struct pasid_entry *entry; 287 308 struct intel_svm_dev *sdev; 288 309 struct intel_svm *svm = NULL; 289 310 struct mm_struct *mm = NULL; ··· 293 310 int pasid_max; 294 311 int ret; 295 312 296 - if (WARN_ON(!iommu || !iommu->pasid_table)) 313 + if (!iommu) 297 314 return -EINVAL; 298 315 299 316 if (dev_is_pci(dev)) { ··· 313 330 314 331 mutex_lock(&pasid_mutex); 315 332 if (pasid && !(flags & SVM_FLAG_PRIVATE_PASID)) { 316 - int i; 333 + struct intel_svm *t; 317 334 318 - idr_for_each_entry(&iommu->pasid_idr, svm, i) { 319 - if (svm->mm != mm || 320 - (svm->flags & SVM_FLAG_PRIVATE_PASID)) 335 + list_for_each_entry(t, &global_svm_list, list) { 336 + if (t->mm != mm || (t->flags & SVM_FLAG_PRIVATE_PASID)) 321 337 continue; 322 338 339 + svm = t; 323 340 if (svm->pasid >= pasid_max) { 324 341 dev_warn(dev, 325 342 "Limited PASID width. Cannot use existing PASID %d\n", ··· 371 388 } 372 389 svm->iommu = iommu; 373 390 374 - if (pasid_max > iommu->pasid_max) 375 - pasid_max = iommu->pasid_max; 391 + if (pasid_max > intel_pasid_max_id) 392 + pasid_max = intel_pasid_max_id; 376 393 377 394 /* Do not use PASID 0 in caching mode (virtualised IOMMU) */ 378 - ret = idr_alloc(&iommu->pasid_idr, svm, 379 - !!cap_caching_mode(iommu->cap), 380 - pasid_max - 1, GFP_KERNEL); 395 + ret = intel_pasid_alloc_id(svm, 396 + !!cap_caching_mode(iommu->cap), 397 + pasid_max - 1, GFP_KERNEL); 381 398 if (ret < 0) { 382 399 kfree(svm); 383 400 kfree(sdev); ··· 388 405 svm->mm = mm; 389 406 svm->flags = flags; 390 407 INIT_LIST_HEAD_RCU(&svm->devs); 408 + INIT_LIST_HEAD(&svm->list); 391 409 ret = -ENOMEM; 392 410 if (mm) { 393 411 ret = mmu_notifier_register(&svm->notifier, mm); 394 412 if (ret) { 395 - idr_remove(&svm->iommu->pasid_idr, svm->pasid); 413 + intel_pasid_free_id(svm->pasid); 396 414 kfree(svm); 397 415 kfree(sdev); 398 416 goto out; ··· 405 421 if (cpu_feature_enabled(X86_FEATURE_LA57)) 406 422 pasid_entry_val |= PASID_ENTRY_FLPM_5LP; 407 423 408 - iommu->pasid_table[svm->pasid].val = pasid_entry_val; 424 + entry = intel_pasid_get_entry(dev, svm->pasid); 425 + entry->val = pasid_entry_val; 409 426 410 427 wmb(); 411 428 ··· 416 431 */ 417 432 if (cap_caching_mode(iommu->cap)) 418 433 intel_flush_pasid_dev(svm, sdev, svm->pasid); 434 + 435 + list_add_tail(&svm->list, &global_svm_list); 419 436 } 420 437 list_add_rcu(&sdev->list, &svm->devs); 421 438 ··· 441 454 442 455 mutex_lock(&pasid_mutex); 443 456 iommu = intel_svm_device_to_iommu(dev); 444 - if (!iommu || !iommu->pasid_table) 457 + if (!iommu) 445 458 goto out; 446 459 447 - svm = idr_find(&iommu->pasid_idr, pasid); 460 + svm = intel_pasid_lookup_id(pasid); 448 461 if (!svm) 449 462 goto out; 450 463 ··· 464 477 intel_flush_pasid_dev(svm, sdev, svm->pasid); 465 478 intel_flush_svm_range_dev(svm, sdev, 0, -1, 0, !svm->mm); 466 479 kfree_rcu(sdev, rcu); 480 + intel_pasid_clear_entry(dev, svm->pasid); 467 481 468 482 if (list_empty(&svm->devs)) { 469 - svm->iommu->pasid_table[svm->pasid].val = 0; 470 - wmb(); 471 - 472 - idr_remove(&svm->iommu->pasid_idr, svm->pasid); 483 + intel_pasid_free_id(svm->pasid); 473 484 if (svm->mm) 474 485 mmu_notifier_unregister(&svm->notifier, svm->mm); 486 + 487 + list_del(&svm->list); 475 488 476 489 /* We mandate that no page faults may be outstanding 477 490 * for the PASID when intel_svm_unbind_mm() is called. ··· 499 512 500 513 mutex_lock(&pasid_mutex); 501 514 iommu = intel_svm_device_to_iommu(dev); 502 - if (!iommu || !iommu->pasid_table) 515 + if (!iommu) 503 516 goto out; 504 517 505 - svm = idr_find(&iommu->pasid_idr, pasid); 518 + svm = intel_pasid_lookup_id(pasid); 506 519 if (!svm) 507 520 goto out; 508 521 ··· 601 614 602 615 if (!svm || svm->pasid != req->pasid) { 603 616 rcu_read_lock(); 604 - svm = idr_find(&iommu->pasid_idr, req->pasid); 617 + svm = intel_pasid_lookup_id(req->pasid); 605 618 /* It *can't* go away, because the driver is not permitted 606 619 * to unbind the mm while any page faults are outstanding. 607 620 * So we only need RCU to protect the internal idr code. */

+6 -1

drivers/iommu/io-pgtable-arm-v7s.c

··· 192 192 { 193 193 struct io_pgtable_cfg *cfg = &data->iop.cfg; 194 194 struct device *dev = cfg->iommu_dev; 195 + phys_addr_t phys; 195 196 dma_addr_t dma; 196 197 size_t size = ARM_V7S_TABLE_SIZE(lvl); 197 198 void *table = NULL; ··· 201 200 table = (void *)__get_dma_pages(__GFP_ZERO, get_order(size)); 202 201 else if (lvl == 2) 203 202 table = kmem_cache_zalloc(data->l2_tables, gfp | GFP_DMA); 203 + phys = virt_to_phys(table); 204 + if (phys != (arm_v7s_iopte)phys) 205 + /* Doesn't fit in PTE */ 206 + goto out_free; 204 207 if (table && !(cfg->quirks & IO_PGTABLE_QUIRK_NO_DMA)) { 205 208 dma = dma_map_single(dev, table, size, DMA_TO_DEVICE); 206 209 if (dma_mapping_error(dev, dma)) ··· 214 209 * address directly, so if the DMA layer suggests otherwise by 215 210 * translating or truncating them, that bodes very badly... 216 211 */ 217 - if (dma != virt_to_phys(table)) 212 + if (dma != phys) 218 213 goto out_unmap; 219 214 } 220 215 kmemleak_ignore(table);

+2 -1

drivers/iommu/io-pgtable-arm.c

··· 237 237 void *pages; 238 238 239 239 VM_BUG_ON((gfp & __GFP_HIGHMEM)); 240 - p = alloc_pages_node(dev_to_node(dev), gfp | __GFP_ZERO, order); 240 + p = alloc_pages_node(dev ? dev_to_node(dev) : NUMA_NO_NODE, 241 + gfp | __GFP_ZERO, order); 241 242 if (!p) 242 243 return NULL; 243 244

+66

drivers/iommu/iommu-debugfs.c

··· 1 + // SPDX-License-Identifier: GPL-2.0 2 + /* 3 + * IOMMU debugfs core infrastructure 4 + * 5 + * Copyright (C) 2018 Advanced Micro Devices, Inc. 6 + * 7 + * Author: Gary R Hook <gary.hook@amd.com> 8 + */ 9 + 10 + #include <linux/pci.h> 11 + #include <linux/iommu.h> 12 + #include <linux/debugfs.h> 13 + 14 + struct dentry *iommu_debugfs_dir; 15 + 16 + /** 17 + * iommu_debugfs_setup - create the top-level iommu directory in debugfs 18 + * 19 + * Provide base enablement for using debugfs to expose internal data of an 20 + * IOMMU driver. When called, this function creates the 21 + * /sys/kernel/debug/iommu directory. 22 + * 23 + * Emit a strong warning at boot time to indicate that this feature is 24 + * enabled. 25 + * 26 + * This function is called from iommu_init; drivers may then call 27 + * iommu_debugfs_new_driver_dir() to instantiate a vendor-specific 28 + * directory to be used to expose internal data. 29 + */ 30 + void iommu_debugfs_setup(void) 31 + { 32 + if (!iommu_debugfs_dir) { 33 + iommu_debugfs_dir = debugfs_create_dir("iommu", NULL); 34 + pr_warn("\n"); 35 + pr_warn("*************************************************************\n"); 36 + pr_warn("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n"); 37 + pr_warn("** **\n"); 38 + pr_warn("** IOMMU DebugFS SUPPORT HAS BEEN ENABLED IN THIS KERNEL **\n"); 39 + pr_warn("** **\n"); 40 + pr_warn("** This means that this kernel is built to expose internal **\n"); 41 + pr_warn("** IOMMU data structures, which may compromise security on **\n"); 42 + pr_warn("** your system. **\n"); 43 + pr_warn("** **\n"); 44 + pr_warn("** If you see this message and you are not debugging the **\n"); 45 + pr_warn("** kernel, report this immediately to your vendor! **\n"); 46 + pr_warn("** **\n"); 47 + pr_warn("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n"); 48 + pr_warn("*************************************************************\n"); 49 + } 50 + } 51 + 52 + /** 53 + * iommu_debugfs_new_driver_dir - create a vendor directory under debugfs/iommu 54 + * @vendor: name of the vendor-specific subdirectory to create 55 + * 56 + * This function is called by an IOMMU driver to create the top-level debugfs 57 + * directory for that driver. 58 + * 59 + * Return: upon success, a pointer to the dentry for the new directory. 60 + * NULL in case of failure. 61 + */ 62 + struct dentry *iommu_debugfs_new_driver_dir(const char *vendor) 63 + { 64 + return debugfs_create_dir(vendor, iommu_debugfs_dir); 65 + } 66 + EXPORT_SYMBOL_GPL(iommu_debugfs_new_driver_dir);

+41 -3

drivers/iommu/iommu.c

··· 36 36 37 37 static struct kset *iommu_group_kset; 38 38 static DEFINE_IDA(iommu_group_ida); 39 + #ifdef CONFIG_IOMMU_DEFAULT_PASSTHROUGH 40 + static unsigned int iommu_def_domain_type = IOMMU_DOMAIN_IDENTITY; 41 + #else 39 42 static unsigned int iommu_def_domain_type = IOMMU_DOMAIN_DMA; 43 + #endif 40 44 41 45 struct iommu_callback_data { 42 46 const struct iommu_ops *ops; ··· 298 294 return (str - buf); 299 295 } 300 296 297 + static ssize_t iommu_group_show_type(struct iommu_group *group, 298 + char *buf) 299 + { 300 + char *type = "unknown\n"; 301 + 302 + if (group->default_domain) { 303 + switch (group->default_domain->type) { 304 + case IOMMU_DOMAIN_BLOCKED: 305 + type = "blocked\n"; 306 + break; 307 + case IOMMU_DOMAIN_IDENTITY: 308 + type = "identity\n"; 309 + break; 310 + case IOMMU_DOMAIN_UNMANAGED: 311 + type = "unmanaged\n"; 312 + break; 313 + case IOMMU_DOMAIN_DMA: 314 + type = "DMA"; 315 + break; 316 + } 317 + } 318 + strcpy(buf, type); 319 + 320 + return strlen(type); 321 + } 322 + 301 323 static IOMMU_GROUP_ATTR(name, S_IRUGO, iommu_group_show_name, NULL); 302 324 303 325 static IOMMU_GROUP_ATTR(reserved_regions, 0444, 304 326 iommu_group_show_resv_regions, NULL); 327 + 328 + static IOMMU_GROUP_ATTR(type, 0444, iommu_group_show_type, NULL); 305 329 306 330 static void iommu_group_release(struct kobject *kobj) 307 331 { ··· 409 377 410 378 ret = iommu_group_create_file(group, 411 379 &iommu_group_attr_reserved_regions); 380 + if (ret) 381 + return ERR_PTR(ret); 382 + 383 + ret = iommu_group_create_file(group, &iommu_group_attr_type); 412 384 if (ret) 413 385 return ERR_PTR(ret); 414 386 ··· 1673 1637 } 1674 1638 EXPORT_SYMBOL_GPL(iommu_unmap_fast); 1675 1639 1676 - size_t default_iommu_map_sg(struct iommu_domain *domain, unsigned long iova, 1677 - struct scatterlist *sg, unsigned int nents, int prot) 1640 + size_t iommu_map_sg(struct iommu_domain *domain, unsigned long iova, 1641 + struct scatterlist *sg, unsigned int nents, int prot) 1678 1642 { 1679 1643 struct scatterlist *s; 1680 1644 size_t mapped = 0; ··· 1714 1678 return 0; 1715 1679 1716 1680 } 1717 - EXPORT_SYMBOL_GPL(default_iommu_map_sg); 1681 + EXPORT_SYMBOL_GPL(iommu_map_sg); 1718 1682 1719 1683 int iommu_domain_window_enable(struct iommu_domain *domain, u32 wnd_nr, 1720 1684 phys_addr_t paddr, u64 size, int prot) ··· 1783 1747 iommu_group_kset = kset_create_and_add("iommu_groups", 1784 1748 NULL, kernel_kobj); 1785 1749 BUG_ON(!iommu_group_kset); 1750 + 1751 + iommu_debugfs_setup(); 1786 1752 1787 1753 return 0; 1788 1754 }

+46 -13

drivers/iommu/ipmmu-vmsa.c

··· 47 47 unsigned int number_of_contexts; 48 48 bool setup_imbuscr; 49 49 bool twobit_imttbcr_sl0; 50 + bool reserved_context; 50 51 }; 51 52 52 53 struct ipmmu_vmsa_device { ··· 74 73 struct io_pgtable_ops *iop; 75 74 76 75 unsigned int context_id; 77 - spinlock_t lock; /* Protects mappings */ 76 + struct mutex mutex; /* Protects mappings */ 78 77 }; 79 78 80 79 static struct ipmmu_vmsa_domain *to_vmsa_domain(struct iommu_domain *dom) ··· 195 194 #define IMPMBA(n) (0x0280 + ((n) * 4)) 196 195 #define IMPMBD(n) (0x02c0 + ((n) * 4)) 197 196 198 - #define IMUCTR(n) (0x0300 + ((n) * 16)) 197 + #define IMUCTR(n) ((n) < 32 ? IMUCTR0(n) : IMUCTR32(n)) 198 + #define IMUCTR0(n) (0x0300 + ((n) * 16)) 199 + #define IMUCTR32(n) (0x0600 + (((n) - 32) * 16)) 199 200 #define IMUCTR_FIXADDEN (1 << 31) 200 201 #define IMUCTR_FIXADD_MASK (0xff << 16) 201 202 #define IMUCTR_FIXADD_SHIFT 16 ··· 207 204 #define IMUCTR_FLUSH (1 << 1) 208 205 #define IMUCTR_MMUEN (1 << 0) 209 206 210 - #define IMUASID(n) (0x0308 + ((n) * 16)) 207 + #define IMUASID(n) ((n) < 32 ? IMUASID0(n) : IMUASID32(n)) 208 + #define IMUASID0(n) (0x0308 + ((n) * 16)) 209 + #define IMUASID32(n) (0x0608 + (((n) - 32) * 16)) 211 210 #define IMUASID_ASID8_MASK (0xff << 8) 212 211 #define IMUASID_ASID8_SHIFT 8 213 212 #define IMUASID_ASID0_MASK (0xff << 0) ··· 600 595 if (!domain) 601 596 return NULL; 602 597 603 - spin_lock_init(&domain->lock); 598 + mutex_init(&domain->mutex); 604 599 605 600 return &domain->io_domain; 606 601 } ··· 646 641 struct iommu_fwspec *fwspec = dev->iommu_fwspec; 647 642 struct ipmmu_vmsa_device *mmu = to_ipmmu(dev); 648 643 struct ipmmu_vmsa_domain *domain = to_vmsa_domain(io_domain); 649 - unsigned long flags; 650 644 unsigned int i; 651 645 int ret = 0; 652 646 ··· 654 650 return -ENXIO; 655 651 } 656 652 657 - spin_lock_irqsave(&domain->lock, flags); 653 + mutex_lock(&domain->mutex); 658 654 659 655 if (!domain->mmu) { 660 656 /* The domain hasn't been used yet, initialize it. */ ··· 678 674 } else 679 675 dev_info(dev, "Reusing IPMMU context %u\n", domain->context_id); 680 676 681 - spin_unlock_irqrestore(&domain->lock, flags); 677 + mutex_unlock(&domain->mutex); 682 678 683 679 if (ret < 0) 684 680 return ret; ··· 760 756 return false; 761 757 } 762 758 763 - static const struct soc_device_attribute soc_r8a7795[] = { 759 + static const struct soc_device_attribute soc_rcar_gen3[] = { 764 760 { .soc_id = "r8a7795", }, 761 + { .soc_id = "r8a7796", }, 762 + { .soc_id = "r8a77965", }, 763 + { .soc_id = "r8a77970", }, 764 + { .soc_id = "r8a77995", }, 765 765 { /* sentinel */ } 766 766 }; 767 767 ··· 773 765 struct of_phandle_args *spec) 774 766 { 775 767 /* For R-Car Gen3 use a white list to opt-in slave devices */ 776 - if (soc_device_match(soc_r8a7795) && !ipmmu_slave_whitelist(dev)) 768 + if (soc_device_match(soc_rcar_gen3) && !ipmmu_slave_whitelist(dev)) 777 769 return -ENODEV; 778 770 779 771 iommu_fwspec_add_ids(dev, spec->args, 1); ··· 897 889 .unmap = ipmmu_unmap, 898 890 .flush_iotlb_all = ipmmu_iotlb_sync, 899 891 .iotlb_sync = ipmmu_iotlb_sync, 900 - .map_sg = default_iommu_map_sg, 901 892 .iova_to_phys = ipmmu_iova_to_phys, 902 893 .add_device = ipmmu_add_device, 903 894 .remove_device = ipmmu_remove_device, ··· 924 917 .number_of_contexts = 1, /* software only tested with one context */ 925 918 .setup_imbuscr = true, 926 919 .twobit_imttbcr_sl0 = false, 920 + .reserved_context = false, 927 921 }; 928 922 929 - static const struct ipmmu_features ipmmu_features_r8a7795 = { 923 + static const struct ipmmu_features ipmmu_features_rcar_gen3 = { 930 924 .use_ns_alias_offset = false, 931 925 .has_cache_leaf_nodes = true, 932 926 .number_of_contexts = 8, 933 927 .setup_imbuscr = false, 934 928 .twobit_imttbcr_sl0 = true, 929 + .reserved_context = true, 935 930 }; 936 931 937 932 static const struct of_device_id ipmmu_of_ids[] = { ··· 942 933 .data = &ipmmu_features_default, 943 934 }, { 944 935 .compatible = "renesas,ipmmu-r8a7795", 945 - .data = &ipmmu_features_r8a7795, 936 + .data = &ipmmu_features_rcar_gen3, 937 + }, { 938 + .compatible = "renesas,ipmmu-r8a7796", 939 + .data = &ipmmu_features_rcar_gen3, 940 + }, { 941 + .compatible = "renesas,ipmmu-r8a77965", 942 + .data = &ipmmu_features_rcar_gen3, 943 + }, { 944 + .compatible = "renesas,ipmmu-r8a77970", 945 + .data = &ipmmu_features_rcar_gen3, 946 + }, { 947 + .compatible = "renesas,ipmmu-r8a77995", 948 + .data = &ipmmu_features_rcar_gen3, 946 949 }, { 947 950 /* Terminator */ 948 951 }, ··· 976 955 } 977 956 978 957 mmu->dev = &pdev->dev; 979 - mmu->num_utlbs = 32; 958 + mmu->num_utlbs = 48; 980 959 spin_lock_init(&mmu->lock); 981 960 bitmap_zero(mmu->ctx, IPMMU_CTX_MAX); 982 961 mmu->features = of_device_get_match_data(&pdev->dev); ··· 1039 1018 } 1040 1019 1041 1020 ipmmu_device_reset(mmu); 1021 + 1022 + if (mmu->features->reserved_context) { 1023 + dev_info(&pdev->dev, "IPMMU context 0 is reserved\n"); 1024 + set_bit(0, mmu->ctx); 1025 + } 1042 1026 } 1043 1027 1044 1028 /* ··· 1107 1081 1108 1082 static int __init ipmmu_init(void) 1109 1083 { 1084 + struct device_node *np; 1110 1085 static bool setup_done; 1111 1086 int ret; 1112 1087 1113 1088 if (setup_done) 1114 1089 return 0; 1090 + 1091 + np = of_find_matching_node(NULL, ipmmu_of_ids); 1092 + if (!np) 1093 + return 0; 1094 + 1095 + of_node_put(np); 1115 1096 1116 1097 ret = platform_driver_register(&ipmmu_driver); 1117 1098 if (ret < 0)

+5 -12

drivers/iommu/msm_iommu.c

··· 395 395 struct msm_iommu_dev *iommu; 396 396 struct iommu_group *group; 397 397 unsigned long flags; 398 - int ret = 0; 399 398 400 399 spin_lock_irqsave(&msm_iommu_lock, flags); 401 - 402 400 iommu = find_iommu_for_dev(dev); 401 + spin_unlock_irqrestore(&msm_iommu_lock, flags); 402 + 403 403 if (iommu) 404 404 iommu_device_link(&iommu->iommu, dev); 405 405 else 406 - ret = -ENODEV; 407 - 408 - spin_unlock_irqrestore(&msm_iommu_lock, flags); 409 - 410 - if (ret) 411 - return ret; 406 + return -ENODEV; 412 407 413 408 group = iommu_group_get_for_dev(dev); 414 409 if (IS_ERR(group)) ··· 420 425 unsigned long flags; 421 426 422 427 spin_lock_irqsave(&msm_iommu_lock, flags); 423 - 424 428 iommu = find_iommu_for_dev(dev); 429 + spin_unlock_irqrestore(&msm_iommu_lock, flags); 430 + 425 431 if (iommu) 426 432 iommu_device_unlink(&iommu->iommu, dev); 427 - 428 - spin_unlock_irqrestore(&msm_iommu_lock, flags); 429 433 430 434 iommu_group_remove_device(dev); 431 435 } ··· 702 708 .detach_dev = msm_iommu_detach_dev, 703 709 .map = msm_iommu_map, 704 710 .unmap = msm_iommu_unmap, 705 - .map_sg = default_iommu_map_sg, 706 711 .iova_to_phys = msm_iommu_iova_to_phys, 707 712 .add_device = msm_iommu_add_device, 708 713 .remove_device = msm_iommu_remove_device,

-1

drivers/iommu/mtk_iommu.c

··· 495 495 .detach_dev = mtk_iommu_detach_device, 496 496 .map = mtk_iommu_map, 497 497 .unmap = mtk_iommu_unmap, 498 - .map_sg = default_iommu_map_sg, 499 498 .flush_iotlb_all = mtk_iommu_iotlb_sync, 500 499 .iotlb_sync = mtk_iommu_iotlb_sync, 501 500 .iova_to_phys = mtk_iommu_iova_to_phys,

-1

drivers/iommu/mtk_iommu_v1.c

··· 531 531 .detach_dev = mtk_iommu_detach_device, 532 532 .map = mtk_iommu_map, 533 533 .unmap = mtk_iommu_unmap, 534 - .map_sg = default_iommu_map_sg, 535 534 .iova_to_phys = mtk_iommu_iova_to_phys, 536 535 .add_device = mtk_iommu_add_device, 537 536 .remove_device = mtk_iommu_remove_device,

+2 -3

drivers/iommu/omap-iommu.c

··· 550 550 551 551 pte_ready: 552 552 iopte = iopte_offset(iopgd, da); 553 - *pt_dma = virt_to_phys(iopte); 553 + *pt_dma = iopgd_page_paddr(iopgd); 554 554 dev_vdbg(obj->dev, 555 555 "%s: da:%08x pgd:%p *pgd:%08x pte:%p *pte:%08x\n", 556 556 __func__, da, iopgd, *iopgd, iopte, *iopte); ··· 738 738 } 739 739 bytes *= nent; 740 740 memset(iopte, 0, nent * sizeof(*iopte)); 741 - pt_dma = virt_to_phys(iopte); 741 + pt_dma = iopgd_page_paddr(iopgd); 742 742 flush_iopte_range(obj->dev, pt_dma, pt_offset, nent); 743 743 744 744 /* ··· 1548 1548 .detach_dev = omap_iommu_detach_dev, 1549 1549 .map = omap_iommu_map, 1550 1550 .unmap = omap_iommu_unmap, 1551 - .map_sg = default_iommu_map_sg, 1552 1551 .iova_to_phys = omap_iommu_iova_to_phys, 1553 1552 .add_device = omap_iommu_add_device, 1554 1553 .remove_device = omap_iommu_remove_device,

-1

drivers/iommu/qcom_iommu.c

··· 590 590 .detach_dev = qcom_iommu_detach_dev, 591 591 .map = qcom_iommu_map, 592 592 .unmap = qcom_iommu_unmap, 593 - .map_sg = default_iommu_map_sg, 594 593 .flush_iotlb_all = qcom_iommu_iotlb_sync, 595 594 .iotlb_sync = qcom_iommu_iotlb_sync, 596 595 .iova_to_phys = qcom_iommu_iova_to_phys,

-1

drivers/iommu/rockchip-iommu.c

··· 1110 1110 .detach_dev = rk_iommu_detach_device, 1111 1111 .map = rk_iommu_map, 1112 1112 .unmap = rk_iommu_unmap, 1113 - .map_sg = default_iommu_map_sg, 1114 1113 .add_device = rk_iommu_add_device, 1115 1114 .remove_device = rk_iommu_remove_device, 1116 1115 .iova_to_phys = rk_iommu_iova_to_phys,

-1

drivers/iommu/tegra-gart.c

··· 377 377 .remove_device = gart_iommu_remove_device, 378 378 .device_group = generic_device_group, 379 379 .map = gart_iommu_map, 380 - .map_sg = default_iommu_map_sg, 381 380 .unmap = gart_iommu_unmap, 382 381 .iova_to_phys = gart_iommu_iova_to_phys, 383 382 .pgsize_bitmap = GART_IOMMU_PGSIZES,

-1

drivers/iommu/tegra-smmu.c

··· 876 876 .device_group = tegra_smmu_device_group, 877 877 .map = tegra_smmu_map, 878 878 .unmap = tegra_smmu_unmap, 879 - .map_sg = default_iommu_map_sg, 880 879 .iova_to_phys = tegra_smmu_iova_to_phys, 881 880 .of_xlate = tegra_smmu_of_xlate, 882 881 .pgsize_bitmap = SZ_4K,

-5

include/linux/dmar.h

··· 265 265 #define PDA_LOW_BIT 26 266 266 #define PDA_HIGH_BIT 32 267 267 268 - enum { 269 - IRQ_REMAP_XAPIC_MODE, 270 - IRQ_REMAP_X2APIC_MODE, 271 - }; 272 - 273 268 /* Can't use the common MSI interrupt functions 274 269 * since DMAR is not a pci device 275 270 */

+73 -7

include/linux/intel-iommu.h

··· 31 31 #include <linux/list.h> 32 32 #include <linux/iommu.h> 33 33 #include <linux/io-64-nonatomic-lo-hi.h> 34 + #include <linux/dmar.h> 34 35 35 36 #include <asm/cacheflush.h> 36 37 #include <asm/iommu.h> ··· 115 114 * Extended Capability Register 116 115 */ 117 116 117 + #define ecap_dit(e) ((e >> 41) & 0x1) 118 118 #define ecap_pasid(e) ((e >> 40) & 0x1) 119 119 #define ecap_pss(e) ((e >> 35) & 0x1f) 120 120 #define ecap_eafs(e) ((e >> 34) & 0x1) ··· 286 284 #define QI_DEV_IOTLB_SID(sid) ((u64)((sid) & 0xffff) << 32) 287 285 #define QI_DEV_IOTLB_QDEP(qdep) (((qdep) & 0x1f) << 16) 288 286 #define QI_DEV_IOTLB_ADDR(addr) ((u64)(addr) & VTD_PAGE_MASK) 287 + #define QI_DEV_IOTLB_PFSID(pfsid) (((u64)(pfsid & 0xf) << 12) | ((u64)(pfsid & 0xfff) << 52)) 289 288 #define QI_DEV_IOTLB_SIZE 1 290 289 #define QI_DEV_IOTLB_MAX_INVS 32 291 290 ··· 311 308 #define QI_DEV_EIOTLB_PASID(p) (((u64)p) << 32) 312 309 #define QI_DEV_EIOTLB_SID(sid) ((u64)((sid) & 0xffff) << 16) 313 310 #define QI_DEV_EIOTLB_QDEP(qd) ((u64)((qd) & 0x1f) << 4) 311 + #define QI_DEV_EIOTLB_PFSID(pfsid) (((u64)(pfsid & 0xf) << 12) | ((u64)(pfsid & 0xfff) << 52)) 314 312 #define QI_DEV_EIOTLB_MAX_INVS 32 315 313 316 314 #define QI_PGRP_IDX(idx) (((u64)(idx)) << 55) ··· 389 385 struct pasid_state_entry; 390 386 struct page_req_dsc; 391 387 388 + struct dmar_domain { 389 + int nid; /* node id */ 390 + 391 + unsigned iommu_refcnt[DMAR_UNITS_SUPPORTED]; 392 + /* Refcount of devices per iommu */ 393 + 394 + 395 + u16 iommu_did[DMAR_UNITS_SUPPORTED]; 396 + /* Domain ids per IOMMU. Use u16 since 397 + * domain ids are 16 bit wide according 398 + * to VT-d spec, section 9.3 */ 399 + 400 + bool has_iotlb_device; 401 + struct list_head devices; /* all devices' list */ 402 + struct iova_domain iovad; /* iova's that belong to this domain */ 403 + 404 + struct dma_pte *pgd; /* virtual address */ 405 + int gaw; /* max guest address width */ 406 + 407 + /* adjusted guest address width, 0 is level 2 30-bit */ 408 + int agaw; 409 + 410 + int flags; /* flags to find out type of domain */ 411 + 412 + int iommu_coherency;/* indicate coherency of iommu access */ 413 + int iommu_snooping; /* indicate snooping control feature*/ 414 + int iommu_count; /* reference count of iommu */ 415 + int iommu_superpage;/* Level of superpages supported: 416 + 0 == 4KiB (no superpages), 1 == 2MiB, 417 + 2 == 1GiB, 3 == 512GiB, 4 == 1TiB */ 418 + u64 max_addr; /* maximum mapped address */ 419 + 420 + struct iommu_domain domain; /* generic domain data structure for 421 + iommu core */ 422 + }; 423 + 392 424 struct intel_iommu { 393 425 void __iomem *reg; /* Pointer to hardware regs, virtual addr */ 394 426 u64 reg_phys; /* physical address of hw register set */ ··· 454 414 * devices away to userspace processes (e.g. for DPDK) and don't 455 415 * want to trust that userspace will use *only* the PASID it was 456 416 * told to. But while it's all driver-arbitrated, we're fine. */ 457 - struct pasid_entry *pasid_table; 458 417 struct pasid_state_entry *pasid_state_table; 459 418 struct page_req_dsc *prq; 460 419 unsigned char prq_name[16]; /* Name for PRQ interrupt */ 461 - struct idr pasid_idr; 462 420 u32 pasid_max; 463 421 #endif 464 422 struct q_inval *qi; /* Queued invalidation info */ ··· 470 432 struct iommu_device iommu; /* IOMMU core code handle */ 471 433 int node; 472 434 u32 flags; /* Software defined flags */ 435 + }; 436 + 437 + /* PCI domain-device relationship */ 438 + struct device_domain_info { 439 + struct list_head link; /* link to domain siblings */ 440 + struct list_head global; /* link to global list */ 441 + struct list_head table; /* link to pasid table */ 442 + u8 bus; /* PCI bus number */ 443 + u8 devfn; /* PCI devfn number */ 444 + u16 pfsid; /* SRIOV physical function source ID */ 445 + u8 pasid_supported:3; 446 + u8 pasid_enabled:1; 447 + u8 pri_supported:1; 448 + u8 pri_enabled:1; 449 + u8 ats_supported:1; 450 + u8 ats_enabled:1; 451 + u8 ats_qdep; 452 + struct device *dev; /* it's NULL for PCIe-to-PCI bridge */ 453 + struct intel_iommu *iommu; /* IOMMU used by this device */ 454 + struct dmar_domain *domain; /* pointer to domain */ 455 + struct pasid_table *pasid_table; /* pasid table */ 473 456 }; 474 457 475 458 static inline void __iommu_flush_cache( ··· 512 453 u8 fm, u64 type); 513 454 extern void qi_flush_iotlb(struct intel_iommu *iommu, u16 did, u64 addr, 514 455 unsigned int size_order, u64 type); 515 - extern void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 qdep, 516 - u64 addr, unsigned mask); 517 - 456 + extern void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 pfsid, 457 + u16 qdep, u64 addr, unsigned mask); 518 458 extern int qi_submit_sync(struct qi_desc *desc, struct intel_iommu *iommu); 519 459 520 460 extern int dmar_ir_support(void); 521 461 462 + struct dmar_domain *get_valid_domain_for_dev(struct device *dev); 463 + void *alloc_pgtable_page(int node); 464 + void free_pgtable_page(void *vaddr); 465 + struct intel_iommu *domain_get_iommu(struct dmar_domain *domain); 466 + int for_each_device_domain(int (*fn)(struct device_domain_info *info, 467 + void *data), void *data); 468 + 522 469 #ifdef CONFIG_INTEL_IOMMU_SVM 523 - extern int intel_svm_alloc_pasid_tables(struct intel_iommu *iommu); 524 - extern int intel_svm_free_pasid_tables(struct intel_iommu *iommu); 470 + int intel_svm_init(struct intel_iommu *iommu); 471 + int intel_svm_exit(struct intel_iommu *iommu); 525 472 extern int intel_svm_enable_prq(struct intel_iommu *iommu); 526 473 extern int intel_svm_finish_prq(struct intel_iommu *iommu); 527 474 ··· 551 486 int flags; 552 487 int pasid; 553 488 struct list_head devs; 489 + struct list_head list; 554 490 }; 555 491 556 492 extern int intel_iommu_enable_pasid(struct intel_iommu *iommu, struct intel_svm_dev *sdev);

+9 -14

include/linux/iommu.h

··· 166 166 * @detach_dev: detach device from an iommu domain 167 167 * @map: map a physically contiguous memory region to an iommu domain 168 168 * @unmap: unmap a physically contiguous memory region from an iommu domain 169 - * @map_sg: map a scatter-gather list of physically contiguous memory chunks 170 - * to an iommu domain 171 169 * @flush_tlb_all: Synchronously flush all hardware TLBs for this domain 172 170 * @tlb_range_add: Add a given iova range to the flush queue for this domain 173 171 * @tlb_sync: Flush all queued ranges from the hardware TLBs and empty flush ··· 199 201 phys_addr_t paddr, size_t size, int prot); 200 202 size_t (*unmap)(struct iommu_domain *domain, unsigned long iova, 201 203 size_t size); 202 - size_t (*map_sg)(struct iommu_domain *domain, unsigned long iova, 203 - struct scatterlist *sg, unsigned int nents, int prot); 204 204 void (*flush_iotlb_all)(struct iommu_domain *domain); 205 205 void (*iotlb_range_add)(struct iommu_domain *domain, 206 206 unsigned long iova, size_t size); ··· 299 303 size_t size); 300 304 extern size_t iommu_unmap_fast(struct iommu_domain *domain, 301 305 unsigned long iova, size_t size); 302 - extern size_t default_iommu_map_sg(struct iommu_domain *domain, unsigned long iova, 303 - struct scatterlist *sg,unsigned int nents, 304 - int prot); 306 + extern size_t iommu_map_sg(struct iommu_domain *domain, unsigned long iova, 307 + struct scatterlist *sg,unsigned int nents, int prot); 305 308 extern phys_addr_t iommu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova); 306 309 extern void iommu_set_fault_handler(struct iommu_domain *domain, 307 310 iommu_fault_handler_t handler, void *token); ··· 371 376 { 372 377 if (domain->ops->iotlb_sync) 373 378 domain->ops->iotlb_sync(domain); 374 - } 375 - 376 - static inline size_t iommu_map_sg(struct iommu_domain *domain, 377 - unsigned long iova, struct scatterlist *sg, 378 - unsigned int nents, int prot) 379 - { 380 - return domain->ops->map_sg(domain, iova, sg, nents, prot); 381 379 } 382 380 383 381 /* PCI device grouping function */ ··· 685 697 } 686 698 687 699 #endif /* CONFIG_IOMMU_API */ 700 + 701 + #ifdef CONFIG_IOMMU_DEBUGFS 702 + extern struct dentry *iommu_debugfs_dir; 703 + void iommu_debugfs_setup(void); 704 + #else 705 + static inline void iommu_debugfs_setup(void) {} 706 + #endif 688 707 689 708 #endif /* __LINUX_IOMMU_H */

-18

include/linux/platform_data/sh_ipmmu.h

··· 1 - /* sh_ipmmu.h 2 - * 3 - * Copyright (C) 2012 Hideki EIRAKU 4 - * 5 - * This program is free software; you can redistribute it and/or modify 6 - * it under the terms of the GNU General Public License as published by 7 - * the Free Software Foundation; version 2 of the License. 8 - */ 9 - 10 - #ifndef __SH_IPMMU_H__ 11 - #define __SH_IPMMU_H__ 12 - 13 - struct shmobile_ipmmu_platform_data { 14 - const char * const *dev_names; 15 - unsigned int num_dev_names; 16 - }; 17 - 18 - #endif /* __SH_IPMMU_H__ */

Configure Feed

Configure Feed