Merge tag 'iommu-fixes-v5.9-rc3' of git://git.kernel.org/pub/scm/linux/kernel/git/joro/iommu

+1 -1

MAINTAINERS

··· 14388 14388 L: iommu@lists.linux-foundation.org 14389 14389 L: linux-arm-msm@vger.kernel.org 14390 14390 S: Maintained 14391 - F: drivers/iommu/qcom_iommu.c 14391 + F: drivers/iommu/arm/arm-smmu/qcom_iommu.c 14392 14392 14393 14393 QUALCOMM IPCC MAILBOX DRIVER 14394 14394 M: Manivannan Sadhasivam <manivannan.sadhasivam@linaro.org>

+1 -1

drivers/iommu/amd/Kconfig

··· 10 10 select IOMMU_API 11 11 select IOMMU_IOVA 12 12 select IOMMU_DMA 13 - depends on X86_64 && PCI && ACPI 13 + depends on X86_64 && PCI && ACPI && HAVE_CMPXCHG_DOUBLE 14 14 help 15 15 With this option you can enable support for AMD IOMMU hardware in 16 16 your system. An IOMMU is a hardware component which provides

+19 -2

drivers/iommu/amd/init.c

··· 1511 1511 iommu->mmio_phys_end = MMIO_REG_END_OFFSET; 1512 1512 else 1513 1513 iommu->mmio_phys_end = MMIO_CNTR_CONF_OFFSET; 1514 - if (((h->efr_attr & (0x1 << IOMMU_FEAT_GASUP_SHIFT)) == 0)) 1514 + 1515 + /* 1516 + * Note: GA (128-bit IRTE) mode requires cmpxchg16b supports. 1517 + * GAM also requires GA mode. Therefore, we need to 1518 + * check cmpxchg16b support before enabling it. 1519 + */ 1520 + if (!boot_cpu_has(X86_FEATURE_CX16) || 1521 + ((h->efr_attr & (0x1 << IOMMU_FEAT_GASUP_SHIFT)) == 0)) 1515 1522 amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_LEGACY; 1516 1523 break; 1517 1524 case 0x11: ··· 1527 1520 iommu->mmio_phys_end = MMIO_REG_END_OFFSET; 1528 1521 else 1529 1522 iommu->mmio_phys_end = MMIO_CNTR_CONF_OFFSET; 1530 - if (((h->efr_reg & (0x1 << IOMMU_EFR_GASUP_SHIFT)) == 0)) 1523 + 1524 + /* 1525 + * Note: GA (128-bit IRTE) mode requires cmpxchg16b supports. 1526 + * XT, GAM also requires GA mode. Therefore, we need to 1527 + * check cmpxchg16b support before enabling them. 1528 + */ 1529 + if (!boot_cpu_has(X86_FEATURE_CX16) || 1530 + ((h->efr_reg & (0x1 << IOMMU_EFR_GASUP_SHIFT)) == 0)) { 1531 1531 amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_LEGACY; 1532 + break; 1533 + } 1534 + 1532 1535 /* 1533 1536 * Note: Since iommu_update_intcapxt() leverages 1534 1537 * the IOMMU MMIO access to MSI capability block registers

+21 -5

drivers/iommu/amd/iommu.c

··· 2659 2659 if (!dev_data) 2660 2660 return 0; 2661 2661 2662 - if (dev_data->iommu_v2) 2662 + /* 2663 + * Do not identity map IOMMUv2 capable devices when memory encryption is 2664 + * active, because some of those devices (AMD GPUs) don't have the 2665 + * encryption bit in their DMA-mask and require remapping. 2666 + */ 2667 + if (!mem_encrypt_active() && dev_data->iommu_v2) 2663 2668 return IOMMU_DOMAIN_IDENTITY; 2664 2669 2665 2670 return 0; ··· 3297 3292 static int modify_irte_ga(u16 devid, int index, struct irte_ga *irte, 3298 3293 struct amd_ir_data *data) 3299 3294 { 3295 + bool ret; 3300 3296 struct irq_remap_table *table; 3301 3297 struct amd_iommu *iommu; 3302 3298 unsigned long flags; ··· 3315 3309 3316 3310 entry = (struct irte_ga *)table->table; 3317 3311 entry = &entry[index]; 3318 - entry->lo.fields_remap.valid = 0; 3319 - entry->hi.val = irte->hi.val; 3320 - entry->lo.val = irte->lo.val; 3321 - entry->lo.fields_remap.valid = 1; 3312 + 3313 + ret = cmpxchg_double(&entry->lo.val, &entry->hi.val, 3314 + entry->lo.val, entry->hi.val, 3315 + irte->lo.val, irte->hi.val); 3316 + /* 3317 + * We use cmpxchg16 to atomically update the 128-bit IRTE, 3318 + * and it cannot be updated by the hardware or other processors 3319 + * behind us, so the return value of cmpxchg16 should be the 3320 + * same as the old value. 3321 + */ 3322 + WARN_ON(!ret); 3323 + 3322 3324 if (data) 3323 3325 data->ref = entry; 3324 3326 ··· 3864 3850 struct amd_ir_data *ir_data = (struct amd_ir_data *)data; 3865 3851 struct irte_ga *entry = (struct irte_ga *) ir_data->entry; 3866 3852 struct irq_cfg *cfg = ir_data->cfg; 3853 + u64 valid = entry->lo.fields_remap.valid; 3867 3854 3868 3855 if (!AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir) || 3869 3856 !entry || !entry->lo.fields_vapic.guest_mode) ··· 3873 3858 entry->lo.val = 0; 3874 3859 entry->hi.val = 0; 3875 3860 3861 + entry->lo.fields_remap.valid = valid; 3876 3862 entry->lo.fields_remap.dm = apic->irq_dest_mode; 3877 3863 entry->lo.fields_remap.int_type = apic->irq_delivery_mode; 3878 3864 entry->hi.fields.vector = cfg->vector;

+7

drivers/iommu/amd/iommu_v2.c

··· 737 737 738 738 might_sleep(); 739 739 740 + /* 741 + * When memory encryption is active the device is likely not in a 742 + * direct-mapped domain. Forbid using IOMMUv2 functionality for now. 743 + */ 744 + if (mem_encrypt_active()) 745 + return -ENODEV; 746 + 740 747 if (!amd_iommu_v2_supported()) 741 748 return -ENODEV; 742 749

+62 -52

drivers/iommu/intel/iommu.c

··· 123 123 return (level - 1) * LEVEL_STRIDE; 124 124 } 125 125 126 - static inline int pfn_level_offset(unsigned long pfn, int level) 126 + static inline int pfn_level_offset(u64 pfn, int level) 127 127 { 128 128 return (pfn >> level_to_offset_bits(level)) & LEVEL_MASK; 129 129 } 130 130 131 - static inline unsigned long level_mask(int level) 131 + static inline u64 level_mask(int level) 132 132 { 133 - return -1UL << level_to_offset_bits(level); 133 + return -1ULL << level_to_offset_bits(level); 134 134 } 135 135 136 - static inline unsigned long level_size(int level) 136 + static inline u64 level_size(int level) 137 137 { 138 - return 1UL << level_to_offset_bits(level); 138 + return 1ULL << level_to_offset_bits(level); 139 139 } 140 140 141 - static inline unsigned long align_to_level(unsigned long pfn, int level) 141 + static inline u64 align_to_level(u64 pfn, int level) 142 142 { 143 143 return (pfn + level_size(level) - 1) & level_mask(level); 144 144 } 145 145 146 146 static inline unsigned long lvl_to_nr_pages(unsigned int lvl) 147 147 { 148 - return 1 << min_t(int, (lvl - 1) * LEVEL_STRIDE, MAX_AGAW_PFN_WIDTH); 148 + return 1UL << min_t(int, (lvl - 1) * LEVEL_STRIDE, MAX_AGAW_PFN_WIDTH); 149 149 } 150 150 151 151 /* VT-d pages must always be _smaller_ than MM pages. Otherwise things ··· 364 364 int intel_iommu_gfx_mapped; 365 365 EXPORT_SYMBOL_GPL(intel_iommu_gfx_mapped); 366 366 367 - #define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1)) 368 367 #define DEFER_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-2)) 369 368 struct device_domain_info *get_domain_info(struct device *dev) 370 369 { ··· 373 374 return NULL; 374 375 375 376 info = dev_iommu_priv_get(dev); 376 - if (unlikely(info == DUMMY_DEVICE_DOMAIN_INFO || 377 - info == DEFER_DEVICE_DOMAIN_INFO)) 377 + if (unlikely(info == DEFER_DEVICE_DOMAIN_INFO)) 378 378 return NULL; 379 379 380 380 return info; ··· 740 742 return &context[devfn]; 741 743 } 742 744 743 - static int iommu_dummy(struct device *dev) 744 - { 745 - return dev_iommu_priv_get(dev) == DUMMY_DEVICE_DOMAIN_INFO; 746 - } 747 - 748 745 static bool attach_deferred(struct device *dev) 749 746 { 750 747 return dev_iommu_priv_get(dev) == DEFER_DEVICE_DOMAIN_INFO; ··· 772 779 return false; 773 780 } 774 781 782 + static bool quirk_ioat_snb_local_iommu(struct pci_dev *pdev) 783 + { 784 + struct dmar_drhd_unit *drhd; 785 + u32 vtbar; 786 + int rc; 787 + 788 + /* We know that this device on this chipset has its own IOMMU. 789 + * If we find it under a different IOMMU, then the BIOS is lying 790 + * to us. Hope that the IOMMU for this device is actually 791 + * disabled, and it needs no translation... 792 + */ 793 + rc = pci_bus_read_config_dword(pdev->bus, PCI_DEVFN(0, 0), 0xb0, &vtbar); 794 + if (rc) { 795 + /* "can't" happen */ 796 + dev_info(&pdev->dev, "failed to run vt-d quirk\n"); 797 + return false; 798 + } 799 + vtbar &= 0xffff0000; 800 + 801 + /* we know that the this iommu should be at offset 0xa000 from vtbar */ 802 + drhd = dmar_find_matched_drhd_unit(pdev); 803 + if (!drhd || drhd->reg_base_addr - vtbar != 0xa000) { 804 + pr_warn_once(FW_BUG "BIOS assigned incorrect VT-d unit for Intel(R) QuickData Technology device\n"); 805 + add_taint(TAINT_FIRMWARE_WORKAROUND, LOCKDEP_STILL_OK); 806 + return true; 807 + } 808 + 809 + return false; 810 + } 811 + 812 + static bool iommu_is_dummy(struct intel_iommu *iommu, struct device *dev) 813 + { 814 + if (!iommu || iommu->drhd->ignored) 815 + return true; 816 + 817 + if (dev_is_pci(dev)) { 818 + struct pci_dev *pdev = to_pci_dev(dev); 819 + 820 + if (pdev->vendor == PCI_VENDOR_ID_INTEL && 821 + pdev->device == PCI_DEVICE_ID_INTEL_IOAT_SNB && 822 + quirk_ioat_snb_local_iommu(pdev)) 823 + return true; 824 + } 825 + 826 + return false; 827 + } 828 + 775 829 struct intel_iommu *device_to_iommu(struct device *dev, u8 *bus, u8 *devfn) 776 830 { 777 831 struct dmar_drhd_unit *drhd = NULL; ··· 828 788 u16 segment = 0; 829 789 int i; 830 790 831 - if (!dev || iommu_dummy(dev)) 791 + if (!dev) 832 792 return NULL; 833 793 834 794 if (dev_is_pci(dev)) { ··· 845 805 dev = &ACPI_COMPANION(dev)->dev; 846 806 847 807 rcu_read_lock(); 848 - for_each_active_iommu(iommu, drhd) { 808 + for_each_iommu(iommu, drhd) { 849 809 if (pdev && segment != drhd->segment) 850 810 continue; 851 811 ··· 881 841 } 882 842 iommu = NULL; 883 843 out: 844 + if (iommu_is_dummy(iommu, dev)) 845 + iommu = NULL; 846 + 884 847 rcu_read_unlock(); 885 848 886 849 return iommu; ··· 2490 2447 { 2491 2448 struct device_domain_info *info; 2492 2449 2493 - if (unlikely(attach_deferred(dev) || iommu_dummy(dev))) 2450 + if (unlikely(attach_deferred(dev))) 2494 2451 return NULL; 2495 2452 2496 2453 /* No lock here, assumes no domain exit in normal case */ ··· 4032 3989 iova_cache_put(); 4033 3990 } 4034 3991 4035 - static void quirk_ioat_snb_local_iommu(struct pci_dev *pdev) 4036 - { 4037 - struct dmar_drhd_unit *drhd; 4038 - u32 vtbar; 4039 - int rc; 4040 - 4041 - /* We know that this device on this chipset has its own IOMMU. 4042 - * If we find it under a different IOMMU, then the BIOS is lying 4043 - * to us. Hope that the IOMMU for this device is actually 4044 - * disabled, and it needs no translation... 4045 - */ 4046 - rc = pci_bus_read_config_dword(pdev->bus, PCI_DEVFN(0, 0), 0xb0, &vtbar); 4047 - if (rc) { 4048 - /* "can't" happen */ 4049 - dev_info(&pdev->dev, "failed to run vt-d quirk\n"); 4050 - return; 4051 - } 4052 - vtbar &= 0xffff0000; 4053 - 4054 - /* we know that the this iommu should be at offset 0xa000 from vtbar */ 4055 - drhd = dmar_find_matched_drhd_unit(pdev); 4056 - if (!drhd || drhd->reg_base_addr - vtbar != 0xa000) { 4057 - pr_warn_once(FW_BUG "BIOS assigned incorrect VT-d unit for Intel(R) QuickData Technology device\n"); 4058 - add_taint(TAINT_FIRMWARE_WORKAROUND, LOCKDEP_STILL_OK); 4059 - dev_iommu_priv_set(&pdev->dev, DUMMY_DEVICE_DOMAIN_INFO); 4060 - } 4061 - } 4062 - DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB, quirk_ioat_snb_local_iommu); 4063 - 4064 3992 static void __init init_no_remapping_devices(void) 4065 3993 { 4066 3994 struct dmar_drhd_unit *drhd; ··· 4063 4049 /* This IOMMU has *only* gfx devices. Either bypass it or 4064 4050 set the gfx_mapped flag, as appropriate */ 4065 4051 drhd->gfx_dedicated = 1; 4066 - if (!dmar_map_gfx) { 4052 + if (!dmar_map_gfx) 4067 4053 drhd->ignored = 1; 4068 - for_each_active_dev_scope(drhd->devices, 4069 - drhd->devices_cnt, i, dev) 4070 - dev_iommu_priv_set(dev, DUMMY_DEVICE_DOMAIN_INFO); 4071 - } 4072 4054 } 4073 4055 } 4074 4056

+8 -2

drivers/iommu/intel/irq_remapping.c

··· 508 508 509 509 /* Enable interrupt-remapping */ 510 510 iommu->gcmd |= DMA_GCMD_IRE; 511 - iommu->gcmd &= ~DMA_GCMD_CFI; /* Block compatibility-format MSIs */ 512 511 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG); 513 - 514 512 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, 515 513 readl, (sts & DMA_GSTS_IRES), sts); 514 + 515 + /* Block compatibility-format MSIs */ 516 + if (sts & DMA_GSTS_CFIS) { 517 + iommu->gcmd &= ~DMA_GCMD_CFI; 518 + writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG); 519 + IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, 520 + readl, !(sts & DMA_GSTS_CFIS), sts); 521 + } 516 522 517 523 /* 518 524 * With CFI clear in the Global Command register, we should be

Configure Feed

Configure Feed